Annotation of parser3/src/main/pa_file.C, revision 1.7
1.1 moko 1: /** @file
2: Parser: file and file system related functions.
3:
4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
6: */
7:
8: #include "pa_file.h"
9: #include "pa_dir.h"
10: #include "pa_charsets.h"
11: #include "pa_http.h"
12: #include "pa_convert_utf.h"
13:
14: #ifdef _MSC_VER
15: #include <windows.h>
16: #include <direct.h>
17: #endif
18:
1.7 ! moko 19: volatile const char * IDENT_PA_FILE_C="$Id: pa_file.C,v 1.6 2026/09/30 16:44:20 moko Exp $" IDENT_PA_FILE_H;
1.1 moko 20:
21: // some maybe-undefined constants
22:
23: #ifndef _O_TEXT
24: # define _O_TEXT 0
25: #endif
26: #ifndef _O_BINARY
27: # define _O_BINARY 0
28: #endif
29:
30: #ifdef HAVE_FTRUNCATE
31: # define PA_O_TRUNC 0
32: #else
33: # ifdef _O_TRUNC
34: # define PA_O_TRUNC _O_TRUNC
35: # else
36: # error you must have either ftruncate function or _O_TRUNC bit declared
37: # endif
38: #endif
39:
40: #ifdef WIN32
41:
42: bool is_os_absolute_path(const String& path) {
43: return is_os_absolute_path(path.cstr());
44: }
45:
46: bool is_os_absolute_path(const char* path) {
47: return path[0] && path[0]!=':' && path[1]==':' // DRIVE: (including drive-relative paths, for compatibility)
1.4 moko 48: || path[0]=='\\' && path[1]=='\\'; // UNC, only '\\' is supported as '//' is a document root relative path
1.1 moko 49: }
50: #endif
51:
1.5 moko 52: uint path_scheme(const String& path) {
53: if(path.starts_with("file://"))
54: return PA_SCHEME_FILE;
55: if(path.starts_with("http://"))
56: return PA_SCHEME_HTTP;
57: if(path.starts_with("parser://"))
58: return PA_SCHEME_PARSER;
59: return PA_SCHEME_NONE;
60: }
61:
62: void check_path_scheme(uint scheme, const String& path, uint allowed) {
63: if(scheme!=PA_SCHEME_FILE && !(allowed&scheme))
64: throw Exception(PARSER_RUNTIME, &path, "unsupported path scheme");
65: }
66:
67: const String* file_uri_to_path(const String& uri) {
68: const String* rest=&uri.mid(7 /* "file://" */, uri.length());
69: if(rest->starts_with("localhost/"))
70: rest=&rest->mid(9 /* "localhost", keeping the slash */, rest->length());
71: if(rest->first_char()=='/') {
72: // file:///C:/path and file://localhost/C:/path: the slash before a drive name is not a part of the path
73: const String& disk=rest->mid(1, rest->length());
74: return is_os_absolute_path(disk) ? &disk : rest;
75: }
76: if(is_os_absolute_path(*rest))
77: return rest; // file://C:/path, not a standard form but in use
1.7 ! moko 78: #ifdef WIN32
! 79: if(rest->pos('/')!=STRING_NOT_FOUND) { // file://server/share/path is the UNC path \\server\share\path
! 80: String& result=*new String("//");
! 81: result << *rest;
! 82: return &result;
! 83: }
! 84: #endif
1.5 moko 85: return 0;
86: }
87:
1.1 moko 88: // externs
89:
90: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset);
91:
92: // functions
93:
94: #ifdef _MSC_VER
95:
96: #define PA_UTF16_ENC(value) (const wchar_t *)pa_utf16_encode(value, pa_thread_request().charsets.source())
97:
98: int pa_stat(const char *pathname, struct stat *buffer){
99: return _wstat64(PA_UTF16_ENC(pathname), buffer);
100: }
101:
102: int pa_open(const char *pathname, int flags, int mode){
103: return _wopen(PA_UTF16_ENC(pathname), flags, mode);
104: }
105:
106: FILE *pa_fopen(const char *pathname, const char *mode){
107: return _wfopen(PA_UTF16_ENC(pathname), PA_UTF16_ENC(mode));
108: }
109:
110: int pa_mkdir(const char *pathname, int){
111: return _wmkdir(PA_UTF16_ENC(pathname));
112: }
113:
114: int pa_rmdir(const char *pathname){
115: return _wrmdir(PA_UTF16_ENC(pathname));
116: }
117:
118: int pa_rename(const char *oldpath, const char *newpath){
119: return _wrename(PA_UTF16_ENC(oldpath), PA_UTF16_ENC(newpath));
120: }
121:
122: int pa_unlink(const char *pathname){
123: return _wunlink(PA_UTF16_ENC(pathname));
124: }
125:
126: #else
127:
128: #define pa_mkdir mkdir
129: #define pa_rmdir rmdir
130: #define pa_rename rename
131: #define pa_unlink unlink
132:
133: #endif
134:
135: /// these options were handled but not checked elsewhere, now check them
136: int pa_get_valid_file_options_count(HashStringValue& options) {
137: int result=0;
138: if(options.get(PA_SQL_LIMIT_NAME))
139: result++;
140: if(options.get(PA_SQL_OFFSET_NAME))
141: result++;
142: if(options.get(PA_COLUMN_SEPARATOR_NAME))
143: result++;
144: if(options.get(PA_COLUMN_ENCLOSER_NAME))
145: result++;
146: if(options.get(PA_CHARSET_NAME))
147: result++;
148: return result;
149: }
150:
151: #ifndef DOXYGEN
152: struct File_read_action_info {
153: char **data; size_t *data_size;
154: char* buf; uint64_t offset; size_t limit;
1.3 moko 155: };
1.1 moko 156: #endif
157:
158: static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) {
1.3 moko 159: File_read_action_info& info = *static_cast<File_read_action_info *>(context);
1.1 moko 160: size_t to_read_size = check_file_size(info.limit && info.limit < (size_t)finfo.st_size ? info.limit : (size_t)finfo.st_size, &file_spec);
161: if(to_read_size) {
162: if(info.offset)
163: pa_lseek(f, info.offset, SEEK_SET); // seek never fails as POSIX allows the file offset to be set beyond the EOF
164: *info.data = info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1);
165: ssize_t result = read(f, *info.data, to_read_size);
166: if(result<0)
167: throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno);
168: *info.data_size = result;
169: } else { // empty file
170: // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files
171: *info.data = (char *)pa_malloc_atomic(1);
172: *(char*)(*info.data) = 0;
173: *info.data_size = 0;
174: return;
175: }
176: }
177:
178: File_read_result file_read_binary(const String& file_spec, bool fail_on_read_problem, char* buf, uint64_t offset, size_t limit) {
179: File_read_result result = {false, 0, 0, 0};
180: File_read_action_info info = {&result.str, &result.length, buf, offset, limit};
181:
182: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, 0, fail_on_read_problem);
183: return result;
184: }
185:
1.3 moko 186: File_read_result file_read(Request_charsets& charsets, const String& file_spec,
1.1 moko 187: bool as_text, HashStringValue *options,
188: bool fail_on_read_problem,
189: size_t offset = 0, size_t limit = 0, bool transcode_text_result = true) {
190: File_read_result result = {false, 0, 0, 0};
191: if(options){
192: int valid_options = pa_get_valid_file_options_count(*options);
193: if(valid_options != options->count())
194: throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
195: }
196:
1.3 moko 197: File_read_action_info info = {&result.str, &result.length, 0, offset, limit};
1.1 moko 198:
1.3 moko 199: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem);
1.1 moko 200:
201: if(as_text){
202: if(result.success){
203: Charset* asked_charset = 0;
204: if(options)
205: if(Value* vcharset_name = options->get(PA_CHARSET_NAME))
206: asked_charset = &pa_charsets.get(vcharset_name->as_string());
207:
208: asked_charset = pa_charsets.checkBOM(result.str, result.length, asked_charset);
209:
210: if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
211: String::C body = String::C(result.str, result.length);
212: body=Charset::transcode(body, *asked_charset, charsets.source());
213:
214: result.str = const_cast<char*>(body.str); // hacking a little
215: result.length = body.length;
216: }
217: }
218: if(result.length)
219: fix_line_breaks(result.str, result.length);
220: }
221:
222: return result;
223: }
224:
1.3 moko 225: File_read_result file_load(Request& r, const String& file_spec,
1.1 moko 226: bool as_text, HashStringValue *options,
227: bool fail_on_read_problem,
228: bool transcode_text_result) {
229:
230: size_t offset = 0;
231: size_t limit = 0;
232:
233: if(options){
234: if(Value *voffset = (Value *)options->get(sql_offset_name))
235: offset = r.process(*voffset).as_int();
236: if(Value *vlimit = (Value *)options->get(sql_limit_name))
237: limit = r.process(*vlimit).as_int();
238: // no check on options count here
239: }
240:
241: if(file_spec.starts_with("http://")) {
242: if(offset || limit)
243: throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load");
244:
245: // fail on read problem
246: File_read_http_result http = pa_internal_file_read_http(r, file_spec, as_text, options, transcode_text_result);
247:
248: File_read_result result = {true, http.str, http.length, http.headers};
249: return result;
250: } else
251: return file_read(r.charsets, file_spec, as_text, options, fail_on_read_problem, offset, limit, transcode_text_result);
252: }
253:
254: char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem) {
255: File_read_result file = file_read(charsets, file_spec, true, 0, fail_on_read_problem);
256: return file.success ? file.str : 0;
257: }
258:
259: char* file_load_text(Request& r, const String& file_spec, bool fail_on_read_problem, HashStringValue* options, bool transcode_result) {
260: File_read_result file = file_load(r, file_spec, true, options, fail_on_read_problem, transcode_result);
261: return file.success ? file.str : 0;
262: }
263:
264: #ifdef PA_SAFE_MODE
265: void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) {
266: if(finfo.st_uid/*foreign?*/!=geteuid()
267: && finfo.st_gid/*foreign?*/!=getegid())
268: throw Exception(PARSER_RUNTIME,
269: &file_spec,
270: "parser is in safe mode: reading files of foreign group and user disabled "
271: "[recompile parser with --disable-safe-mode configure option], "
272: "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)",
273: fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid()
274: );
275: }
276: #else
277: void check_safe_mode(struct stat, const String&, const char*) {
278: }
279: #endif
280:
281:
282: bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context,
283: bool as_text, bool fail_on_read_problem) {
284:
285: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
286: int f;
287:
288: // first open, next stat:
289: // directory update of NTFS hard links performed on open.
1.3 moko 290: // ex:
1.1 moko 291: // a.html:^test[] and b.html hardlink to a.html
292: // user inserts ! before ^test in a.html
1.3 moko 293: // directory entry of b.html in NTFS not updated at once,
1.1 moko 294: // they delay update till open, so we would receive "!^test[" string
295: // if would do stat, next open.
296: // later: it seems, even this does not help sometimes
297: if((f=pa_open(fname, O_RDONLY | (as_text ? _O_TEXT : _O_BINARY) ))>=0) {
298: try {
299: int pa_errno=pa_lock_shared_blocking(f);
300: if(pa_errno!=0)
301: throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
302:
303: struct stat finfo;
304: if(pa_fstat(f, &finfo)!=0)
305: throw Exception("file.missing", // hardly possible: we just opened it OK
306: &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
307:
308: check_safe_mode(finfo, file_spec, fname);
309:
1.3 moko 310: action(finfo, f, file_spec, context);
1.1 moko 311: } catch(...) {
1.3 moko 312: pa_unlock(f);close(f);
1.1 moko 313: if(fail_on_read_problem)
314: rethrow;
315: return false;
1.3 moko 316: }
1.1 moko 317:
1.3 moko 318: pa_unlock(f);close(f);
1.1 moko 319: return true;
320: } else {
321: if(fail_on_read_problem)
322: throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0,
323: &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
324: return false;
325: }
326: }
327:
328: void create_dir_for_file(const String& file_spec) {
329: const char *str=file_spec.taint_cstr(String::L_FILE_SPEC);
330: if(str[0]){
331: const char *pos=str+1;
1.6 moko 332: while((pos=strchr(pos, '/')) && pos[1]) { // to avoid trailing /, see #1166
1.1 moko 333: pa_mkdir(pa_strdup(str,pos-str), 0775);
334: pos++;
335: }
336: }
337: }
338:
339: bool file_write_action_under_lock(const String& file_spec, const char* action_name, File_write_action action, void *context,
340: bool as_text, bool do_append, bool do_block, bool fail_on_lock_problem) {
341:
342: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
343: int f;
344: if(access(fname, W_OK)!=0) // no
345: create_dir_for_file(file_spec);
346:
347: if((f=pa_open(fname, O_CREAT | O_RDWR | (as_text ? _O_TEXT : _O_BINARY) | (do_append ? O_APPEND : PA_O_TRUNC), 0664))>=0) {
348: int pa_errno=do_block ? pa_lock_exclusive_blocking(f) : pa_lock_exclusive_nonblocking(f);
349: if(pa_errno!=0) {
350: Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
351: close(f);
352: if(fail_on_lock_problem)
353: throw e;
354: return false;
355: }
356:
357: try {
358: #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE))
359: struct stat finfo;
360: if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111)
361: fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any
362: #endif
363: action(f, context);
364: } catch(...) {
365: #ifdef HAVE_FTRUNCATE
366: if(!do_append)
367: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // one cannot use O_TRUNC, read lower
368: #endif
369: pa_unlock(f);close(f);
370: rethrow;
371: }
372:
373: #ifdef HAVE_FTRUNCATE
374: if(!do_append)
375: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // O_TRUNC truncates even exclusevely write-locked file [thanks to Igor Milyakov <virtan@rotabanner.com> for discovering]
376: #endif
377: pa_unlock(f);close(f);
378: return true;
379: } else
380: throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
381: // here should be nothing, see rethrow above
382: }
383:
384: #ifndef DOXYGEN
385: struct File_write_action_info {
386: const char* str;
387: size_t length;
388: };
389: #endif
390:
391: // a single write() must stay below INT_MAX as windows _write() reports written bytes as int
392: #define FILE_WRITE_CHUNK_SIZE (256*0x100000)
393:
394: static const char* bytes_left(size_t rest) {
395: return rest ? pa_strcat(", ", pa_uitoa(rest), " bytes left") : "";
396: }
397:
398: static void file_write_action(int f, void *context) {
399: File_write_action_info& info=*static_cast<File_write_action_info *>(context);
400:
401: const char* data=info.str;
402: size_t left=info.length;
403:
404: while(left) {
405: size_t portion=min(left, (size_t)FILE_WRITE_CHUNK_SIZE);
406: ssize_t written=write(f, data, portion);
407: if(written<0)
408: throw Exception("file.write", 0, "error writing %s bytes: %s (%d)%s", pa_uitoa(portion), strerror(errno), errno, bytes_left(left-portion));
409: if(written==0)
410: throw Exception("file.write", 0, "error writing %s bytes: zero bytes written%s", pa_uitoa(portion), bytes_left(left-portion));
411: data+=written;
412: left-=written;
413: }
414: }
415:
416: void file_write(
417: Request_charsets& charsets,
418: const String& file_spec,
419: const char* data,
1.3 moko 420: size_t size,
421: bool as_text,
1.1 moko 422: bool do_append,
423: Charset* asked_charset) {
424:
425: if(as_text && asked_charset){
426: String::C body=String::C(data, size);
427: body=Charset::transcode(body, charsets.source(), *asked_charset);
428: data=body.str;
429: size=body.length;
430: };
431:
432: File_write_action_info info={data, size};
433:
434: file_write_action_under_lock(
435: file_spec,
436: "write",
437: file_write_action,
438: &info,
439: as_text,
440: do_append);
441: }
442:
443: static size_t get_dir(char* fname, size_t helper_length){
444: bool dir=false;
445: size_t pos=0;
446: for(pos=helper_length; pos; pos--){
447: char c=fname[pos-1];
1.6 moko 448: if(c=='/'){
1.1 moko 449: fname[pos-1]=0;
450: dir=true;
451: } else if(dir) break;
452: }
453: return pos;
454: }
455:
456: bool entry_exists(const char* fname, struct stat *afinfo) {
457: struct stat lfinfo;
458: bool result=pa_stat(fname, &lfinfo)==0;
459: if(afinfo)
460: *afinfo=lfinfo;
461: return result;
462: }
463:
464: bool entry_exists(const String& file_spec) {
1.3 moko 465: return entry_exists(file_spec.taint_cstr(String::L_FILE_SPEC), 0);
1.1 moko 466: }
467:
468: static bool entry_ifdir(char *fname, bool need_dir) {
469: if(need_dir){
470: size_t size=strlen(fname);
471: while(size) {
472: char c=fname[size-1];
1.6 moko 473: if(c=='/')
1.1 moko 474: fname[--size]=0;
475: else
476: break;
477: }
478: }
479:
480: struct stat finfo;
481: if(entry_exists(fname, &finfo)) {
482: bool is_dir=(finfo.st_mode&S_IFDIR) != 0;
483: return is_dir==need_dir;
484: }
485: return false;
486: }
487:
488: static bool entry_ifdir(const String& file_spec, bool need_dir) {
489: return entry_ifdir(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir);
490: }
491:
492: // throws nothing! [this is required in file_move & file_delete]
493: static void rmdir(const String& file_spec, size_t pos_after) {
494: char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC);
495: size_t length=strlen(dir_spec);
496: while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){
497: #ifdef _MSC_VER
498: if(!entry_ifdir(dir_spec, true))
499: break;
500: DWORD attrs=GetFileAttributesW(PA_UTF16_ENC(dir_spec));
501: if(
502: (attrs==INVALID_FILE_ATTRIBUTES)
503: || !(attrs & FILE_ATTRIBUTE_DIRECTORY)
504: || (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
505: )
506: break;
507: #endif
508: if(pa_rmdir(dir_spec))
509: break;
510: };
511: }
512:
513: bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) {
1.3 moko 514: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 515: if(pa_unlink(fname)!=0) {
516: if(fail_on_problem)
517: throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0,
518: &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
519: else
520: return false;
521: }
522:
523: if(!keep_empty_dirs)
1.3 moko 524: rmdir(file_spec, 1);
1.1 moko 525:
526: return true;
527: }
528:
529: void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) {
1.3 moko 530: const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC);
531: const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 532:
1.3 moko 533: create_dir_for_file(new_spec);
1.1 moko 534:
535: if(pa_rename(old_spec_cstr, new_spec_cstr)!=0)
536: throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0,
537: &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr);
538:
539: if(!keep_empty_dirs)
1.3 moko 540: rmdir(old_spec, 1);
1.1 moko 541: }
542:
543:
544: bool file_exist(const String& file_spec) {
1.3 moko 545: return entry_ifdir(file_spec, false);
1.1 moko 546: }
547:
548: bool dir_exists(const String& file_spec) {
1.3 moko 549: return entry_ifdir(file_spec, true);
1.1 moko 550: }
551:
552: const String* file_exist(const String& path, const String& name) {
553: String& result=*new String(path);
1.6 moko 554: if(path.last_char() != '/')
1.1 moko 555: result << "/";
556: result << name;
557: return file_exist(result)?&result:0;
558: }
559:
560: bool file_executable(const String& file_spec) {
561: return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0;
562: }
563:
564: bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) {
1.3 moko 565: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 566: struct stat finfo;
567: if(pa_stat(fname, &finfo)!=0) {
568: if(fail_on_read_problem)
569: throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname);
570: else
571: return false;
572: }
573: rsize=finfo.st_size;
574: ratime=(time_t)finfo.st_atime;
575: rmtime=(time_t)finfo.st_mtime;
576: rctime=(time_t)finfo.st_ctime;
577: return true;
578: }
579:
580: size_t check_file_size(uint64_t size, const String* file_spec){
581: if(size > (uint64_t)pa_file_size_limit)
1.2 moko 582: throw Exception(PARSER_RUNTIME, file_spec, "content size of %s bytes exceeds the limit (%s bytes)", pa_uitoa(size), pa_uitoa(pa_file_size_limit));
1.1 moko 583: return (size_t)size;
584: }
585:
1.3 moko 586: /**
1.1 moko 587: String related functions
588: */
589:
590: const char *pa_filename(const char *path){
591: const char *slash=strrpbrk(path, "/\\");
592: return slash ? slash+1 : path;
593: }
E-mail: