Annotation of parser3/src/main/pa_file.C, revision 1.6
1.1 moko 1: /** @file
2: Parser: file and file system related functions.
3:
4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
6: */
7:
8: #include "pa_file.h"
9: #include "pa_dir.h"
10: #include "pa_charsets.h"
11: #include "pa_http.h"
12: #include "pa_convert_utf.h"
13:
14: #ifdef _MSC_VER
15: #include <windows.h>
16: #include <direct.h>
17: #endif
18:
1.6 ! moko 19: volatile const char * IDENT_PA_FILE_C="$Id: pa_file.C,v 1.5 2026/09/29 02:56:52 moko Exp $" IDENT_PA_FILE_H;
1.1 moko 20:
21: // some maybe-undefined constants
22:
23: #ifndef _O_TEXT
24: # define _O_TEXT 0
25: #endif
26: #ifndef _O_BINARY
27: # define _O_BINARY 0
28: #endif
29:
30: #ifdef HAVE_FTRUNCATE
31: # define PA_O_TRUNC 0
32: #else
33: # ifdef _O_TRUNC
34: # define PA_O_TRUNC _O_TRUNC
35: # else
36: # error you must have either ftruncate function or _O_TRUNC bit declared
37: # endif
38: #endif
39:
40: #ifdef WIN32
41:
42: bool is_os_absolute_path(const String& path) {
43: return is_os_absolute_path(path.cstr());
44: }
45:
46: bool is_os_absolute_path(const char* path) {
47: return path[0] && path[0]!=':' && path[1]==':' // DRIVE: (including drive-relative paths, for compatibility)
1.4 moko 48: || path[0]=='\\' && path[1]=='\\'; // UNC, only '\\' is supported as '//' is a document root relative path
1.1 moko 49: }
50: #endif
51:
1.5 moko 52: uint path_scheme(const String& path) {
53: if(path.starts_with("file://"))
54: return PA_SCHEME_FILE;
55: if(path.starts_with("http://"))
56: return PA_SCHEME_HTTP;
57: if(path.starts_with("parser://"))
58: return PA_SCHEME_PARSER;
59: return PA_SCHEME_NONE;
60: }
61:
62: void check_path_scheme(uint scheme, const String& path, uint allowed) {
63: if(scheme!=PA_SCHEME_FILE && !(allowed&scheme))
64: throw Exception(PARSER_RUNTIME, &path, "unsupported path scheme");
65: }
66:
67: const String* file_uri_to_path(const String& uri) {
68: const String* rest=&uri.mid(7 /* "file://" */, uri.length());
69: if(rest->starts_with("localhost/"))
70: rest=&rest->mid(9 /* "localhost", keeping the slash */, rest->length());
71: if(rest->first_char()=='/') {
72: // file:///C:/path and file://localhost/C:/path: the slash before a drive name is not a part of the path
73: const String& disk=rest->mid(1, rest->length());
74: return is_os_absolute_path(disk) ? &disk : rest;
75: }
76: if(is_os_absolute_path(*rest))
77: return rest; // file://C:/path, not a standard form but in use
78: return 0;
79: }
80:
1.1 moko 81: // externs
82:
83: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset);
84:
85: // functions
86:
87: #ifdef _MSC_VER
88:
89: #define PA_UTF16_ENC(value) (const wchar_t *)pa_utf16_encode(value, pa_thread_request().charsets.source())
90:
91: int pa_stat(const char *pathname, struct stat *buffer){
92: return _wstat64(PA_UTF16_ENC(pathname), buffer);
93: }
94:
95: int pa_open(const char *pathname, int flags, int mode){
96: return _wopen(PA_UTF16_ENC(pathname), flags, mode);
97: }
98:
99: FILE *pa_fopen(const char *pathname, const char *mode){
100: return _wfopen(PA_UTF16_ENC(pathname), PA_UTF16_ENC(mode));
101: }
102:
103: int pa_mkdir(const char *pathname, int){
104: return _wmkdir(PA_UTF16_ENC(pathname));
105: }
106:
107: int pa_rmdir(const char *pathname){
108: return _wrmdir(PA_UTF16_ENC(pathname));
109: }
110:
111: int pa_rename(const char *oldpath, const char *newpath){
112: return _wrename(PA_UTF16_ENC(oldpath), PA_UTF16_ENC(newpath));
113: }
114:
115: int pa_unlink(const char *pathname){
116: return _wunlink(PA_UTF16_ENC(pathname));
117: }
118:
119: #else
120:
121: #define pa_mkdir mkdir
122: #define pa_rmdir rmdir
123: #define pa_rename rename
124: #define pa_unlink unlink
125:
126: #endif
127:
128: /// these options were handled but not checked elsewhere, now check them
129: int pa_get_valid_file_options_count(HashStringValue& options) {
130: int result=0;
131: if(options.get(PA_SQL_LIMIT_NAME))
132: result++;
133: if(options.get(PA_SQL_OFFSET_NAME))
134: result++;
135: if(options.get(PA_COLUMN_SEPARATOR_NAME))
136: result++;
137: if(options.get(PA_COLUMN_ENCLOSER_NAME))
138: result++;
139: if(options.get(PA_CHARSET_NAME))
140: result++;
141: return result;
142: }
143:
144: #ifndef DOXYGEN
145: struct File_read_action_info {
146: char **data; size_t *data_size;
147: char* buf; uint64_t offset; size_t limit;
1.3 moko 148: };
1.1 moko 149: #endif
150:
151: static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) {
1.3 moko 152: File_read_action_info& info = *static_cast<File_read_action_info *>(context);
1.1 moko 153: size_t to_read_size = check_file_size(info.limit && info.limit < (size_t)finfo.st_size ? info.limit : (size_t)finfo.st_size, &file_spec);
154: if(to_read_size) {
155: if(info.offset)
156: pa_lseek(f, info.offset, SEEK_SET); // seek never fails as POSIX allows the file offset to be set beyond the EOF
157: *info.data = info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1);
158: ssize_t result = read(f, *info.data, to_read_size);
159: if(result<0)
160: throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno);
161: *info.data_size = result;
162: } else { // empty file
163: // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files
164: *info.data = (char *)pa_malloc_atomic(1);
165: *(char*)(*info.data) = 0;
166: *info.data_size = 0;
167: return;
168: }
169: }
170:
171: File_read_result file_read_binary(const String& file_spec, bool fail_on_read_problem, char* buf, uint64_t offset, size_t limit) {
172: File_read_result result = {false, 0, 0, 0};
173: File_read_action_info info = {&result.str, &result.length, buf, offset, limit};
174:
175: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, 0, fail_on_read_problem);
176: return result;
177: }
178:
1.3 moko 179: File_read_result file_read(Request_charsets& charsets, const String& file_spec,
1.1 moko 180: bool as_text, HashStringValue *options,
181: bool fail_on_read_problem,
182: size_t offset = 0, size_t limit = 0, bool transcode_text_result = true) {
183: File_read_result result = {false, 0, 0, 0};
184: if(options){
185: int valid_options = pa_get_valid_file_options_count(*options);
186: if(valid_options != options->count())
187: throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
188: }
189:
1.3 moko 190: File_read_action_info info = {&result.str, &result.length, 0, offset, limit};
1.1 moko 191:
1.3 moko 192: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem);
1.1 moko 193:
194: if(as_text){
195: if(result.success){
196: Charset* asked_charset = 0;
197: if(options)
198: if(Value* vcharset_name = options->get(PA_CHARSET_NAME))
199: asked_charset = &pa_charsets.get(vcharset_name->as_string());
200:
201: asked_charset = pa_charsets.checkBOM(result.str, result.length, asked_charset);
202:
203: if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
204: String::C body = String::C(result.str, result.length);
205: body=Charset::transcode(body, *asked_charset, charsets.source());
206:
207: result.str = const_cast<char*>(body.str); // hacking a little
208: result.length = body.length;
209: }
210: }
211: if(result.length)
212: fix_line_breaks(result.str, result.length);
213: }
214:
215: return result;
216: }
217:
1.3 moko 218: File_read_result file_load(Request& r, const String& file_spec,
1.1 moko 219: bool as_text, HashStringValue *options,
220: bool fail_on_read_problem,
221: bool transcode_text_result) {
222:
223: size_t offset = 0;
224: size_t limit = 0;
225:
226: if(options){
227: if(Value *voffset = (Value *)options->get(sql_offset_name))
228: offset = r.process(*voffset).as_int();
229: if(Value *vlimit = (Value *)options->get(sql_limit_name))
230: limit = r.process(*vlimit).as_int();
231: // no check on options count here
232: }
233:
234: if(file_spec.starts_with("http://")) {
235: if(offset || limit)
236: throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load");
237:
238: // fail on read problem
239: File_read_http_result http = pa_internal_file_read_http(r, file_spec, as_text, options, transcode_text_result);
240:
241: File_read_result result = {true, http.str, http.length, http.headers};
242: return result;
243: } else
244: return file_read(r.charsets, file_spec, as_text, options, fail_on_read_problem, offset, limit, transcode_text_result);
245: }
246:
247: char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem) {
248: File_read_result file = file_read(charsets, file_spec, true, 0, fail_on_read_problem);
249: return file.success ? file.str : 0;
250: }
251:
252: char* file_load_text(Request& r, const String& file_spec, bool fail_on_read_problem, HashStringValue* options, bool transcode_result) {
253: File_read_result file = file_load(r, file_spec, true, options, fail_on_read_problem, transcode_result);
254: return file.success ? file.str : 0;
255: }
256:
257: #ifdef PA_SAFE_MODE
258: void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) {
259: if(finfo.st_uid/*foreign?*/!=geteuid()
260: && finfo.st_gid/*foreign?*/!=getegid())
261: throw Exception(PARSER_RUNTIME,
262: &file_spec,
263: "parser is in safe mode: reading files of foreign group and user disabled "
264: "[recompile parser with --disable-safe-mode configure option], "
265: "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)",
266: fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid()
267: );
268: }
269: #else
270: void check_safe_mode(struct stat, const String&, const char*) {
271: }
272: #endif
273:
274:
275: bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context,
276: bool as_text, bool fail_on_read_problem) {
277:
278: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
279: int f;
280:
281: // first open, next stat:
282: // directory update of NTFS hard links performed on open.
1.3 moko 283: // ex:
1.1 moko 284: // a.html:^test[] and b.html hardlink to a.html
285: // user inserts ! before ^test in a.html
1.3 moko 286: // directory entry of b.html in NTFS not updated at once,
1.1 moko 287: // they delay update till open, so we would receive "!^test[" string
288: // if would do stat, next open.
289: // later: it seems, even this does not help sometimes
290: if((f=pa_open(fname, O_RDONLY | (as_text ? _O_TEXT : _O_BINARY) ))>=0) {
291: try {
292: int pa_errno=pa_lock_shared_blocking(f);
293: if(pa_errno!=0)
294: throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
295:
296: struct stat finfo;
297: if(pa_fstat(f, &finfo)!=0)
298: throw Exception("file.missing", // hardly possible: we just opened it OK
299: &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
300:
301: check_safe_mode(finfo, file_spec, fname);
302:
1.3 moko 303: action(finfo, f, file_spec, context);
1.1 moko 304: } catch(...) {
1.3 moko 305: pa_unlock(f);close(f);
1.1 moko 306: if(fail_on_read_problem)
307: rethrow;
308: return false;
1.3 moko 309: }
1.1 moko 310:
1.3 moko 311: pa_unlock(f);close(f);
1.1 moko 312: return true;
313: } else {
314: if(fail_on_read_problem)
315: throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0,
316: &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
317: return false;
318: }
319: }
320:
321: void create_dir_for_file(const String& file_spec) {
322: const char *str=file_spec.taint_cstr(String::L_FILE_SPEC);
323: if(str[0]){
324: const char *pos=str+1;
1.6 ! moko 325: while((pos=strchr(pos, '/')) && pos[1]) { // to avoid trailing /, see #1166
1.1 moko 326: pa_mkdir(pa_strdup(str,pos-str), 0775);
327: pos++;
328: }
329: }
330: }
331:
332: bool file_write_action_under_lock(const String& file_spec, const char* action_name, File_write_action action, void *context,
333: bool as_text, bool do_append, bool do_block, bool fail_on_lock_problem) {
334:
335: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
336: int f;
337: if(access(fname, W_OK)!=0) // no
338: create_dir_for_file(file_spec);
339:
340: if((f=pa_open(fname, O_CREAT | O_RDWR | (as_text ? _O_TEXT : _O_BINARY) | (do_append ? O_APPEND : PA_O_TRUNC), 0664))>=0) {
341: int pa_errno=do_block ? pa_lock_exclusive_blocking(f) : pa_lock_exclusive_nonblocking(f);
342: if(pa_errno!=0) {
343: Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
344: close(f);
345: if(fail_on_lock_problem)
346: throw e;
347: return false;
348: }
349:
350: try {
351: #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE))
352: struct stat finfo;
353: if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111)
354: fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any
355: #endif
356: action(f, context);
357: } catch(...) {
358: #ifdef HAVE_FTRUNCATE
359: if(!do_append)
360: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // one cannot use O_TRUNC, read lower
361: #endif
362: pa_unlock(f);close(f);
363: rethrow;
364: }
365:
366: #ifdef HAVE_FTRUNCATE
367: if(!do_append)
368: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // O_TRUNC truncates even exclusevely write-locked file [thanks to Igor Milyakov <virtan@rotabanner.com> for discovering]
369: #endif
370: pa_unlock(f);close(f);
371: return true;
372: } else
373: throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
374: // here should be nothing, see rethrow above
375: }
376:
377: #ifndef DOXYGEN
378: struct File_write_action_info {
379: const char* str;
380: size_t length;
381: };
382: #endif
383:
384: // a single write() must stay below INT_MAX as windows _write() reports written bytes as int
385: #define FILE_WRITE_CHUNK_SIZE (256*0x100000)
386:
387: static const char* bytes_left(size_t rest) {
388: return rest ? pa_strcat(", ", pa_uitoa(rest), " bytes left") : "";
389: }
390:
391: static void file_write_action(int f, void *context) {
392: File_write_action_info& info=*static_cast<File_write_action_info *>(context);
393:
394: const char* data=info.str;
395: size_t left=info.length;
396:
397: while(left) {
398: size_t portion=min(left, (size_t)FILE_WRITE_CHUNK_SIZE);
399: ssize_t written=write(f, data, portion);
400: if(written<0)
401: throw Exception("file.write", 0, "error writing %s bytes: %s (%d)%s", pa_uitoa(portion), strerror(errno), errno, bytes_left(left-portion));
402: if(written==0)
403: throw Exception("file.write", 0, "error writing %s bytes: zero bytes written%s", pa_uitoa(portion), bytes_left(left-portion));
404: data+=written;
405: left-=written;
406: }
407: }
408:
409: void file_write(
410: Request_charsets& charsets,
411: const String& file_spec,
412: const char* data,
1.3 moko 413: size_t size,
414: bool as_text,
1.1 moko 415: bool do_append,
416: Charset* asked_charset) {
417:
418: if(as_text && asked_charset){
419: String::C body=String::C(data, size);
420: body=Charset::transcode(body, charsets.source(), *asked_charset);
421: data=body.str;
422: size=body.length;
423: };
424:
425: File_write_action_info info={data, size};
426:
427: file_write_action_under_lock(
428: file_spec,
429: "write",
430: file_write_action,
431: &info,
432: as_text,
433: do_append);
434: }
435:
436: static size_t get_dir(char* fname, size_t helper_length){
437: bool dir=false;
438: size_t pos=0;
439: for(pos=helper_length; pos; pos--){
440: char c=fname[pos-1];
1.6 ! moko 441: if(c=='/'){
1.1 moko 442: fname[pos-1]=0;
443: dir=true;
444: } else if(dir) break;
445: }
446: return pos;
447: }
448:
449: bool entry_exists(const char* fname, struct stat *afinfo) {
450: struct stat lfinfo;
451: bool result=pa_stat(fname, &lfinfo)==0;
452: if(afinfo)
453: *afinfo=lfinfo;
454: return result;
455: }
456:
457: bool entry_exists(const String& file_spec) {
1.3 moko 458: return entry_exists(file_spec.taint_cstr(String::L_FILE_SPEC), 0);
1.1 moko 459: }
460:
461: static bool entry_ifdir(char *fname, bool need_dir) {
462: if(need_dir){
463: size_t size=strlen(fname);
464: while(size) {
465: char c=fname[size-1];
1.6 ! moko 466: if(c=='/')
1.1 moko 467: fname[--size]=0;
468: else
469: break;
470: }
471: }
472:
473: struct stat finfo;
474: if(entry_exists(fname, &finfo)) {
475: bool is_dir=(finfo.st_mode&S_IFDIR) != 0;
476: return is_dir==need_dir;
477: }
478: return false;
479: }
480:
481: static bool entry_ifdir(const String& file_spec, bool need_dir) {
482: return entry_ifdir(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir);
483: }
484:
485: // throws nothing! [this is required in file_move & file_delete]
486: static void rmdir(const String& file_spec, size_t pos_after) {
487: char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC);
488: size_t length=strlen(dir_spec);
489: while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){
490: #ifdef _MSC_VER
491: if(!entry_ifdir(dir_spec, true))
492: break;
493: DWORD attrs=GetFileAttributesW(PA_UTF16_ENC(dir_spec));
494: if(
495: (attrs==INVALID_FILE_ATTRIBUTES)
496: || !(attrs & FILE_ATTRIBUTE_DIRECTORY)
497: || (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
498: )
499: break;
500: #endif
501: if(pa_rmdir(dir_spec))
502: break;
503: };
504: }
505:
506: bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) {
1.3 moko 507: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 508: if(pa_unlink(fname)!=0) {
509: if(fail_on_problem)
510: throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0,
511: &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
512: else
513: return false;
514: }
515:
516: if(!keep_empty_dirs)
1.3 moko 517: rmdir(file_spec, 1);
1.1 moko 518:
519: return true;
520: }
521:
522: void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) {
1.3 moko 523: const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC);
524: const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 525:
1.3 moko 526: create_dir_for_file(new_spec);
1.1 moko 527:
528: if(pa_rename(old_spec_cstr, new_spec_cstr)!=0)
529: throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0,
530: &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr);
531:
532: if(!keep_empty_dirs)
1.3 moko 533: rmdir(old_spec, 1);
1.1 moko 534: }
535:
536:
537: bool file_exist(const String& file_spec) {
1.3 moko 538: return entry_ifdir(file_spec, false);
1.1 moko 539: }
540:
541: bool dir_exists(const String& file_spec) {
1.3 moko 542: return entry_ifdir(file_spec, true);
1.1 moko 543: }
544:
545: const String* file_exist(const String& path, const String& name) {
546: String& result=*new String(path);
1.6 ! moko 547: if(path.last_char() != '/')
1.1 moko 548: result << "/";
549: result << name;
550: return file_exist(result)?&result:0;
551: }
552:
553: bool file_executable(const String& file_spec) {
554: return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0;
555: }
556:
557: bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) {
1.3 moko 558: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 559: struct stat finfo;
560: if(pa_stat(fname, &finfo)!=0) {
561: if(fail_on_read_problem)
562: throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname);
563: else
564: return false;
565: }
566: rsize=finfo.st_size;
567: ratime=(time_t)finfo.st_atime;
568: rmtime=(time_t)finfo.st_mtime;
569: rctime=(time_t)finfo.st_ctime;
570: return true;
571: }
572:
573: size_t check_file_size(uint64_t size, const String* file_spec){
574: if(size > (uint64_t)pa_file_size_limit)
1.2 moko 575: throw Exception(PARSER_RUNTIME, file_spec, "content size of %s bytes exceeds the limit (%s bytes)", pa_uitoa(size), pa_uitoa(pa_file_size_limit));
1.1 moko 576: return (size_t)size;
577: }
578:
1.3 moko 579: /**
1.1 moko 580: String related functions
581: */
582:
583: const char *pa_filename(const char *path){
584: const char *slash=strrpbrk(path, "/\\");
585: return slash ? slash+1 : path;
586: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_file.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main