Annotation of parser3/src/main/pa_file.C, revision 1.3
1.1 moko 1: /** @file
2: Parser: file and file system related functions.
3:
4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
6: */
7:
8: #include "pa_file.h"
9: #include "pa_dir.h"
10: #include "pa_charsets.h"
11: #include "pa_http.h"
12: #include "pa_convert_utf.h"
13:
14: #ifdef _MSC_VER
15: #include <windows.h>
16: #include <direct.h>
17: #endif
18:
1.3 ! moko 19: volatile const char * IDENT_PA_FILE_C="$Id: pa_file.C,v 1.2 2026/09/27 17:37:51 moko Exp $" IDENT_PA_FILE_H;
1.1 moko 20:
21: // some maybe-undefined constants
22:
23: #ifndef _O_TEXT
24: # define _O_TEXT 0
25: #endif
26: #ifndef _O_BINARY
27: # define _O_BINARY 0
28: #endif
29:
30: #ifdef HAVE_FTRUNCATE
31: # define PA_O_TRUNC 0
32: #else
33: # ifdef _O_TRUNC
34: # define PA_O_TRUNC _O_TRUNC
35: # else
36: # error you must have either ftruncate function or _O_TRUNC bit declared
37: # endif
38: #endif
39:
40: #ifdef WIN32
41:
42: bool is_os_absolute_path(const String& path) {
43: return is_os_absolute_path(path.cstr());
44: }
45:
46: bool is_os_absolute_path(const char* path) {
47: return path[0] && path[0]!=':' && path[1]==':' // DRIVE: (including drive-relative paths, for compatibility)
48: || path[0]=='\\' && path[1]=='\\' // UNC
49: || path[0]=='/' && path[1]=='/';
50: }
51: #endif
52:
53: // externs
54:
55: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset);
56:
57: // functions
58:
59: #ifdef _MSC_VER
60:
61: #define PA_UTF16_ENC(value) (const wchar_t *)pa_utf16_encode(value, pa_thread_request().charsets.source())
62:
63: int pa_stat(const char *pathname, struct stat *buffer){
64: return _wstat64(PA_UTF16_ENC(pathname), buffer);
65: }
66:
67: int pa_open(const char *pathname, int flags, int mode){
68: return _wopen(PA_UTF16_ENC(pathname), flags, mode);
69: }
70:
71: FILE *pa_fopen(const char *pathname, const char *mode){
72: return _wfopen(PA_UTF16_ENC(pathname), PA_UTF16_ENC(mode));
73: }
74:
75: int pa_mkdir(const char *pathname, int){
76: return _wmkdir(PA_UTF16_ENC(pathname));
77: }
78:
79: int pa_rmdir(const char *pathname){
80: return _wrmdir(PA_UTF16_ENC(pathname));
81: }
82:
83: int pa_rename(const char *oldpath, const char *newpath){
84: return _wrename(PA_UTF16_ENC(oldpath), PA_UTF16_ENC(newpath));
85: }
86:
87: int pa_unlink(const char *pathname){
88: return _wunlink(PA_UTF16_ENC(pathname));
89: }
90:
91: #else
92:
93: #define pa_mkdir mkdir
94: #define pa_rmdir rmdir
95: #define pa_rename rename
96: #define pa_unlink unlink
97:
98: #endif
99:
100: /// these options were handled but not checked elsewhere, now check them
101: int pa_get_valid_file_options_count(HashStringValue& options) {
102: int result=0;
103: if(options.get(PA_SQL_LIMIT_NAME))
104: result++;
105: if(options.get(PA_SQL_OFFSET_NAME))
106: result++;
107: if(options.get(PA_COLUMN_SEPARATOR_NAME))
108: result++;
109: if(options.get(PA_COLUMN_ENCLOSER_NAME))
110: result++;
111: if(options.get(PA_CHARSET_NAME))
112: result++;
113: return result;
114: }
115:
116: #ifndef DOXYGEN
117: struct File_read_action_info {
118: char **data; size_t *data_size;
119: char* buf; uint64_t offset; size_t limit;
1.3 ! moko 120: };
1.1 moko 121: #endif
122:
123: static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) {
1.3 ! moko 124: File_read_action_info& info = *static_cast<File_read_action_info *>(context);
1.1 moko 125: size_t to_read_size = check_file_size(info.limit && info.limit < (size_t)finfo.st_size ? info.limit : (size_t)finfo.st_size, &file_spec);
126: if(to_read_size) {
127: if(info.offset)
128: pa_lseek(f, info.offset, SEEK_SET); // seek never fails as POSIX allows the file offset to be set beyond the EOF
129: *info.data = info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1);
130: ssize_t result = read(f, *info.data, to_read_size);
131: if(result<0)
132: throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno);
133: *info.data_size = result;
134: } else { // empty file
135: // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files
136: *info.data = (char *)pa_malloc_atomic(1);
137: *(char*)(*info.data) = 0;
138: *info.data_size = 0;
139: return;
140: }
141: }
142:
143: File_read_result file_read_binary(const String& file_spec, bool fail_on_read_problem, char* buf, uint64_t offset, size_t limit) {
144: File_read_result result = {false, 0, 0, 0};
145: File_read_action_info info = {&result.str, &result.length, buf, offset, limit};
146:
147: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, 0, fail_on_read_problem);
148: return result;
149: }
150:
1.3 ! moko 151: File_read_result file_read(Request_charsets& charsets, const String& file_spec,
1.1 moko 152: bool as_text, HashStringValue *options,
153: bool fail_on_read_problem,
154: size_t offset = 0, size_t limit = 0, bool transcode_text_result = true) {
155: File_read_result result = {false, 0, 0, 0};
156: if(options){
157: int valid_options = pa_get_valid_file_options_count(*options);
158: if(valid_options != options->count())
159: throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
160: }
161:
1.3 ! moko 162: File_read_action_info info = {&result.str, &result.length, 0, offset, limit};
1.1 moko 163:
1.3 ! moko 164: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem);
1.1 moko 165:
166: if(as_text){
167: if(result.success){
168: Charset* asked_charset = 0;
169: if(options)
170: if(Value* vcharset_name = options->get(PA_CHARSET_NAME))
171: asked_charset = &pa_charsets.get(vcharset_name->as_string());
172:
173: asked_charset = pa_charsets.checkBOM(result.str, result.length, asked_charset);
174:
175: if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
176: String::C body = String::C(result.str, result.length);
177: body=Charset::transcode(body, *asked_charset, charsets.source());
178:
179: result.str = const_cast<char*>(body.str); // hacking a little
180: result.length = body.length;
181: }
182: }
183: if(result.length)
184: fix_line_breaks(result.str, result.length);
185: }
186:
187: return result;
188: }
189:
1.3 ! moko 190: File_read_result file_load(Request& r, const String& file_spec,
1.1 moko 191: bool as_text, HashStringValue *options,
192: bool fail_on_read_problem,
193: bool transcode_text_result) {
194:
195: size_t offset = 0;
196: size_t limit = 0;
197:
198: if(options){
199: if(Value *voffset = (Value *)options->get(sql_offset_name))
200: offset = r.process(*voffset).as_int();
201: if(Value *vlimit = (Value *)options->get(sql_limit_name))
202: limit = r.process(*vlimit).as_int();
203: // no check on options count here
204: }
205:
206: if(file_spec.starts_with("http://")) {
207: if(offset || limit)
208: throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load");
209:
210: // fail on read problem
211: File_read_http_result http = pa_internal_file_read_http(r, file_spec, as_text, options, transcode_text_result);
212:
213: File_read_result result = {true, http.str, http.length, http.headers};
214: return result;
215: } else
216: return file_read(r.charsets, file_spec, as_text, options, fail_on_read_problem, offset, limit, transcode_text_result);
217: }
218:
219: char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem) {
220: File_read_result file = file_read(charsets, file_spec, true, 0, fail_on_read_problem);
221: return file.success ? file.str : 0;
222: }
223:
224: char* file_load_text(Request& r, const String& file_spec, bool fail_on_read_problem, HashStringValue* options, bool transcode_result) {
225: File_read_result file = file_load(r, file_spec, true, options, fail_on_read_problem, transcode_result);
226: return file.success ? file.str : 0;
227: }
228:
229: #ifdef PA_SAFE_MODE
230: void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) {
231: if(finfo.st_uid/*foreign?*/!=geteuid()
232: && finfo.st_gid/*foreign?*/!=getegid())
233: throw Exception(PARSER_RUNTIME,
234: &file_spec,
235: "parser is in safe mode: reading files of foreign group and user disabled "
236: "[recompile parser with --disable-safe-mode configure option], "
237: "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)",
238: fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid()
239: );
240: }
241: #else
242: void check_safe_mode(struct stat, const String&, const char*) {
243: }
244: #endif
245:
246:
247: bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context,
248: bool as_text, bool fail_on_read_problem) {
249:
250: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
251: int f;
252:
253: // first open, next stat:
254: // directory update of NTFS hard links performed on open.
1.3 ! moko 255: // ex:
1.1 moko 256: // a.html:^test[] and b.html hardlink to a.html
257: // user inserts ! before ^test in a.html
1.3 ! moko 258: // directory entry of b.html in NTFS not updated at once,
1.1 moko 259: // they delay update till open, so we would receive "!^test[" string
260: // if would do stat, next open.
261: // later: it seems, even this does not help sometimes
262: if((f=pa_open(fname, O_RDONLY | (as_text ? _O_TEXT : _O_BINARY) ))>=0) {
263: try {
264: int pa_errno=pa_lock_shared_blocking(f);
265: if(pa_errno!=0)
266: throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
267:
268: struct stat finfo;
269: if(pa_fstat(f, &finfo)!=0)
270: throw Exception("file.missing", // hardly possible: we just opened it OK
271: &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
272:
273: check_safe_mode(finfo, file_spec, fname);
274:
1.3 ! moko 275: action(finfo, f, file_spec, context);
1.1 moko 276: } catch(...) {
1.3 ! moko 277: pa_unlock(f);close(f);
1.1 moko 278: if(fail_on_read_problem)
279: rethrow;
280: return false;
1.3 ! moko 281: }
1.1 moko 282:
1.3 ! moko 283: pa_unlock(f);close(f);
1.1 moko 284: return true;
285: } else {
286: if(fail_on_read_problem)
287: throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0,
288: &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
289: return false;
290: }
291: }
292:
293: void create_dir_for_file(const String& file_spec) {
294: const char *str=file_spec.taint_cstr(String::L_FILE_SPEC);
295: if(str[0]){
296: const char *pos=str+1;
297: while((pos=strpbrk(pos, "/\\")) && pos[1]) { // to avoid trailing /, see #1166
298: pa_mkdir(pa_strdup(str,pos-str), 0775);
299: pos++;
300: }
301: }
302: }
303:
304: bool file_write_action_under_lock(const String& file_spec, const char* action_name, File_write_action action, void *context,
305: bool as_text, bool do_append, bool do_block, bool fail_on_lock_problem) {
306:
307: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
308: int f;
309: if(access(fname, W_OK)!=0) // no
310: create_dir_for_file(file_spec);
311:
312: if((f=pa_open(fname, O_CREAT | O_RDWR | (as_text ? _O_TEXT : _O_BINARY) | (do_append ? O_APPEND : PA_O_TRUNC), 0664))>=0) {
313: int pa_errno=do_block ? pa_lock_exclusive_blocking(f) : pa_lock_exclusive_nonblocking(f);
314: if(pa_errno!=0) {
315: Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
316: close(f);
317: if(fail_on_lock_problem)
318: throw e;
319: return false;
320: }
321:
322: try {
323: #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE))
324: struct stat finfo;
325: if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111)
326: fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any
327: #endif
328: action(f, context);
329: } catch(...) {
330: #ifdef HAVE_FTRUNCATE
331: if(!do_append)
332: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // one cannot use O_TRUNC, read lower
333: #endif
334: pa_unlock(f);close(f);
335: rethrow;
336: }
337:
338: #ifdef HAVE_FTRUNCATE
339: if(!do_append)
340: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // O_TRUNC truncates even exclusevely write-locked file [thanks to Igor Milyakov <virtan@rotabanner.com> for discovering]
341: #endif
342: pa_unlock(f);close(f);
343: return true;
344: } else
345: throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
346: // here should be nothing, see rethrow above
347: }
348:
349: #ifndef DOXYGEN
350: struct File_write_action_info {
351: const char* str;
352: size_t length;
353: };
354: #endif
355:
356: // a single write() must stay below INT_MAX as windows _write() reports written bytes as int
357: #define FILE_WRITE_CHUNK_SIZE (256*0x100000)
358:
359: static const char* bytes_left(size_t rest) {
360: return rest ? pa_strcat(", ", pa_uitoa(rest), " bytes left") : "";
361: }
362:
363: static void file_write_action(int f, void *context) {
364: File_write_action_info& info=*static_cast<File_write_action_info *>(context);
365:
366: const char* data=info.str;
367: size_t left=info.length;
368:
369: while(left) {
370: size_t portion=min(left, (size_t)FILE_WRITE_CHUNK_SIZE);
371: ssize_t written=write(f, data, portion);
372: if(written<0)
373: throw Exception("file.write", 0, "error writing %s bytes: %s (%d)%s", pa_uitoa(portion), strerror(errno), errno, bytes_left(left-portion));
374: if(written==0)
375: throw Exception("file.write", 0, "error writing %s bytes: zero bytes written%s", pa_uitoa(portion), bytes_left(left-portion));
376: data+=written;
377: left-=written;
378: }
379: }
380:
381: void file_write(
382: Request_charsets& charsets,
383: const String& file_spec,
384: const char* data,
1.3 ! moko 385: size_t size,
! 386: bool as_text,
1.1 moko 387: bool do_append,
388: Charset* asked_charset) {
389:
390: if(as_text && asked_charset){
391: String::C body=String::C(data, size);
392: body=Charset::transcode(body, charsets.source(), *asked_charset);
393: data=body.str;
394: size=body.length;
395: };
396:
397: File_write_action_info info={data, size};
398:
399: file_write_action_under_lock(
400: file_spec,
401: "write",
402: file_write_action,
403: &info,
404: as_text,
405: do_append);
406: }
407:
408: static size_t get_dir(char* fname, size_t helper_length){
409: bool dir=false;
410: size_t pos=0;
411: for(pos=helper_length; pos; pos--){
412: char c=fname[pos-1];
413: if(c=='/' || c=='\\'){
414: fname[pos-1]=0;
415: dir=true;
416: } else if(dir) break;
417: }
418: return pos;
419: }
420:
421: bool entry_exists(const char* fname, struct stat *afinfo) {
422: struct stat lfinfo;
423: bool result=pa_stat(fname, &lfinfo)==0;
424: if(afinfo)
425: *afinfo=lfinfo;
426: return result;
427: }
428:
429: bool entry_exists(const String& file_spec) {
1.3 ! moko 430: return entry_exists(file_spec.taint_cstr(String::L_FILE_SPEC), 0);
1.1 moko 431: }
432:
433: static bool entry_ifdir(char *fname, bool need_dir) {
434: if(need_dir){
435: size_t size=strlen(fname);
436: while(size) {
437: char c=fname[size-1];
438: if(c=='/' || c=='\\')
439: fname[--size]=0;
440: else
441: break;
442: }
443: }
444:
445: struct stat finfo;
446: if(entry_exists(fname, &finfo)) {
447: bool is_dir=(finfo.st_mode&S_IFDIR) != 0;
448: return is_dir==need_dir;
449: }
450: return false;
451: }
452:
453: static bool entry_ifdir(const String& file_spec, bool need_dir) {
454: return entry_ifdir(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir);
455: }
456:
457: // throws nothing! [this is required in file_move & file_delete]
458: static void rmdir(const String& file_spec, size_t pos_after) {
459: char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC);
460: size_t length=strlen(dir_spec);
461: while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){
462: #ifdef _MSC_VER
463: if(!entry_ifdir(dir_spec, true))
464: break;
465: DWORD attrs=GetFileAttributesW(PA_UTF16_ENC(dir_spec));
466: if(
467: (attrs==INVALID_FILE_ATTRIBUTES)
468: || !(attrs & FILE_ATTRIBUTE_DIRECTORY)
469: || (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
470: )
471: break;
472: #endif
473: if(pa_rmdir(dir_spec))
474: break;
475: };
476: }
477:
478: bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) {
1.3 ! moko 479: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 480: if(pa_unlink(fname)!=0) {
481: if(fail_on_problem)
482: throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0,
483: &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
484: else
485: return false;
486: }
487:
488: if(!keep_empty_dirs)
1.3 ! moko 489: rmdir(file_spec, 1);
1.1 moko 490:
491: return true;
492: }
493:
494: void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) {
1.3 ! moko 495: const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC);
! 496: const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 497:
1.3 ! moko 498: create_dir_for_file(new_spec);
1.1 moko 499:
500: if(pa_rename(old_spec_cstr, new_spec_cstr)!=0)
501: throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0,
502: &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr);
503:
504: if(!keep_empty_dirs)
1.3 ! moko 505: rmdir(old_spec, 1);
1.1 moko 506: }
507:
508:
509: bool file_exist(const String& file_spec) {
1.3 ! moko 510: return entry_ifdir(file_spec, false);
1.1 moko 511: }
512:
513: bool dir_exists(const String& file_spec) {
1.3 ! moko 514: return entry_ifdir(file_spec, true);
1.1 moko 515: }
516:
517: const String* file_exist(const String& path, const String& name) {
518: String& result=*new String(path);
519: if(path.last_char() != '/' && path.last_char() != '\\')
520: result << "/";
521: result << name;
522: return file_exist(result)?&result:0;
523: }
524:
525: bool file_executable(const String& file_spec) {
526: return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0;
527: }
528:
529: bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) {
1.3 ! moko 530: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 531: struct stat finfo;
532: if(pa_stat(fname, &finfo)!=0) {
533: if(fail_on_read_problem)
534: throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname);
535: else
536: return false;
537: }
538: rsize=finfo.st_size;
539: ratime=(time_t)finfo.st_atime;
540: rmtime=(time_t)finfo.st_mtime;
541: rctime=(time_t)finfo.st_ctime;
542: return true;
543: }
544:
545: size_t check_file_size(uint64_t size, const String* file_spec){
546: if(size > (uint64_t)pa_file_size_limit)
1.2 moko 547: throw Exception(PARSER_RUNTIME, file_spec, "content size of %s bytes exceeds the limit (%s bytes)", pa_uitoa(size), pa_uitoa(pa_file_size_limit));
1.1 moko 548: return (size_t)size;
549: }
550:
1.3 ! moko 551: /**
1.1 moko 552: String related functions
553: */
554:
555: const char *pa_filename(const char *path){
556: const char *slash=strrpbrk(path, "/\\");
557: return slash ? slash+1 : path;
558: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_file.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main