Annotation of parser3/src/main/pa_file.C, revision 1.4
1.1 moko 1: /** @file
2: Parser: file and file system related functions.
3:
4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
6: */
7:
8: #include "pa_file.h"
9: #include "pa_dir.h"
10: #include "pa_charsets.h"
11: #include "pa_http.h"
12: #include "pa_convert_utf.h"
13:
14: #ifdef _MSC_VER
15: #include <windows.h>
16: #include <direct.h>
17: #endif
18:
1.4 ! moko 19: volatile const char * IDENT_PA_FILE_C="$Id: pa_file.C,v 1.3 2026/09/28 00:23:20 moko Exp $" IDENT_PA_FILE_H;
1.1 moko 20:
21: // some maybe-undefined constants
22:
23: #ifndef _O_TEXT
24: # define _O_TEXT 0
25: #endif
26: #ifndef _O_BINARY
27: # define _O_BINARY 0
28: #endif
29:
30: #ifdef HAVE_FTRUNCATE
31: # define PA_O_TRUNC 0
32: #else
33: # ifdef _O_TRUNC
34: # define PA_O_TRUNC _O_TRUNC
35: # else
36: # error you must have either ftruncate function or _O_TRUNC bit declared
37: # endif
38: #endif
39:
40: #ifdef WIN32
41:
42: bool is_os_absolute_path(const String& path) {
43: return is_os_absolute_path(path.cstr());
44: }
45:
46: bool is_os_absolute_path(const char* path) {
47: return path[0] && path[0]!=':' && path[1]==':' // DRIVE: (including drive-relative paths, for compatibility)
1.4 ! moko 48: || path[0]=='\\' && path[1]=='\\'; // UNC, only '\\' is supported as '//' is a document root relative path
1.1 moko 49: }
50: #endif
51:
52: // externs
53:
54: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset);
55:
56: // functions
57:
58: #ifdef _MSC_VER
59:
60: #define PA_UTF16_ENC(value) (const wchar_t *)pa_utf16_encode(value, pa_thread_request().charsets.source())
61:
62: int pa_stat(const char *pathname, struct stat *buffer){
63: return _wstat64(PA_UTF16_ENC(pathname), buffer);
64: }
65:
66: int pa_open(const char *pathname, int flags, int mode){
67: return _wopen(PA_UTF16_ENC(pathname), flags, mode);
68: }
69:
70: FILE *pa_fopen(const char *pathname, const char *mode){
71: return _wfopen(PA_UTF16_ENC(pathname), PA_UTF16_ENC(mode));
72: }
73:
74: int pa_mkdir(const char *pathname, int){
75: return _wmkdir(PA_UTF16_ENC(pathname));
76: }
77:
78: int pa_rmdir(const char *pathname){
79: return _wrmdir(PA_UTF16_ENC(pathname));
80: }
81:
82: int pa_rename(const char *oldpath, const char *newpath){
83: return _wrename(PA_UTF16_ENC(oldpath), PA_UTF16_ENC(newpath));
84: }
85:
86: int pa_unlink(const char *pathname){
87: return _wunlink(PA_UTF16_ENC(pathname));
88: }
89:
90: #else
91:
92: #define pa_mkdir mkdir
93: #define pa_rmdir rmdir
94: #define pa_rename rename
95: #define pa_unlink unlink
96:
97: #endif
98:
99: /// these options were handled but not checked elsewhere, now check them
100: int pa_get_valid_file_options_count(HashStringValue& options) {
101: int result=0;
102: if(options.get(PA_SQL_LIMIT_NAME))
103: result++;
104: if(options.get(PA_SQL_OFFSET_NAME))
105: result++;
106: if(options.get(PA_COLUMN_SEPARATOR_NAME))
107: result++;
108: if(options.get(PA_COLUMN_ENCLOSER_NAME))
109: result++;
110: if(options.get(PA_CHARSET_NAME))
111: result++;
112: return result;
113: }
114:
115: #ifndef DOXYGEN
116: struct File_read_action_info {
117: char **data; size_t *data_size;
118: char* buf; uint64_t offset; size_t limit;
1.3 moko 119: };
1.1 moko 120: #endif
121:
122: static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) {
1.3 moko 123: File_read_action_info& info = *static_cast<File_read_action_info *>(context);
1.1 moko 124: size_t to_read_size = check_file_size(info.limit && info.limit < (size_t)finfo.st_size ? info.limit : (size_t)finfo.st_size, &file_spec);
125: if(to_read_size) {
126: if(info.offset)
127: pa_lseek(f, info.offset, SEEK_SET); // seek never fails as POSIX allows the file offset to be set beyond the EOF
128: *info.data = info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1);
129: ssize_t result = read(f, *info.data, to_read_size);
130: if(result<0)
131: throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno);
132: *info.data_size = result;
133: } else { // empty file
134: // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files
135: *info.data = (char *)pa_malloc_atomic(1);
136: *(char*)(*info.data) = 0;
137: *info.data_size = 0;
138: return;
139: }
140: }
141:
142: File_read_result file_read_binary(const String& file_spec, bool fail_on_read_problem, char* buf, uint64_t offset, size_t limit) {
143: File_read_result result = {false, 0, 0, 0};
144: File_read_action_info info = {&result.str, &result.length, buf, offset, limit};
145:
146: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, 0, fail_on_read_problem);
147: return result;
148: }
149:
1.3 moko 150: File_read_result file_read(Request_charsets& charsets, const String& file_spec,
1.1 moko 151: bool as_text, HashStringValue *options,
152: bool fail_on_read_problem,
153: size_t offset = 0, size_t limit = 0, bool transcode_text_result = true) {
154: File_read_result result = {false, 0, 0, 0};
155: if(options){
156: int valid_options = pa_get_valid_file_options_count(*options);
157: if(valid_options != options->count())
158: throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
159: }
160:
1.3 moko 161: File_read_action_info info = {&result.str, &result.length, 0, offset, limit};
1.1 moko 162:
1.3 moko 163: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem);
1.1 moko 164:
165: if(as_text){
166: if(result.success){
167: Charset* asked_charset = 0;
168: if(options)
169: if(Value* vcharset_name = options->get(PA_CHARSET_NAME))
170: asked_charset = &pa_charsets.get(vcharset_name->as_string());
171:
172: asked_charset = pa_charsets.checkBOM(result.str, result.length, asked_charset);
173:
174: if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
175: String::C body = String::C(result.str, result.length);
176: body=Charset::transcode(body, *asked_charset, charsets.source());
177:
178: result.str = const_cast<char*>(body.str); // hacking a little
179: result.length = body.length;
180: }
181: }
182: if(result.length)
183: fix_line_breaks(result.str, result.length);
184: }
185:
186: return result;
187: }
188:
1.3 moko 189: File_read_result file_load(Request& r, const String& file_spec,
1.1 moko 190: bool as_text, HashStringValue *options,
191: bool fail_on_read_problem,
192: bool transcode_text_result) {
193:
194: size_t offset = 0;
195: size_t limit = 0;
196:
197: if(options){
198: if(Value *voffset = (Value *)options->get(sql_offset_name))
199: offset = r.process(*voffset).as_int();
200: if(Value *vlimit = (Value *)options->get(sql_limit_name))
201: limit = r.process(*vlimit).as_int();
202: // no check on options count here
203: }
204:
205: if(file_spec.starts_with("http://")) {
206: if(offset || limit)
207: throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load");
208:
209: // fail on read problem
210: File_read_http_result http = pa_internal_file_read_http(r, file_spec, as_text, options, transcode_text_result);
211:
212: File_read_result result = {true, http.str, http.length, http.headers};
213: return result;
214: } else
215: return file_read(r.charsets, file_spec, as_text, options, fail_on_read_problem, offset, limit, transcode_text_result);
216: }
217:
218: char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem) {
219: File_read_result file = file_read(charsets, file_spec, true, 0, fail_on_read_problem);
220: return file.success ? file.str : 0;
221: }
222:
223: char* file_load_text(Request& r, const String& file_spec, bool fail_on_read_problem, HashStringValue* options, bool transcode_result) {
224: File_read_result file = file_load(r, file_spec, true, options, fail_on_read_problem, transcode_result);
225: return file.success ? file.str : 0;
226: }
227:
228: #ifdef PA_SAFE_MODE
229: void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) {
230: if(finfo.st_uid/*foreign?*/!=geteuid()
231: && finfo.st_gid/*foreign?*/!=getegid())
232: throw Exception(PARSER_RUNTIME,
233: &file_spec,
234: "parser is in safe mode: reading files of foreign group and user disabled "
235: "[recompile parser with --disable-safe-mode configure option], "
236: "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)",
237: fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid()
238: );
239: }
240: #else
241: void check_safe_mode(struct stat, const String&, const char*) {
242: }
243: #endif
244:
245:
246: bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context,
247: bool as_text, bool fail_on_read_problem) {
248:
249: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
250: int f;
251:
252: // first open, next stat:
253: // directory update of NTFS hard links performed on open.
1.3 moko 254: // ex:
1.1 moko 255: // a.html:^test[] and b.html hardlink to a.html
256: // user inserts ! before ^test in a.html
1.3 moko 257: // directory entry of b.html in NTFS not updated at once,
1.1 moko 258: // they delay update till open, so we would receive "!^test[" string
259: // if would do stat, next open.
260: // later: it seems, even this does not help sometimes
261: if((f=pa_open(fname, O_RDONLY | (as_text ? _O_TEXT : _O_BINARY) ))>=0) {
262: try {
263: int pa_errno=pa_lock_shared_blocking(f);
264: if(pa_errno!=0)
265: throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
266:
267: struct stat finfo;
268: if(pa_fstat(f, &finfo)!=0)
269: throw Exception("file.missing", // hardly possible: we just opened it OK
270: &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
271:
272: check_safe_mode(finfo, file_spec, fname);
273:
1.3 moko 274: action(finfo, f, file_spec, context);
1.1 moko 275: } catch(...) {
1.3 moko 276: pa_unlock(f);close(f);
1.1 moko 277: if(fail_on_read_problem)
278: rethrow;
279: return false;
1.3 moko 280: }
1.1 moko 281:
1.3 moko 282: pa_unlock(f);close(f);
1.1 moko 283: return true;
284: } else {
285: if(fail_on_read_problem)
286: throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0,
287: &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
288: return false;
289: }
290: }
291:
292: void create_dir_for_file(const String& file_spec) {
293: const char *str=file_spec.taint_cstr(String::L_FILE_SPEC);
294: if(str[0]){
295: const char *pos=str+1;
296: while((pos=strpbrk(pos, "/\\")) && pos[1]) { // to avoid trailing /, see #1166
297: pa_mkdir(pa_strdup(str,pos-str), 0775);
298: pos++;
299: }
300: }
301: }
302:
303: bool file_write_action_under_lock(const String& file_spec, const char* action_name, File_write_action action, void *context,
304: bool as_text, bool do_append, bool do_block, bool fail_on_lock_problem) {
305:
306: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
307: int f;
308: if(access(fname, W_OK)!=0) // no
309: create_dir_for_file(file_spec);
310:
311: if((f=pa_open(fname, O_CREAT | O_RDWR | (as_text ? _O_TEXT : _O_BINARY) | (do_append ? O_APPEND : PA_O_TRUNC), 0664))>=0) {
312: int pa_errno=do_block ? pa_lock_exclusive_blocking(f) : pa_lock_exclusive_nonblocking(f);
313: if(pa_errno!=0) {
314: Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
315: close(f);
316: if(fail_on_lock_problem)
317: throw e;
318: return false;
319: }
320:
321: try {
322: #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE))
323: struct stat finfo;
324: if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111)
325: fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any
326: #endif
327: action(f, context);
328: } catch(...) {
329: #ifdef HAVE_FTRUNCATE
330: if(!do_append)
331: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // one cannot use O_TRUNC, read lower
332: #endif
333: pa_unlock(f);close(f);
334: rethrow;
335: }
336:
337: #ifdef HAVE_FTRUNCATE
338: if(!do_append)
339: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // O_TRUNC truncates even exclusevely write-locked file [thanks to Igor Milyakov <virtan@rotabanner.com> for discovering]
340: #endif
341: pa_unlock(f);close(f);
342: return true;
343: } else
344: throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
345: // here should be nothing, see rethrow above
346: }
347:
348: #ifndef DOXYGEN
349: struct File_write_action_info {
350: const char* str;
351: size_t length;
352: };
353: #endif
354:
355: // a single write() must stay below INT_MAX as windows _write() reports written bytes as int
356: #define FILE_WRITE_CHUNK_SIZE (256*0x100000)
357:
358: static const char* bytes_left(size_t rest) {
359: return rest ? pa_strcat(", ", pa_uitoa(rest), " bytes left") : "";
360: }
361:
362: static void file_write_action(int f, void *context) {
363: File_write_action_info& info=*static_cast<File_write_action_info *>(context);
364:
365: const char* data=info.str;
366: size_t left=info.length;
367:
368: while(left) {
369: size_t portion=min(left, (size_t)FILE_WRITE_CHUNK_SIZE);
370: ssize_t written=write(f, data, portion);
371: if(written<0)
372: throw Exception("file.write", 0, "error writing %s bytes: %s (%d)%s", pa_uitoa(portion), strerror(errno), errno, bytes_left(left-portion));
373: if(written==0)
374: throw Exception("file.write", 0, "error writing %s bytes: zero bytes written%s", pa_uitoa(portion), bytes_left(left-portion));
375: data+=written;
376: left-=written;
377: }
378: }
379:
380: void file_write(
381: Request_charsets& charsets,
382: const String& file_spec,
383: const char* data,
1.3 moko 384: size_t size,
385: bool as_text,
1.1 moko 386: bool do_append,
387: Charset* asked_charset) {
388:
389: if(as_text && asked_charset){
390: String::C body=String::C(data, size);
391: body=Charset::transcode(body, charsets.source(), *asked_charset);
392: data=body.str;
393: size=body.length;
394: };
395:
396: File_write_action_info info={data, size};
397:
398: file_write_action_under_lock(
399: file_spec,
400: "write",
401: file_write_action,
402: &info,
403: as_text,
404: do_append);
405: }
406:
407: static size_t get_dir(char* fname, size_t helper_length){
408: bool dir=false;
409: size_t pos=0;
410: for(pos=helper_length; pos; pos--){
411: char c=fname[pos-1];
412: if(c=='/' || c=='\\'){
413: fname[pos-1]=0;
414: dir=true;
415: } else if(dir) break;
416: }
417: return pos;
418: }
419:
420: bool entry_exists(const char* fname, struct stat *afinfo) {
421: struct stat lfinfo;
422: bool result=pa_stat(fname, &lfinfo)==0;
423: if(afinfo)
424: *afinfo=lfinfo;
425: return result;
426: }
427:
428: bool entry_exists(const String& file_spec) {
1.3 moko 429: return entry_exists(file_spec.taint_cstr(String::L_FILE_SPEC), 0);
1.1 moko 430: }
431:
432: static bool entry_ifdir(char *fname, bool need_dir) {
433: if(need_dir){
434: size_t size=strlen(fname);
435: while(size) {
436: char c=fname[size-1];
437: if(c=='/' || c=='\\')
438: fname[--size]=0;
439: else
440: break;
441: }
442: }
443:
444: struct stat finfo;
445: if(entry_exists(fname, &finfo)) {
446: bool is_dir=(finfo.st_mode&S_IFDIR) != 0;
447: return is_dir==need_dir;
448: }
449: return false;
450: }
451:
452: static bool entry_ifdir(const String& file_spec, bool need_dir) {
453: return entry_ifdir(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir);
454: }
455:
456: // throws nothing! [this is required in file_move & file_delete]
457: static void rmdir(const String& file_spec, size_t pos_after) {
458: char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC);
459: size_t length=strlen(dir_spec);
460: while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){
461: #ifdef _MSC_VER
462: if(!entry_ifdir(dir_spec, true))
463: break;
464: DWORD attrs=GetFileAttributesW(PA_UTF16_ENC(dir_spec));
465: if(
466: (attrs==INVALID_FILE_ATTRIBUTES)
467: || !(attrs & FILE_ATTRIBUTE_DIRECTORY)
468: || (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
469: )
470: break;
471: #endif
472: if(pa_rmdir(dir_spec))
473: break;
474: };
475: }
476:
477: bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) {
1.3 moko 478: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 479: if(pa_unlink(fname)!=0) {
480: if(fail_on_problem)
481: throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0,
482: &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
483: else
484: return false;
485: }
486:
487: if(!keep_empty_dirs)
1.3 moko 488: rmdir(file_spec, 1);
1.1 moko 489:
490: return true;
491: }
492:
493: void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) {
1.3 moko 494: const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC);
495: const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 496:
1.3 moko 497: create_dir_for_file(new_spec);
1.1 moko 498:
499: if(pa_rename(old_spec_cstr, new_spec_cstr)!=0)
500: throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0,
501: &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr);
502:
503: if(!keep_empty_dirs)
1.3 moko 504: rmdir(old_spec, 1);
1.1 moko 505: }
506:
507:
508: bool file_exist(const String& file_spec) {
1.3 moko 509: return entry_ifdir(file_spec, false);
1.1 moko 510: }
511:
512: bool dir_exists(const String& file_spec) {
1.3 moko 513: return entry_ifdir(file_spec, true);
1.1 moko 514: }
515:
516: const String* file_exist(const String& path, const String& name) {
517: String& result=*new String(path);
518: if(path.last_char() != '/' && path.last_char() != '\\')
519: result << "/";
520: result << name;
521: return file_exist(result)?&result:0;
522: }
523:
524: bool file_executable(const String& file_spec) {
525: return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0;
526: }
527:
528: bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) {
1.3 moko 529: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
1.1 moko 530: struct stat finfo;
531: if(pa_stat(fname, &finfo)!=0) {
532: if(fail_on_read_problem)
533: throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname);
534: else
535: return false;
536: }
537: rsize=finfo.st_size;
538: ratime=(time_t)finfo.st_atime;
539: rmtime=(time_t)finfo.st_mtime;
540: rctime=(time_t)finfo.st_ctime;
541: return true;
542: }
543:
544: size_t check_file_size(uint64_t size, const String* file_spec){
545: if(size > (uint64_t)pa_file_size_limit)
1.2 moko 546: throw Exception(PARSER_RUNTIME, file_spec, "content size of %s bytes exceeds the limit (%s bytes)", pa_uitoa(size), pa_uitoa(pa_file_size_limit));
1.1 moko 547: return (size_t)size;
548: }
549:
1.3 moko 550: /**
1.1 moko 551: String related functions
552: */
553:
554: const char *pa_filename(const char *path){
555: const char *slash=strrpbrk(path, "/\\");
556: return slash ? slash+1 : path;
557: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_file.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main