Annotation of parser3/src/main/pa_file.C, revision 1.1
1.1 ! moko 1: /** @file
! 2: Parser: file and file system related functions.
! 3:
! 4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
! 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
! 6: */
! 7:
! 8: #include "pa_file.h"
! 9: #include "pa_dir.h"
! 10: #include "pa_charsets.h"
! 11: #include "pa_http.h"
! 12: #include "pa_convert_utf.h"
! 13:
! 14: #ifdef _MSC_VER
! 15: #include <windows.h>
! 16: #include <direct.h>
! 17: #endif
! 18:
! 19: volatile const char * IDENT_PA_FILE_C="$Id$" IDENT_PA_FILE_H;
! 20:
! 21: // some maybe-undefined constants
! 22:
! 23: #ifndef _O_TEXT
! 24: # define _O_TEXT 0
! 25: #endif
! 26: #ifndef _O_BINARY
! 27: # define _O_BINARY 0
! 28: #endif
! 29:
! 30: #ifdef HAVE_FTRUNCATE
! 31: # define PA_O_TRUNC 0
! 32: #else
! 33: # ifdef _O_TRUNC
! 34: # define PA_O_TRUNC _O_TRUNC
! 35: # else
! 36: # error you must have either ftruncate function or _O_TRUNC bit declared
! 37: # endif
! 38: #endif
! 39:
! 40: #ifdef WIN32
! 41:
! 42: bool is_os_absolute_path(const String& path) {
! 43: return is_os_absolute_path(path.cstr());
! 44: }
! 45:
! 46: bool is_os_absolute_path(const char* path) {
! 47: return path[0] && path[0]!=':' && path[1]==':' // DRIVE: (including drive-relative paths, for compatibility)
! 48: || path[0]=='\\' && path[1]=='\\' // UNC
! 49: || path[0]=='/' && path[1]=='/';
! 50: }
! 51: #endif
! 52:
! 53: // externs
! 54:
! 55: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset);
! 56:
! 57: // functions
! 58:
! 59: #ifdef _MSC_VER
! 60:
! 61: #define PA_UTF16_ENC(value) (const wchar_t *)pa_utf16_encode(value, pa_thread_request().charsets.source())
! 62:
! 63: int pa_stat(const char *pathname, struct stat *buffer){
! 64: return _wstat64(PA_UTF16_ENC(pathname), buffer);
! 65: }
! 66:
! 67: int pa_open(const char *pathname, int flags, int mode){
! 68: return _wopen(PA_UTF16_ENC(pathname), flags, mode);
! 69: }
! 70:
! 71: FILE *pa_fopen(const char *pathname, const char *mode){
! 72: return _wfopen(PA_UTF16_ENC(pathname), PA_UTF16_ENC(mode));
! 73: }
! 74:
! 75: int pa_mkdir(const char *pathname, int){
! 76: return _wmkdir(PA_UTF16_ENC(pathname));
! 77: }
! 78:
! 79: int pa_rmdir(const char *pathname){
! 80: return _wrmdir(PA_UTF16_ENC(pathname));
! 81: }
! 82:
! 83: int pa_rename(const char *oldpath, const char *newpath){
! 84: return _wrename(PA_UTF16_ENC(oldpath), PA_UTF16_ENC(newpath));
! 85: }
! 86:
! 87: int pa_unlink(const char *pathname){
! 88: return _wunlink(PA_UTF16_ENC(pathname));
! 89: }
! 90:
! 91: #else
! 92:
! 93: #define pa_mkdir mkdir
! 94: #define pa_rmdir rmdir
! 95: #define pa_rename rename
! 96: #define pa_unlink unlink
! 97:
! 98: #endif
! 99:
! 100: /// these options were handled but not checked elsewhere, now check them
! 101: int pa_get_valid_file_options_count(HashStringValue& options) {
! 102: int result=0;
! 103: if(options.get(PA_SQL_LIMIT_NAME))
! 104: result++;
! 105: if(options.get(PA_SQL_OFFSET_NAME))
! 106: result++;
! 107: if(options.get(PA_COLUMN_SEPARATOR_NAME))
! 108: result++;
! 109: if(options.get(PA_COLUMN_ENCLOSER_NAME))
! 110: result++;
! 111: if(options.get(PA_CHARSET_NAME))
! 112: result++;
! 113: return result;
! 114: }
! 115:
! 116: #ifndef DOXYGEN
! 117: struct File_read_action_info {
! 118: char **data; size_t *data_size;
! 119: char* buf; uint64_t offset; size_t limit;
! 120: };
! 121: #endif
! 122:
! 123: static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) {
! 124: File_read_action_info& info = *static_cast<File_read_action_info *>(context);
! 125: size_t to_read_size = check_file_size(info.limit && info.limit < (size_t)finfo.st_size ? info.limit : (size_t)finfo.st_size, &file_spec);
! 126: if(to_read_size) {
! 127: if(info.offset)
! 128: pa_lseek(f, info.offset, SEEK_SET); // seek never fails as POSIX allows the file offset to be set beyond the EOF
! 129: *info.data = info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1);
! 130: ssize_t result = read(f, *info.data, to_read_size);
! 131: if(result<0)
! 132: throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno);
! 133: *info.data_size = result;
! 134: } else { // empty file
! 135: // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files
! 136: *info.data = (char *)pa_malloc_atomic(1);
! 137: *(char*)(*info.data) = 0;
! 138: *info.data_size = 0;
! 139: return;
! 140: }
! 141: }
! 142:
! 143: File_read_result file_read_binary(const String& file_spec, bool fail_on_read_problem, char* buf, uint64_t offset, size_t limit) {
! 144: File_read_result result = {false, 0, 0, 0};
! 145: File_read_action_info info = {&result.str, &result.length, buf, offset, limit};
! 146:
! 147: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, 0, fail_on_read_problem);
! 148: return result;
! 149: }
! 150:
! 151: File_read_result file_read(Request_charsets& charsets, const String& file_spec,
! 152: bool as_text, HashStringValue *options,
! 153: bool fail_on_read_problem,
! 154: size_t offset = 0, size_t limit = 0, bool transcode_text_result = true) {
! 155: File_read_result result = {false, 0, 0, 0};
! 156: if(options){
! 157: int valid_options = pa_get_valid_file_options_count(*options);
! 158: if(valid_options != options->count())
! 159: throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
! 160: }
! 161:
! 162: File_read_action_info info = {&result.str, &result.length, 0, offset, limit};
! 163:
! 164: result.success = file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem);
! 165:
! 166: if(as_text){
! 167: if(result.success){
! 168: Charset* asked_charset = 0;
! 169: if(options)
! 170: if(Value* vcharset_name = options->get(PA_CHARSET_NAME))
! 171: asked_charset = &pa_charsets.get(vcharset_name->as_string());
! 172:
! 173: asked_charset = pa_charsets.checkBOM(result.str, result.length, asked_charset);
! 174:
! 175: if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
! 176: String::C body = String::C(result.str, result.length);
! 177: body=Charset::transcode(body, *asked_charset, charsets.source());
! 178:
! 179: result.str = const_cast<char*>(body.str); // hacking a little
! 180: result.length = body.length;
! 181: }
! 182: }
! 183: if(result.length)
! 184: fix_line_breaks(result.str, result.length);
! 185: }
! 186:
! 187: return result;
! 188: }
! 189:
! 190: File_read_result file_load(Request& r, const String& file_spec,
! 191: bool as_text, HashStringValue *options,
! 192: bool fail_on_read_problem,
! 193: bool transcode_text_result) {
! 194:
! 195: size_t offset = 0;
! 196: size_t limit = 0;
! 197:
! 198: if(options){
! 199: if(Value *voffset = (Value *)options->get(sql_offset_name))
! 200: offset = r.process(*voffset).as_int();
! 201: if(Value *vlimit = (Value *)options->get(sql_limit_name))
! 202: limit = r.process(*vlimit).as_int();
! 203: // no check on options count here
! 204: }
! 205:
! 206: if(file_spec.starts_with("http://")) {
! 207: if(offset || limit)
! 208: throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load");
! 209:
! 210: // fail on read problem
! 211: File_read_http_result http = pa_internal_file_read_http(r, file_spec, as_text, options, transcode_text_result);
! 212:
! 213: File_read_result result = {true, http.str, http.length, http.headers};
! 214: return result;
! 215: } else
! 216: return file_read(r.charsets, file_spec, as_text, options, fail_on_read_problem, offset, limit, transcode_text_result);
! 217: }
! 218:
! 219: char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem) {
! 220: File_read_result file = file_read(charsets, file_spec, true, 0, fail_on_read_problem);
! 221: return file.success ? file.str : 0;
! 222: }
! 223:
! 224: char* file_load_text(Request& r, const String& file_spec, bool fail_on_read_problem, HashStringValue* options, bool transcode_result) {
! 225: File_read_result file = file_load(r, file_spec, true, options, fail_on_read_problem, transcode_result);
! 226: return file.success ? file.str : 0;
! 227: }
! 228:
! 229: #ifdef PA_SAFE_MODE
! 230: void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) {
! 231: if(finfo.st_uid/*foreign?*/!=geteuid()
! 232: && finfo.st_gid/*foreign?*/!=getegid())
! 233: throw Exception(PARSER_RUNTIME,
! 234: &file_spec,
! 235: "parser is in safe mode: reading files of foreign group and user disabled "
! 236: "[recompile parser with --disable-safe-mode configure option], "
! 237: "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)",
! 238: fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid()
! 239: );
! 240: }
! 241: #else
! 242: void check_safe_mode(struct stat, const String&, const char*) {
! 243: }
! 244: #endif
! 245:
! 246:
! 247: bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context,
! 248: bool as_text, bool fail_on_read_problem) {
! 249:
! 250: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
! 251: int f;
! 252:
! 253: // first open, next stat:
! 254: // directory update of NTFS hard links performed on open.
! 255: // ex:
! 256: // a.html:^test[] and b.html hardlink to a.html
! 257: // user inserts ! before ^test in a.html
! 258: // directory entry of b.html in NTFS not updated at once,
! 259: // they delay update till open, so we would receive "!^test[" string
! 260: // if would do stat, next open.
! 261: // later: it seems, even this does not help sometimes
! 262: if((f=pa_open(fname, O_RDONLY | (as_text ? _O_TEXT : _O_BINARY) ))>=0) {
! 263: try {
! 264: int pa_errno=pa_lock_shared_blocking(f);
! 265: if(pa_errno!=0)
! 266: throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
! 267:
! 268: struct stat finfo;
! 269: if(pa_fstat(f, &finfo)!=0)
! 270: throw Exception("file.missing", // hardly possible: we just opened it OK
! 271: &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
! 272:
! 273: check_safe_mode(finfo, file_spec, fname);
! 274:
! 275: action(finfo, f, file_spec, context);
! 276: } catch(...) {
! 277: pa_unlock(f);close(f);
! 278: if(fail_on_read_problem)
! 279: rethrow;
! 280: return false;
! 281: }
! 282:
! 283: pa_unlock(f);close(f);
! 284: return true;
! 285: } else {
! 286: if(fail_on_read_problem)
! 287: throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0,
! 288: &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
! 289: return false;
! 290: }
! 291: }
! 292:
! 293: void create_dir_for_file(const String& file_spec) {
! 294: const char *str=file_spec.taint_cstr(String::L_FILE_SPEC);
! 295: if(str[0]){
! 296: const char *pos=str+1;
! 297: while((pos=strpbrk(pos, "/\\")) && pos[1]) { // to avoid trailing /, see #1166
! 298: pa_mkdir(pa_strdup(str,pos-str), 0775);
! 299: pos++;
! 300: }
! 301: }
! 302: }
! 303:
! 304: bool file_write_action_under_lock(const String& file_spec, const char* action_name, File_write_action action, void *context,
! 305: bool as_text, bool do_append, bool do_block, bool fail_on_lock_problem) {
! 306:
! 307: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
! 308: int f;
! 309: if(access(fname, W_OK)!=0) // no
! 310: create_dir_for_file(file_spec);
! 311:
! 312: if((f=pa_open(fname, O_CREAT | O_RDWR | (as_text ? _O_TEXT : _O_BINARY) | (do_append ? O_APPEND : PA_O_TRUNC), 0664))>=0) {
! 313: int pa_errno=do_block ? pa_lock_exclusive_blocking(f) : pa_lock_exclusive_nonblocking(f);
! 314: if(pa_errno!=0) {
! 315: Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(pa_errno), pa_errno, fname);
! 316: close(f);
! 317: if(fail_on_lock_problem)
! 318: throw e;
! 319: return false;
! 320: }
! 321:
! 322: try {
! 323: #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE))
! 324: struct stat finfo;
! 325: if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111)
! 326: fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any
! 327: #endif
! 328: action(f, context);
! 329: } catch(...) {
! 330: #ifdef HAVE_FTRUNCATE
! 331: if(!do_append)
! 332: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // one cannot use O_TRUNC, read lower
! 333: #endif
! 334: pa_unlock(f);close(f);
! 335: rethrow;
! 336: }
! 337:
! 338: #ifdef HAVE_FTRUNCATE
! 339: if(!do_append)
! 340: PA_UNUSED int ignore_result=ftruncate(f, lseek(f, 0, SEEK_CUR)); // O_TRUNC truncates even exclusevely write-locked file [thanks to Igor Milyakov <virtan@rotabanner.com> for discovering]
! 341: #endif
! 342: pa_unlock(f);close(f);
! 343: return true;
! 344: } else
! 345: throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname);
! 346: // here should be nothing, see rethrow above
! 347: }
! 348:
! 349: #ifndef DOXYGEN
! 350: struct File_write_action_info {
! 351: const char* str;
! 352: size_t length;
! 353: };
! 354: #endif
! 355:
! 356: // a single write() must stay below INT_MAX as windows _write() reports written bytes as int
! 357: #define FILE_WRITE_CHUNK_SIZE (256*0x100000)
! 358:
! 359: static const char* bytes_left(size_t rest) {
! 360: return rest ? pa_strcat(", ", pa_uitoa(rest), " bytes left") : "";
! 361: }
! 362:
! 363: static void file_write_action(int f, void *context) {
! 364: File_write_action_info& info=*static_cast<File_write_action_info *>(context);
! 365:
! 366: const char* data=info.str;
! 367: size_t left=info.length;
! 368:
! 369: while(left) {
! 370: size_t portion=min(left, (size_t)FILE_WRITE_CHUNK_SIZE);
! 371: ssize_t written=write(f, data, portion);
! 372: if(written<0)
! 373: throw Exception("file.write", 0, "error writing %s bytes: %s (%d)%s", pa_uitoa(portion), strerror(errno), errno, bytes_left(left-portion));
! 374: if(written==0)
! 375: throw Exception("file.write", 0, "error writing %s bytes: zero bytes written%s", pa_uitoa(portion), bytes_left(left-portion));
! 376: data+=written;
! 377: left-=written;
! 378: }
! 379: }
! 380:
! 381: void file_write(
! 382: Request_charsets& charsets,
! 383: const String& file_spec,
! 384: const char* data,
! 385: size_t size,
! 386: bool as_text,
! 387: bool do_append,
! 388: Charset* asked_charset) {
! 389:
! 390: if(as_text && asked_charset){
! 391: String::C body=String::C(data, size);
! 392: body=Charset::transcode(body, charsets.source(), *asked_charset);
! 393: data=body.str;
! 394: size=body.length;
! 395: };
! 396:
! 397: File_write_action_info info={data, size};
! 398:
! 399: file_write_action_under_lock(
! 400: file_spec,
! 401: "write",
! 402: file_write_action,
! 403: &info,
! 404: as_text,
! 405: do_append);
! 406: }
! 407:
! 408: static size_t get_dir(char* fname, size_t helper_length){
! 409: bool dir=false;
! 410: size_t pos=0;
! 411: for(pos=helper_length; pos; pos--){
! 412: char c=fname[pos-1];
! 413: if(c=='/' || c=='\\'){
! 414: fname[pos-1]=0;
! 415: dir=true;
! 416: } else if(dir) break;
! 417: }
! 418: return pos;
! 419: }
! 420:
! 421: bool entry_exists(const char* fname, struct stat *afinfo) {
! 422: struct stat lfinfo;
! 423: bool result=pa_stat(fname, &lfinfo)==0;
! 424: if(afinfo)
! 425: *afinfo=lfinfo;
! 426: return result;
! 427: }
! 428:
! 429: bool entry_exists(const String& file_spec) {
! 430: return entry_exists(file_spec.taint_cstr(String::L_FILE_SPEC), 0);
! 431: }
! 432:
! 433: static bool entry_ifdir(char *fname, bool need_dir) {
! 434: if(need_dir){
! 435: size_t size=strlen(fname);
! 436: while(size) {
! 437: char c=fname[size-1];
! 438: if(c=='/' || c=='\\')
! 439: fname[--size]=0;
! 440: else
! 441: break;
! 442: }
! 443: }
! 444:
! 445: struct stat finfo;
! 446: if(entry_exists(fname, &finfo)) {
! 447: bool is_dir=(finfo.st_mode&S_IFDIR) != 0;
! 448: return is_dir==need_dir;
! 449: }
! 450: return false;
! 451: }
! 452:
! 453: static bool entry_ifdir(const String& file_spec, bool need_dir) {
! 454: return entry_ifdir(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir);
! 455: }
! 456:
! 457: // throws nothing! [this is required in file_move & file_delete]
! 458: static void rmdir(const String& file_spec, size_t pos_after) {
! 459: char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC);
! 460: size_t length=strlen(dir_spec);
! 461: while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){
! 462: #ifdef _MSC_VER
! 463: if(!entry_ifdir(dir_spec, true))
! 464: break;
! 465: DWORD attrs=GetFileAttributesW(PA_UTF16_ENC(dir_spec));
! 466: if(
! 467: (attrs==INVALID_FILE_ATTRIBUTES)
! 468: || !(attrs & FILE_ATTRIBUTE_DIRECTORY)
! 469: || (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
! 470: )
! 471: break;
! 472: #endif
! 473: if(pa_rmdir(dir_spec))
! 474: break;
! 475: };
! 476: }
! 477:
! 478: bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) {
! 479: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
! 480: if(pa_unlink(fname)!=0) {
! 481: if(fail_on_problem)
! 482: throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0,
! 483: &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname);
! 484: else
! 485: return false;
! 486: }
! 487:
! 488: if(!keep_empty_dirs)
! 489: rmdir(file_spec, 1);
! 490:
! 491: return true;
! 492: }
! 493:
! 494: void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) {
! 495: const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC);
! 496: const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC);
! 497:
! 498: create_dir_for_file(new_spec);
! 499:
! 500: if(pa_rename(old_spec_cstr, new_spec_cstr)!=0)
! 501: throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0,
! 502: &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr);
! 503:
! 504: if(!keep_empty_dirs)
! 505: rmdir(old_spec, 1);
! 506: }
! 507:
! 508:
! 509: bool file_exist(const String& file_spec) {
! 510: return entry_ifdir(file_spec, false);
! 511: }
! 512:
! 513: bool dir_exists(const String& file_spec) {
! 514: return entry_ifdir(file_spec, true);
! 515: }
! 516:
! 517: const String* file_exist(const String& path, const String& name) {
! 518: String& result=*new String(path);
! 519: if(path.last_char() != '/' && path.last_char() != '\\')
! 520: result << "/";
! 521: result << name;
! 522: return file_exist(result)?&result:0;
! 523: }
! 524:
! 525: bool file_executable(const String& file_spec) {
! 526: return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0;
! 527: }
! 528:
! 529: bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) {
! 530: const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC);
! 531: struct stat finfo;
! 532: if(pa_stat(fname, &finfo)!=0) {
! 533: if(fail_on_read_problem)
! 534: throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname);
! 535: else
! 536: return false;
! 537: }
! 538: rsize=finfo.st_size;
! 539: ratime=(time_t)finfo.st_atime;
! 540: rmtime=(time_t)finfo.st_mtime;
! 541: rctime=(time_t)finfo.st_ctime;
! 542: return true;
! 543: }
! 544:
! 545: size_t check_file_size(uint64_t size, const String* file_spec){
! 546: if(size > (uint64_t)pa_file_size_limit)
! 547: throw Exception(PARSER_RUNTIME, file_spec, "content size of %.15g bytes exceeds the limit (%.15g bytes)", (double)size, (double)pa_file_size_limit);
! 548: return (size_t)size;
! 549: }
! 550:
! 551: /**
! 552: String related functions
! 553: */
! 554:
! 555: const char *pa_filename(const char *path){
! 556: const char *slash=strrpbrk(path, "/\\");
! 557: return slash ? slash+1 : path;
! 558: }
E-mail: