--- parser3/src/main/pa_common.C 2008/06/06 17:24:23 1.232 +++ parser3/src/main/pa_common.C 2010/08/10 05:27:46 1.263 @@ -1,7 +1,7 @@ /** @file Parser: commonly functions. - Copyright(c) 2001-2005 ArtLebedev Group (http://www.artlebedev.com) + Copyright(c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) * BASE64 part @@ -26,7 +26,7 @@ * */ -static const char * const IDENT_COMMON_C="$Date: 2008/06/06 17:24:23 $"; +static const char * const IDENT_COMMON_C="$Date: 2010/08/10 05:27:46 $"; #include "pa_common.h" #include "pa_exception.h" @@ -35,6 +35,8 @@ static const char * const IDENT_COMMON_C #include "pa_charsets.h" #include "pa_http.h" #include "pa_request_charsets.h" +#include "pcre.h" +#include "pa_request.h" // some maybe-undefined constants @@ -65,6 +67,31 @@ const String file_status_name(FILE_STATU // functions +bool capitalized(const char* s){ + bool upper=true; + for(const char* c=s; *c; c++){ + if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c))) + return false; + upper=strchr("-_ ", *c) != 0; + } + return true; +} + +const char* capitalize(const char* s){ + if(!s || capitalized(s)) + return s; + + char* result=pa_strdup(s); + if(result){ + bool upper=true; + for(char* c=result; *c; c++){ + *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c); + upper=strchr("-_ ", *c) != 0; + } + } + return (const char*)result; +} + void fix_line_breaks(char *str, size_t& length) { //_asm int 3; const char* const eob=str+length; @@ -75,7 +102,7 @@ void fix_line_breaks(char *str, size_t& while(char* eol=(char*)memchr(bol, '\r', eob -bol)) { size_t len=eol-bol; if(dest!=bol) - memcpy(dest, bol, len); + memmove(dest, bol, len); dest+=len; *dest++='\n'; @@ -87,22 +114,32 @@ void fix_line_breaks(char *str, size_t& } // last piece without \r if(dest!=bol) - memcpy(dest, bol, eob-bol); + memmove(dest, bol, eob-bol); str[length]=0; // terminating } char* file_read_text(Request_charsets& charsets, const String& file_spec, bool fail_on_read_problem, - HashStringValue* params/*, HashStringValue* * out_fields*/) { + HashStringValue* params, + bool transcode_result) { File_read_result file= - file_read(charsets, file_spec, true, params, fail_on_read_problem); + file_read(charsets, file_spec, true, params, fail_on_read_problem, 0, 0, 0, transcode_result); + return file.success?file.str:0; +} + +char* file_load_text(Request& r, + const String& file_spec, + bool fail_on_read_problem, + HashStringValue* params, + bool transcode_result) { + File_read_result file= + file_load(r, file_spec, true, params, fail_on_read_problem, 0, 0, 0, transcode_result); return file.success?file.str:0; } /// these options were handled but not checked elsewhere, now check them -int pa_get_valid_file_options_count(HashStringValue& options) -{ +int pa_get_valid_file_options_count(HashStringValue& options) { int result=0; if(options.get(PA_SQL_LIMIT_NAME)) result++; @@ -138,26 +175,74 @@ static void file_read_action( lseek(f, info.offset, SEEK_SET); *info.data=info.buf ? info.buf - : new(PointerFreeGC) char[to_read_size+(as_text?1:0)]; + : (char *)pa_malloc_atomic(to_read_size+1); *info.data_size=(size_t)read(f, *info.data, to_read_size); if(ssize_t(*info.data_size)<0 || *info.data_size>to_read_size) - throw Exception(0, + throw Exception("file.read", &file_spec, "read failed: actually read %u bytes count not in [0..%u] valid range", *info.data_size, to_read_size); } else { // empty file // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files - *info.data=new(PointerFreeGC) char[1]; + *info.data=(char *)pa_malloc_atomic(1); *(char*)(*info.data)=0; *info.data_size=0; return; } } + File_read_result file_read(Request_charsets& charsets, const String& file_spec, bool as_text, HashStringValue *params, bool fail_on_read_problem, - char* buf, size_t offset, size_t count) { + char* buf, size_t offset, size_t count, bool transcode_text_result) { + File_read_result result={false, 0, 0, 0}; + if(params){ + int valid_options=pa_get_valid_file_options_count(*params); + if(valid_options!=params->count()) + throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION); + } + + File_read_action_info info={&result.str, &result.length, buf, offset, count}; + + result.success=file_read_action_under_lock(file_spec, + "read", file_read_action, &info, + as_text, fail_on_read_problem); + + if(as_text){ + if(result.success){ + Charset* asked_charset=0; + if(result.length>=3 && strncmp(result.str, "\xEF\xBB\xBF", 3)==0){ + // skip UTF-8 signature (BOM code) + result.str+=3; + result.length-=3; + asked_charset=&UTF8_charset; + } + + if(params) + if(Value* vcharset_name=params->get(PA_CHARSET_NAME)) + asked_charset=&::charsets.get(vcharset_name->as_string().change_case(charsets.source(), String::CC_UPPER)); + + if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below + String::C body=String::C(result.str, result.length); + body=Charset::transcode(body, *asked_charset, charsets.source()); + + result.str=const_cast(body.str); // hacking a little + result.length=body.length; + } + } + if(result.length) + fix_line_breaks(result.str, result.length); + } + + return result; +} + +File_read_result file_load(Request& r, const String& file_spec, + bool as_text, HashStringValue *params, + bool fail_on_read_problem, + char* buf, size_t offset, size_t count, bool transcode_text_result) { + File_read_result result={false, 0, 0, 0}; if(file_spec.starts_with("http://")) { if(offset || count) @@ -166,57 +251,21 @@ File_read_result file_read(Request_chars "offset and load options are not supported for HTTP:// file load"); // fail on read problem - File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params); + File_read_http_result http=pa_internal_file_read_http(r, file_spec, as_text, params, transcode_text_result); result.success=true; result.str=http.str; result.length=http.length; result.headers=http.headers; - } else { - if(params) { - int valid_options=pa_get_valid_file_options_count(*params); - if(valid_options!=params->count()) - throw Exception(PARSER_RUNTIME, - 0, - "invalid option passed"); - } - - File_read_action_info info={&result.str, &result.length, - buf, offset, count}; - result.success=file_read_action_under_lock(file_spec, - "read", file_read_action, &info, - as_text, fail_on_read_problem); - - if(result.length && as_text && params) { - if( Value* vcharset_name=params->get(PA_CHARSET_NAME) ) { - Charset asked_charset=::charsets.get(vcharset_name->as_string(). - change_case(charsets.source(), String::CC_UPPER)); - - String::C body=String::C(result.str, result.length); - body=Charset::transcode(body, asked_charset, charsets.source()); - - result.str=const_cast(body.str); // hacking a little - result.length=body.length; - } - } - } - - if(result.success && as_text) { - // UTF-8 signature: EF BB BF - if(result.length>=3) { - char *in=(char *)result.str; - if(strncmp(in, "\xEF\xBB\xBF", 3)==0) { - result.str=in+3; result.length-=3;// skip prefix - } - } - - fix_line_breaks((char *)(result.str), result.length); - } + } else + result= + file_read(r.charsets, file_spec, as_text, params, fail_on_read_problem, buf, offset, count, transcode_text_result); return result; } + #ifdef PA_SAFE_MODE -void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) { +void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) { if(finfo.st_uid/*foreign?*/!=geteuid() && finfo.st_gid/*foreign?*/!=getegid()) throw Exception(PARSER_RUNTIME, @@ -228,15 +277,20 @@ void check_safe_mode(struct stat finfo, "fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)", fname, finfo.st_uid, geteuid(), - finfo.st_gid, getegid()); -} -#endif + finfo.st_gid, getegid()); +} +#else +void check_safe_mode(struct stat, const String&, const char*) { +} +#endif + + bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context, bool as_text, bool fail_on_read_problem) { - const char* fname=file_spec.cstr(String::L_FILE_SPEC); + const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); int f; // first open, next stat: @@ -257,15 +311,13 @@ bool file_read_action_under_lock(const S strerror(errno), errno, fname); struct stat finfo; - if(stat(fname, &finfo)!=0) + if(fstat(f, &finfo)!=0) throw Exception("file.missing", // hardly possible: we just opened it OK &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); -#ifdef PA_SAFE_MODE check_safe_mode(finfo, file_spec, fname); -#endif action(finfo, f, file_spec, fname, as_text, context); } catch(...) { @@ -279,7 +331,8 @@ bool file_read_action_under_lock(const S return true; } else { if(fail_on_read_problem) - throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, + throw Exception(errno==EACCES?"file.access" + :(errno==ENOENT || errno==ENOTDIR || errno==ENODEV)?"file.missing":0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname); @@ -291,7 +344,7 @@ void create_dir_for_file(const String& f size_t pos_after=1; size_t pos_before; while((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) { - mkdir(file_spec.mid(0, pos_before).cstr(String::L_FILE_SPEC), 0775); + mkdir(file_spec.mid(0, pos_before).taint_cstr(String::L_FILE_SPEC), 0775); pos_after=pos_before+1; } } @@ -305,7 +358,7 @@ bool file_write_action_under_lock( bool do_append, bool do_block, bool fail_on_lock_problem) { - const char* fname=file_spec.cstr(String::L_FILE_SPEC); + const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); int f; if(access(fname, W_OK)!=0) // no create_dir_for_file(file_spec); @@ -326,7 +379,12 @@ bool file_write_action_under_lock( } try { - action(f, context); +#if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE)) + struct stat finfo; + if(fstat(f, &finfo)==0 && finfo.st_mode & 0111) + fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any +#endif + action(f, context); } catch(...) { #ifdef HAVE_FTRUNCATE if(!do_append) @@ -352,7 +410,8 @@ bool file_write_action_under_lock( #ifndef DOXYGEN struct File_write_action_info { - const char* str; size_t length; + const char* str; + size_t length; }; #endif static void file_write_action(int f, void *context) { @@ -360,16 +419,27 @@ static void file_write_action(int f, voi if(info.length) { int written=write(f, info.str, info.length); if(written<0) - throw Exception(0, + throw Exception("file.access", 0, "write failed: %s (%d)", strerror(errno), errno); } } void file_write( - const String& file_spec, - const char* data, size_t size, + Request_charsets& charsets, + const String& file_spec, + const char* data, + size_t size, bool as_text, - bool do_append) { + bool do_append, + Charset* asked_charset) { + + if(as_text && asked_charset){ + String::C body=String::C(data, size); + body=Charset::transcode(body, charsets.source(), *asked_charset); + data=body.str; + size=body.length; + }; + File_write_action_info info={data, size}; file_write_action_under_lock( @@ -381,16 +451,66 @@ void file_write( do_append); } +static size_t get_dir(char* fname, size_t helper_length){ + bool dir=false; + size_t pos=0; + for(pos=helper_length; pos; pos--){ + char c=fname[pos-1]; + if(c=='/' || c=='\\'){ + fname[pos-1]=0; + dir=true; + } else if(dir) break; + } + return pos; +} + +static bool entry_readable(char* fname, bool need_dir) { + if(need_dir){ + size_t size=strlen(fname); + while(size) { + char c=fname[size-1]; + if(c=='/' || c=='\\') + fname[--size]=0; + else + break; + } + } + + struct stat finfo; + if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { + bool is_dir=(finfo.st_mode&S_IFDIR) != 0; + return is_dir==need_dir; + } + return false; +} + +static bool entry_readable(const String& file_spec, bool need_dir) { + return entry_readable(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir); +} + // throws nothing! [this is required in file_move & file_delete] -static void rmdir(const String& file_spec, size_t pos_after) { - size_t pos_before; - if((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) - rmdir(file_spec, pos_before+1); - - rmdir(file_spec.mid(0, pos_after-1/* / */).cstr(String::L_FILE_SPEC)); +static void rmdir(const String& file_spec, size_t pos_after=0) { + char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC); + size_t length=strlen(dir_spec); + while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){ +#ifdef WIN32 + if(!entry_readable(dir_spec, true)) + break; + DWORD attrs=GetFileAttributes(dir_spec); + if( + (attrs==INVALID_FILE_ATTRIBUTES) + || !(attrs & FILE_ATTRIBUTE_DIRECTORY) + || (attrs & FILE_ATTRIBUTE_REPARSE_POINT) + ) + break; +#endif + if( rmdir(dir_spec) ) + break; + }; } + bool file_delete(const String& file_spec, bool fail_on_problem) { - const char* fname=file_spec.cstr(String::L_FILE_SPEC); + const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); if(unlink(fname)!=0) if(fail_on_problem) throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, @@ -403,9 +523,10 @@ bool file_delete(const String& file_spec rmdir(file_spec, 1); return true; } + void file_move(const String& old_spec, const String& new_spec) { - const char* old_spec_cstr=old_spec.cstr(String::L_FILE_SPEC); - const char* new_spec_cstr=new_spec.cstr(String::L_FILE_SPEC); + const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC); + const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC); create_dir_for_file(new_spec); @@ -428,43 +549,27 @@ bool entry_exists(const char* fname, str } bool entry_exists(const String& file_spec) { - const char* fname=file_spec.cstr(String::L_FILE_SPEC); + const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); return entry_exists(fname, 0); } -static bool entry_readable(const String& file_spec, bool need_dir) { - char* fname=file_spec.cstrm(String::L_FILE_SPEC); - if(need_dir) { - size_t size=strlen(fname); - while(size) { - char c=fname[size-1]; - if(c=='/' || c=='\\') - fname[--size]=0; - else - break; - } - } - struct stat finfo; - if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { - bool is_dir=(finfo.st_mode&S_IFDIR) != 0; - return is_dir==need_dir; - } - return false; -} bool file_exist(const String& file_spec) { return entry_readable(file_spec, false); } + bool dir_exists(const String& file_spec) { return entry_readable(file_spec, true); } + const String* file_exist(const String& path, const String& name) { String& result=*new String(path); result << "/"; result << name; return file_exist(result)?&result:0; } + bool file_executable(const String& file_spec) { - return access(file_spec.cstr(String::L_FILE_SPEC), X_OK)==0; + return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0; } bool file_stat(const String& file_spec, @@ -473,7 +578,7 @@ bool file_stat(const String& file_spec, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) { - const char* fname=file_spec.cstr(String::L_FILE_SPEC); + const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); struct stat finfo; if(stat(fname, &finfo)!=0) if(fail_on_read_problem) @@ -598,7 +703,7 @@ FormatType format_type(char* fmt){ const char* format(double value, char* fmt) { char local_buf[MAX_NUMBER]; - size_t size; + int size=-1; if(fmt && strlen(fmt)){ switch(format_type(fmt)){ @@ -625,7 +730,7 @@ const char* format(double value, char* f "Error occure white executing snprintf with format string '%s'.", fmt); } - return pa_strdup(local_buf, size); + return pa_strdup(local_buf, (size_t)size); } size_t stdout_write(const void *buf, size_t size) { @@ -651,57 +756,23 @@ enum EscapeState { EscapeSecond, EscapeUnicode }; -/* -char* unescape_chars(const char* cp, int len) { - char* s=new(PointerFreeGC) char[len + 1]; - EscapeState escapeState=EscapeRest; - uint escapedValue=0; - int srcPos=0; - int dstPos=0; - while(srcPos < len) { - uchar ch=(uchar)cp[srcPos]; - switch(escapeState) { - case EscapeRest: - if(ch=='%') { - escapeState=EscapeFirst; - } else if(ch=='+'){ - s[dstPos++]=' '; - } else { - s[dstPos++]=ch; - } - break; - case EscapeFirst: - escapedValue=hex_value[ch] << 4; - escapeState=EscapeSecond; - break; - case EscapeSecond: - escapedValue +=hex_value[ch]; - s[dstPos++]=(char)escapedValue; - escapeState=EscapeRest; - break; - } - srcPos++; - } - s[dstPos]=0; - return s; -} -*/ -char* unescape_chars(const char* cp, int len, Charset* charset){ - char* s=new(PointerFreeGC) char[len + 1]; // enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes) +// @todo prescan for reduce required size (unescaped sting in 1 byte charset requires less memory usually) +char* unescape_chars(const char* cp, int len, Charset* charset, bool js){ + char* s=new(PointerFreeGC) char[len+1]; // must be enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes) char* dst=s; EscapeState escapeState=EscapeRest; uint escapedValue=0; int srcPos=0; short int jsCnt=0; - while(srcPos < len) { + while(srcPos> 6)]; *outptr++ = base64_alphabet [c3 & 0x3f]; /* this is a bit ugly ... */ - if ((++already) >= 19) { + if ((++already) >= BASE64_GROUPS_IN_LINE) { *outptr++ = '\n'; already = 0; } @@ -1205,25 +1249,25 @@ g_mime_utils_base64_decode_step (const u } -char* pa_base64_encode(const char *in, size_t in_size) -{ - /* wont go to more than 2x size (overly conservative) */ - char* result=new(PointerFreeGC) char[in_size * 2 + 6]; +char* pa_base64_encode(const char *in, size_t in_size){ + size_t new_size = ((in_size / 3 + 1) * 4); + new_size += new_size / (BASE64_GROUPS_IN_LINE * 4)/*new lines*/ + 1/*zero terminator*/; + char* result = new(PointerFreeGC) char[new_size]; int state=0; int save=0; #ifndef NDEBUG size_t filled= #endif - g_mime_utils_base64_encode_close ((const unsigned char*)in, in_size, - (unsigned char*)result, &state, &save); - assert(filled <= in_size * 2 + 6); + g_mime_utils_base64_encode_close ((const unsigned char*)in, in_size, (unsigned char*)result, &state, &save); + + //throw Exception(PARSER_RUNTIME, 0, "%d %d %d", in_size, new_size, filled); + assert(filled <= new_size); return result; } -char* pa_base64_encode(const String& file_spec) -{ +char* pa_base64_encode(const String& file_spec){ unsigned char* base64=0; File_base64_action_info info={&base64}; @@ -1259,8 +1303,7 @@ static void file_base64_file_action( } } -void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) -{ +void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) { /* wont go to more than had (overly conservative) */ result=new(PointerFreeGC) char[in_size+1/*terminator*/]; int state=0; @@ -1276,24 +1319,23 @@ void pa_base64_decode(const char *in, si int file_block_read(const int f, unsigned char* buffer, const size_t size){ int nCount = read(f, buffer, size); if (nCount < 0) - throw Exception(0, + throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno); return nCount; } -const unsigned long pa_crc32(const char *in, size_t in_size) -{ +const unsigned long pa_crc32(const char *in, size_t in_size){ unsigned long crc32=0xFFFFFFFF; - InitCrc32Table(); - for(size_t i = 0; i < in_size; i++) CalcCrc32(in[i], crc32); + InitCrc32Table(); + for(size_t i = 0; i(context); if(finfo.st_size) { InitCrc32Table();