--- parser3/src/main/pa_common.C 2007/05/23 08:31:47 1.225 +++ parser3/src/main/pa_common.C 2009/01/12 07:15:45 1.239 @@ -26,7 +26,7 @@ * */ -static const char * const IDENT_COMMON_C="$Date: 2007/05/23 08:31:47 $"; +static const char * const IDENT_COMMON_C="$Date: 2009/01/12 07:15:45 $"; #include "pa_common.h" #include "pa_exception.h" @@ -35,6 +35,7 @@ static const char * const IDENT_COMMON_C #include "pa_charsets.h" #include "pa_http.h" #include "pa_request_charsets.h" +#include "pcre.h" // some maybe-undefined constants @@ -92,17 +93,17 @@ void fix_line_breaks(char *str, size_t& } char* file_read_text(Request_charsets& charsets, - const String& file_spec, - bool fail_on_read_problem, - HashStringValue* params/*, HashStringValue* * out_fields*/) { + const String& file_spec, + bool fail_on_read_problem, + HashStringValue* params, + bool transcode_result) { File_read_result file= - file_read(charsets, file_spec, true, params, fail_on_read_problem); + file_read(charsets, file_spec, true, params, fail_on_read_problem, 0, 0, 0, transcode_result); return file.success?file.str:0; } /// these options were handled but not checked elsewhere, now check them -int pa_get_valid_file_options_count(HashStringValue& options) -{ +int pa_get_valid_file_options_count(HashStringValue& options) { int result=0; if(options.get(PA_SQL_LIMIT_NAME)) result++; @@ -124,10 +125,10 @@ struct File_read_action_info { }; #endif static void file_read_action( - struct stat& finfo, - int f, - const String& file_spec, const char* /*fname*/, bool as_text, - void *context) { + struct stat& finfo, + int f, + const String& file_spec, const char* /*fname*/, bool as_text, + void *context) { File_read_action_info& info=*static_cast(context); size_t to_read_size=info.count; if(!to_read_size) @@ -142,7 +143,7 @@ static void file_read_action( *info.data_size=(size_t)read(f, *info.data, to_read_size); if(ssize_t(*info.data_size)<0 || *info.data_size>to_read_size) - throw Exception(0, + throw Exception("file.read", &file_spec, "read failed: actually read %u bytes count not in [0..%u] valid range", *info.data_size, to_read_size); @@ -155,9 +156,9 @@ static void file_read_action( } } File_read_result file_read(Request_charsets& charsets, const String& file_spec, - bool as_text, HashStringValue *params, - bool fail_on_read_problem, - char* buf, size_t offset, size_t count) { + bool as_text, HashStringValue *params, + bool fail_on_read_problem, + char* buf, size_t offset, size_t count, bool transcode_text_result) { File_read_result result={false, 0, 0, 0}; if(file_spec.starts_with("http://")) { if(offset || count) @@ -166,13 +167,13 @@ File_read_result file_read(Request_chars "offset and load options are not supported for HTTP:// file load"); // fail on read problem - File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params); + File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params, transcode_text_result); result.success=true; result.str=http.str; result.length=http.length; result.headers=http.headers; } else { - if(params) { + if(params){ int valid_options=pa_get_valid_file_options_count(*params); if(valid_options!=params->count()) throw Exception(PARSER_RUNTIME, @@ -186,31 +187,30 @@ File_read_result file_read(Request_chars "read", file_read_action, &info, as_text, fail_on_read_problem); - if(result.length && as_text && params) { - if( Value* vcharset_name=params->get(PA_CHARSET_NAME) ) { - Charset asked_charset=::charsets.get(vcharset_name->as_string(). - change_case(charsets.source(), String::CC_UPPER)); - - String::C body=String::C(result.str, result.length); - body=Charset::transcode(body, asked_charset, charsets.source()); - - result.str=const_cast(body.str); // hacking a little - result.length=body.length; - } - } - } - - if(result.success && as_text) { - // UTF-8 signature: EF BB BF - if(result.length>=3) { - char *in=(char *)result.str; - if(strncmp(in, "\xEF\xBB\xBF", 3)==0) { - result.str=in+3; result.length-=3;// skip prefix + if(as_text && result.success){ + if(result.length>=3 && strncmp(result.str, "\xEF\xBB\xBF", 3)==0){ + // skip UTF-8 signature: EF BB BF (BOM code) + result.str+=3; + result.length-=3; + } + + if(result.length && transcode_text_result && params){ // must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below + if( Value* vcharset_name=params->get(PA_CHARSET_NAME) ){ + Charset asked_charset=::charsets.get(vcharset_name->as_string(). + change_case(charsets.source(), String::CC_UPPER)); + + String::C body=String::C(result.str, result.length); + body=Charset::transcode(body, asked_charset, charsets.source()); + + result.str=const_cast(body.str); // hacking a little + result.length=body.length; + } } } - - fix_line_breaks((char *)(result.str), result.length); } + + if(as_text && result.length) + fix_line_breaks(result.str, result.length); return result; } @@ -248,7 +248,7 @@ bool file_read_action_under_lock(const S // they delay update till open, so we would receive "!^test[" string // if would do stat, next open. // later: it seems, even this does not help sometimes - if((f=open(fname, O_RDONLY|(as_text?_O_TEXT:_O_BINARY)))>=0) { + if((f=open(fname, O_RDONLY|(as_text?_O_TEXT:_O_BINARY)))>=0) { try { if(pa_lock_shared_blocking(f)!=0) throw Exception("file.lock", @@ -277,7 +277,7 @@ bool file_read_action_under_lock(const S pa_unlock(f);close(f); return true; - } else { + } else { if(fail_on_read_problem) throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, &file_spec, @@ -360,7 +360,7 @@ static void file_write_action(int f, voi if(info.length) { int written=write(f, info.str, info.length); if(written<0) - throw Exception(0, + throw Exception("file.access", 0, "write failed: %s (%d)", strerror(errno), errno); } @@ -389,6 +389,7 @@ static void rmdir(const String& file_spe rmdir(file_spec.mid(0, pos_after-1/* / */).cstr(String::L_FILE_SPEC)); } + bool file_delete(const String& file_spec, bool fail_on_problem) { const char* fname=file_spec.cstr(String::L_FILE_SPEC); if(unlink(fname)!=0) @@ -403,6 +404,7 @@ bool file_delete(const String& file_spec rmdir(file_spec, 1); return true; } + void file_move(const String& old_spec, const String& new_spec) { const char* old_spec_cstr=old_spec.cstr(String::L_FILE_SPEC); const char* new_spec_cstr=new_spec.cstr(String::L_FILE_SPEC); @@ -451,28 +453,32 @@ static bool entry_readable(const String& } return false; } + bool file_exist(const String& file_spec) { return entry_readable(file_spec, false); } + bool dir_exists(const String& file_spec) { return entry_readable(file_spec, true); } + const String* file_exist(const String& path, const String& name) { String& result=*new String(path); result << "/"; result << name; return file_exist(result)?&result:0; } + bool file_executable(const String& file_spec) { - return access(file_spec.cstr(String::L_FILE_SPEC), X_OK)==0; + return access(file_spec.cstr(String::L_FILE_SPEC), X_OK)==0; } bool file_stat(const String& file_spec, - size_t& rsize, - time_t& ratime, - time_t& rmtime, - time_t& rctime, - bool fail_on_read_problem) { + size_t& rsize, + time_t& ratime, + time_t& rmtime, + time_t& rctime, + bool fail_on_read_problem) { const char* fname=file_spec.cstr(String::L_FILE_SPEC); struct stat finfo; if(stat(fname, &finfo)!=0) @@ -491,63 +497,141 @@ bool file_stat(const String& file_spec, } char* getrow(char* *row_ref, char delim) { - char* result=*row_ref; - if(result) { + char* result=*row_ref; + if(result) { *row_ref=strchr(result, delim); if(*row_ref) *((*row_ref)++)=0; else if(!*result) return 0; - } - return result; + } + return result; } char* lsplit(char* string, char delim) { - if(string) { + if(string) { char* v=strchr(string, delim); if(v) { *v=0; return v+1; } - } - return 0; + } + return 0; } char* lsplit(char* *string_ref, char delim) { - char* result=*string_ref; + char* result=*string_ref; char* next=lsplit(*string_ref, delim); - *string_ref=next; - return result; + *string_ref=next; + return result; } char* rsplit(char* string, char delim) { - if(string) { + if(string) { char* v=strrchr(string, delim); if(v) { *v=0; return v+1; } - } - return NULL; + } + return NULL; +} + + +// format: %[flags][width][.precision]type http://msdn.microsoft.com/ru-ru/library/56e442dc(en-us,VS.80).aspx +// flags: '-', '+', ' ', '#', '0' http://msdn.microsoft.com/ru-ru/library/8aky45ct(en-us,VS.80).aspx +// width, precision: non negative decimal number +enum FormatType { + FormatInvalid, + FormatInt, + FormatUInt, + FormatDouble +}; +FormatType format_type(char* fmt){ + enum FormatState { + Percent, + Flags, + Width, + Precision, + Done + } state=Percent; + + FormatType result=FormatInvalid; + + char* pos=fmt; + while(char c=*(pos++)){ + switch(state){ + case Percent: + if(c=='%'){ + state=Flags; + } else { + return FormatInvalid; // 1st char must be '%' only + } + break; + case Flags: + if(strchr("-+ #0", c)!=0){ + break; + } + // go to the next step + case Width: + if(c=='.'){ + state=Precision; + break; + } + // go to the next step + case Precision: + if(c>='0' && c<='9'){ + if(state == Flags) state=Width; // no more flags + break; + } else if(c=='d' || c=='i'){ + result=FormatInt; + } else if(strchr("feEgG", c)!=0){ + result=FormatDouble; + } else if(strchr("uoxX", c)!=0){ + result=FormatUInt; + } else { + return FormatInvalid; // invalid char + } + state=Done; + break; + case Done: + return FormatInvalid; // no chars allowed after 'type' + } + } + return result; } -/// @todo less stupid type detection + const char* format(double value, char* fmt) { - char local_buf[MAX_NUMBER]; - size_t size; - - if(fmt) - if(strpbrk(fmt, "diouxX")) - if(strpbrk(fmt, "ouxX")) - size=snprintf(local_buf, sizeof(local_buf), fmt, (uint)value); - else + char local_buf[MAX_NUMBER]; + int size=-1; + + if(fmt && strlen(fmt)){ + switch(format_type(fmt)){ + case FormatDouble: + size=snprintf(local_buf, sizeof(local_buf), fmt, value); + break; + case FormatInt: size=snprintf(local_buf, sizeof(local_buf), fmt, (int)value); - else - size=snprintf(local_buf, sizeof(local_buf), fmt, value); - else - size=snprintf(local_buf, sizeof(local_buf), "%d", (int)value); - - return pa_strdup(local_buf, size); + break; + case FormatUInt: + size=snprintf(local_buf, sizeof(local_buf), fmt, (uint)value); + break; + case FormatInvalid: + throw Exception(PARSER_RUNTIME, + 0, + "Incorrect format string '%s' was specified.", fmt); + } + } else + size=snprintf(local_buf, sizeof(local_buf), "%d", (int)value); + + if(size < 0 || size >= MAX_NUMBER-1){ // on win32 we manually reduce max size while printing + throw Exception(PARSER_RUNTIME, + 0, + "Error occure white executing snprintf with format string '%s'.", fmt); + } + + return pa_strdup(local_buf, (size_t)size); } size_t stdout_write(const void *buf, size_t size) { @@ -567,41 +651,77 @@ size_t stdout_write(const void *buf, siz #endif } -char* unescape_chars(const char* cp, int len) { - char* s=new(PointerFreeGC) char[len + 1]; - enum EscapeState { - EscapeRest, - EscapeFirst, - EscapeSecond - } escapeState=EscapeRest; +enum EscapeState { + EscapeRest, + EscapeFirst, + EscapeSecond, + EscapeUnicode +}; + +// @todo prescan for reduce required size (unescaped sting in 1 byte charset requires less memory usually) +char* unescape_chars(const char* cp, int len, Charset* charset, bool ignore_plus){ + char* s=new(PointerFreeGC) char[len+1]; // must be enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes) + char* dst=s; + EscapeState escapeState=EscapeRest; uint escapedValue=0; int srcPos=0; - int dstPos=0; - while(srcPos < len) { - uchar ch=(uchar)cp[srcPos]; - switch(escapeState) { - case EscapeRest: - if(ch=='%') { - escapeState=EscapeFirst; - } else if(ch=='+') { - s[dstPos++]=' '; - } else { - s[dstPos++]=ch; + short int jsCnt=0; + while(srcPosstore_Char((XMLByte*&)dst, (XMLCh)escapedValue, '?'); + escapeState=EscapeRest; + } + } else { + // not full unicode value + escapeState=EscapeRest; + } + break; } - break; - case EscapeFirst: - escapedValue=hex_value[ch] << 4; - escapeState=EscapeSecond; - break; - case EscapeSecond: - escapedValue +=hex_value[ch]; - s[dstPos++]=(char)escapedValue; - escapeState=EscapeRest; - break; } - srcPos++; + + srcPos++; } - s[dstPos]=0; + + *dst=0; // zero-termination return s; } @@ -622,55 +742,145 @@ void slashes_to_back_slashes(char* s) { */ #endif -bool StrEqNc(const char* s1, const char* s2, bool strict) { +bool StrStartFromNC(const char* str, const char* substr, bool equal){ while(true) { - if(!(*s1)) { - if(!(*s2)) + if(!(*substr)){ + if(!(*str)) return true; else - return !strict; - } else if(!(*s2)) - return !strict; - if(isalpha((unsigned char)*s1)) { - if(tolower((unsigned char)*s1) !=tolower((unsigned char)*s2)) + return !equal; + } + if(!(*str)) + return false; + if(isalpha((unsigned char)*str)) { + if(tolower((unsigned char)*str)!=tolower((unsigned char)*substr)) return false; - } else if((*s1) !=(*s2)) + } else if((*str) != (*substr)) return false; - s1++; - s2++; + str++; + substr++; + } +} + +size_t strpos(const char *str, const char *substr) { + const char *p = strstr(str, substr); + return (p==0)?STRING_NOT_FOUND:p-str; +} + +// content-type: xxx; charset=WE-NEED-THIS +// content-type: xxx; charset="WE-NEED-THIS" +// content-type: xxx; charset="WE-NEED-THIS"; +Charset* detect_charset(const char* content_type){ + if(content_type){ + size_t len=strlen(content_type); + char* CONTENT_TYPE=new(PointerFreeGC) char[len+1]; + memcpy(CONTENT_TYPE, content_type, len); + for(char *p=CONTENT_TYPE; *p; p++) + *p=(char)toupper((unsigned char)*p); + + if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){ + begin+=8; // skip "CHARSET=" + char* end=0; + if(*begin && (*begin=='"' || *begin =='\'')){ + char quote=*begin; + begin++; + end=(char*)strchr(begin, quote); + } + if(!end) + end=(char*)strchr(begin, ';'); + + size_t len; + if(end){ + *end=0; // terminator + len=end-begin; + } else { + len=strlen(begin); + } + + String::Body NAME=String::Body(begin, len); + return &charsets.get(NAME); + } } + return 0; } +Charset* detect_charset(Charset& source_charset, const String& content_type){ + const String& CONTENT_TYPE=content_type.change_case(source_charset, String::CC_UPPER); + size_t begin=CONTENT_TYPE.pos("CHARSET="); + if(begin!=STRING_NOT_FOUND) { + begin+=8; // skip "CHARSET=" + size_t end=STRING_NOT_FOUND; + + if(CONTENT_TYPE.pos('"', begin)==begin){ + begin++; + end=CONTENT_TYPE.pos('"', begin); + } else if(CONTENT_TYPE.pos('\'', begin)==begin){ + begin++; + end=CONTENT_TYPE.pos('\'', begin); + } + + if(end==STRING_NOT_FOUND) + end=CONTENT_TYPE.pos(';', begin); + + if(end==STRING_NOT_FOUND) + end=CONTENT_TYPE.length(); + + const String::Body NAME=CONTENT_TYPE.mid(begin, end); + return &charsets.get(NAME); + } + return 0; +} + + static bool isLeap(int year) { - return !( - (year % 4) || ((year % 400) && !(year % 100)) - ); + return !( + (year % 4) || ((year % 400) && !(year % 100)) + ); } int getMonthDays(int year, int month) { static int monthDays[]={ - 31, - 28, - 31, - 30, - 31, - 30, - 31, - 31, - 30, - 31, - 30, - 31 - }; - return (month == 2 && isLeap(year)) ? 29 : monthDays[month]; -} - -void remove_crlf(char* start, char* end) { - for(char* p=start; p(context); @@ -1025,8 +1233,7 @@ static void file_base64_file_action( } } -void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) -{ +void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) { /* wont go to more than had (overly conservative) */ result=new(PointerFreeGC) char[in_size+1/*terminator*/]; int state=0; @@ -1042,35 +1249,33 @@ void pa_base64_decode(const char *in, si int file_block_read(const int f, unsigned char* buffer, const size_t size){ int nCount = read(f, buffer, size); if (nCount < 0) - throw Exception(0, + throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno); return nCount; } -const unsigned long pa_crc32(const char *in, size_t in_size) -{ +const unsigned long pa_crc32(const char *in, size_t in_size){ unsigned long crc32=0xFFFFFFFF; InitCrc32Table(); - for(size_t i = 0; i < in_size; i++) CalcCrc32(in[i], crc32); + for(size_t i = 0; i(context); if(finfo.st_size) { InitCrc32Table(); @@ -1083,3 +1288,13 @@ static void file_crc32_file_action( } } +char* print_pcre_exec_error_text(int exec_result){ + switch(exec_result){ + case PCRE_ERROR_BADUTF8: + return "validation of UTF-8 string failed while pcre_exec (%d)."; + break; + default: + return "regular expression execute error (%d)"; + } +} +