--- parser3/src/main/pa_common.C 2010/05/25 09:30:30 1.259 +++ parser3/src/main/pa_common.C 2015/04/21 22:12:27 1.283 @@ -1,7 +1,7 @@ /** @file Parser: commonly functions. - Copyright(c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com) + Copyright (c) 2000-2012 Art. Lebedev Studio (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) * BASE64 part @@ -26,8 +26,6 @@ * */ -static const char * const IDENT_COMMON_C="$Date: 2010/05/25 09:30:30 $"; - #include "pa_common.h" #include "pa_exception.h" #include "pa_hash.h" @@ -38,6 +36,22 @@ static const char * const IDENT_COMMON_C #include "pcre.h" #include "pa_request.h" +#include "pa_idna.h" +#include "pa_convert_utf.h" + +#ifdef _MSC_VER +#include +#include +#endif + +#ifdef _MSC_VER +#define pa_mkdir(path, mode) _mkdir(path) +#else +#define pa_mkdir(path, mode) mkdir(path, mode) +#endif + +volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.283 2015/04/21 22:12:27 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H; + // some maybe-undefined constants #ifndef _O_TEXT @@ -67,74 +81,13 @@ const String file_status_name(FILE_STATU // functions -bool capitalized(const char* s){ - bool upper=true; - for(const char* c=s; *c; c++){ - if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c))) - return false; - upper=strchr("-_ ", *c) != 0; - } - return true; -} - -const char* capitalize(const char* s){ - if(!s || capitalized(s)) - return s; - - char* result=pa_strdup(s); - if(result){ - bool upper=true; - for(char* c=result; *c; c++){ - *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c); - upper=strchr("-_ ", *c) != 0; - } - } - return (const char*)result; -} - -void fix_line_breaks(char *str, size_t& length) { - //_asm int 3; - const char* const eob=str+length; - char* dest=str; - // fix DOS: \r\n -> \n - // fix Macintosh: \r -> \n - char* bol=str; - while(char* eol=(char*)memchr(bol, '\r', eob -bol)) { - size_t len=eol-bol; - if(dest!=bol) - memcpy(dest, bol, len); - dest+=len; - *dest++='\n'; - - if(&eol[1](context); size_t to_read_size=info.count; if(!to_read_size) to_read_size=(size_t)finfo.st_size; assert( !(info.buf && as_text) ); - if(to_read_size) { + if(to_read_size) { if(info.offset) lseek(f, info.offset, SEEK_SET); - *info.data=info.buf - ? info.buf - : (char *)pa_malloc_atomic(to_read_size+1); - *info.data_size=(size_t)read(f, *info.data, to_read_size); - - if(ssize_t(*info.data_size)<0 || *info.data_size>to_read_size) - throw Exception("file.read", - &file_spec, - "read failed: actually read %u bytes count not in [0..%u] valid range", - *info.data_size, to_read_size); + *info.data=info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1); + ssize_t result=read(f, *info.data, to_read_size); + if(result<0) + throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno); + *info.data_size=result; } else { // empty file // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files *info.data=(char *)pa_malloc_atomic(1); @@ -200,9 +145,7 @@ File_read_result file_read(Request_chars if(params){ int valid_options=pa_get_valid_file_options_count(*params); if(valid_options!=params->count()) - throw Exception(PARSER_RUNTIME, - 0, - INVALID_OPTION_PASSED); + throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION); } File_read_action_info info={&result.str, &result.length, buf, offset, count}; @@ -213,23 +156,24 @@ File_read_result file_read(Request_chars if(as_text){ if(result.success){ + Charset* asked_charset=0; if(result.length>=3 && strncmp(result.str, "\xEF\xBB\xBF", 3)==0){ // skip UTF-8 signature (BOM code) result.str+=3; result.length-=3; + asked_charset=&UTF8_charset; } - if(result.length && transcode_text_result && params){ // must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below - if(Value* vcharset_name=params->get(PA_CHARSET_NAME)){ - Charset asked_charset=::charsets.get(vcharset_name->as_string(). - change_case(charsets.source(), String::CC_UPPER)); + if(params) + if(Value* vcharset_name=params->get(PA_CHARSET_NAME)) + asked_charset=&::charsets.get(vcharset_name->as_string().change_case(charsets.source(), String::CC_UPPER)); + + if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below + String::C body=String::C(result.str, result.length); + body=Charset::transcode(body, *asked_charset, charsets.source()); - String::C body=String::C(result.str, result.length); - body=Charset::transcode(body, asked_charset, charsets.source()); - - result.str=const_cast(body.str); // hacking a little - result.length=body.length; - } + result.str=const_cast(body.str); // hacking a little + result.length=body.length; } } if(result.length) @@ -345,7 +289,7 @@ void create_dir_for_file(const String& f size_t pos_after=1; size_t pos_before; while((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) { - mkdir(file_spec.mid(0, pos_before).taint_cstr(String::L_FILE_SPEC), 0775); + pa_mkdir(file_spec.mid(0, pos_before).taint_cstr(String::L_FILE_SPEC), 0775); pos_after=pos_before+1; } } @@ -415,16 +359,18 @@ struct File_write_action_info { size_t length; }; #endif + static void file_write_action(int f, void *context) { File_write_action_info& info=*static_cast(context); if(info.length) { - int written=write(f, info.str, info.length); + ssize_t written=write(f, info.str, info.length); if(written<0) - throw Exception("file.access", - 0, - "write failed: %s (%d)", strerror(errno), errno); + throw Exception("file.write", 0, "write failed: %s (%d)", strerror(errno), errno); + if((size_t)written!=info.length) + throw Exception("file.write", 0, "write failed: %u of %u bytes written", written, info.length); } } + void file_write( Request_charsets& charsets, const String& file_spec, @@ -452,18 +398,67 @@ void file_write( do_append); } +static size_t get_dir(char* fname, size_t helper_length){ + bool dir=false; + size_t pos=0; + for(pos=helper_length; pos; pos--){ + char c=fname[pos-1]; + if(c=='/' || c=='\\'){ + fname[pos-1]=0; + dir=true; + } else if(dir) break; + } + return pos; +} + +static bool entry_readable(char* fname, bool need_dir) { + if(need_dir){ + size_t size=strlen(fname); + while(size) { + char c=fname[size-1]; + if(c=='/' || c=='\\') + fname[--size]=0; + else + break; + } + } + + struct stat finfo; + if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { + bool is_dir=(finfo.st_mode&S_IFDIR) != 0; + return is_dir==need_dir; + } + return false; +} + +static bool entry_readable(const String& file_spec, bool need_dir) { + return entry_readable(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir); +} + // throws nothing! [this is required in file_move & file_delete] static void rmdir(const String& file_spec, size_t pos_after) { - size_t pos_before; - if((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) - rmdir(file_spec, pos_before+1); - - rmdir(file_spec.mid(0, pos_after-1/* / */).taint_cstr(String::L_FILE_SPEC)); + char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC); + size_t length=strlen(dir_spec); + while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){ +#ifdef _MSC_VER + if(!entry_readable(dir_spec, true)) + break; + DWORD attrs=GetFileAttributes(dir_spec); + if( + (attrs==INVALID_FILE_ATTRIBUTES) + || !(attrs & FILE_ATTRIBUTE_DIRECTORY) + || (attrs & FILE_ATTRIBUTE_REPARSE_POINT) + ) + break; +#endif + if( rmdir(dir_spec) ) + break; + }; } -bool file_delete(const String& file_spec, bool fail_on_problem) { +bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) { const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); - if(unlink(fname)!=0) + if(unlink(fname)!=0) { if(fail_on_problem) throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, &file_spec, @@ -471,12 +466,15 @@ bool file_delete(const String& file_spec strerror(errno), errno, fname); else return false; + } + + if(!keep_empty_dirs) + rmdir(file_spec, 1); - rmdir(file_spec, 1); return true; } -void file_move(const String& old_spec, const String& new_spec) { +void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) { const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC); const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC); @@ -488,7 +486,8 @@ void file_move(const String& old_spec, c "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr); - rmdir(old_spec, 1); + if(!keep_empty_dirs) + rmdir(old_spec, 1); } @@ -505,26 +504,6 @@ bool entry_exists(const String& file_spe return entry_exists(fname, 0); } -static bool entry_readable(const String& file_spec, bool need_dir) { - char* fname=file_spec.taint_cstrm(String::L_FILE_SPEC); - if(need_dir) { - size_t size=strlen(fname); - while(size) { - char c=fname[size-1]; - if(c=='/' || c=='\\') - fname[--size]=0; - else - break; - } - } - struct stat finfo; - if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { - bool is_dir=(finfo.st_mode&S_IFDIR) != 0; - return is_dir==need_dir; - } - return false; -} - bool file_exist(const String& file_spec) { return entry_readable(file_spec, false); } @@ -535,7 +514,8 @@ bool dir_exists(const String& file_spec) const String* file_exist(const String& path, const String& name) { String& result=*new String(path); - result << "/"; + if(path.last_char() != '/') + result << "/"; result << name; return file_exist(result)?&result:0; } @@ -552,7 +532,7 @@ bool file_stat(const String& file_spec, bool fail_on_read_problem) { const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); struct stat finfo; - if(stat(fname, &finfo)!=0) + if(stat(fname, &finfo)!=0) { if(fail_on_read_problem) throw Exception("file.missing", &file_spec, @@ -560,6 +540,7 @@ bool file_stat(const String& file_spec, strerror(errno), errno, fname); else return false; + } rsize=finfo.st_size; ratime=finfo.st_atime; rmtime=finfo.st_mtime; @@ -567,6 +548,68 @@ bool file_stat(const String& file_spec, return true; } +/** + String related functions +*/ + +bool capitalized(const char* s){ + bool upper=true; + for(const char* c=s; *c; c++){ + if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c))) + return false; + upper=strchr("-_ ", *c) != 0; + } + return true; +} + +const char* capitalize(const char* s){ + if(!s || capitalized(s)) + return s; + + char* result=pa_strdup(s); + if(result){ + bool upper=true; + for(char* c=result; *c; c++){ + *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c); + upper=strchr("-_ ", *c) != 0; + } + } + return (const char*)result; +} + +void fix_line_breaks(char *str, size_t& length) { + //_asm int 3; + const char* const eob=str+length; + char* dest=str; + // fix DOS: \r\n -> \n + // fix Macintosh: \r -> \n + char* bol=str; + while(char* eol=(char*)memchr(bol, '\r', eob -bol)) { + size_t len=eol-bol; + if(dest!=bol) + memmove(dest, bol, len); + dest+=len; + *dest++='\n'; + + if(&eol[1]hh*/+1/*for zero-teminator*/]; + unsigned char *src=bytes; + unsigned char *end=bytes+size; + char *dest=bytes_hex; + + const char *hex=upcase? hex_digits : "0123456789abcdef"; + + for(; src> 16); - *outptr++ = (unsigned char)(saved >> 8); - *outptr++ = (unsigned char)(saved); - i = 0; - } + switch(c) { + case 0xff: // non-base64 and non-whitespace chars. not allowed in strict mode + if(strict) + throw Exception(BASE64_FORMAT, 0, "Invalid base64 char on position %d is detected", inptr-in-1); + case 0xfe: // whitespace chars 0x09, 0x0A, 0x0D, 0x20 are allowed in any mode + break; + default: + saved = (saved << 6) | c; + i++; + if (i == 4) { + *outptr++ = (unsigned char)(saved >> 16); + *outptr++ = (unsigned char)(saved >> 8); + *outptr++ = (unsigned char)(saved); + i = 0; + } } } @@ -1209,7 +1214,7 @@ g_mime_utils_base64_decode_step (const u i = 2; while (inptr > in && i) { inptr--; - if (gmime_base64_rank[*inptr] != 0xff) { + if (gmime_base64_rank[*inptr] <= 0xfe) { if (*inptr == '=' && outptr > out) outptr--; i--; @@ -1238,17 +1243,9 @@ char* pa_base64_encode(const char *in, s return result; } - -char* pa_base64_encode(const String& file_spec){ - unsigned char* base64=0; - File_base64_action_info info={&base64}; - - file_read_action_under_lock(file_spec, - "pa_base64_encode", file_base64_file_action, &info); - - return (char*)base64; -} - +struct File_base64_action_info { + unsigned char** base64; +}; static void file_base64_file_action( struct stat& finfo, @@ -1275,16 +1272,32 @@ static void file_base64_file_action( } } -void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) { - /* wont go to more than had (overly conservative) */ - result=new(PointerFreeGC) char[in_size+1/*terminator*/]; +char* pa_base64_encode(const String& file_spec){ + unsigned char* base64=0; + File_base64_action_info info={&base64}; + + file_read_action_under_lock(file_spec, + "pa_base64_encode", file_base64_file_action, &info); + + return (char*)base64; +} + +void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size, bool strict) { + // every 4 base64 bytes are converted into 3 normal bytes + // not full set (tail) of 4-bytes set is ignored + size_t new_size=in_size/4*3; + result=new(PointerFreeGC) char[new_size+1/*terminator*/]; + int state=0; int save=0; result_size= - g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, - (unsigned char*)result, &state, &save); - assert(result_size <= in_size); + g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, + (unsigned char*)result, &state, &save, strict); + assert(result_size <= new_size); result[result_size]=0; // for text files + + if(strict && state!=0) + throw Exception(BASE64_FORMAT, 0, "Unexpected end of chars"); } @@ -1297,6 +1310,35 @@ int file_block_read(const int f, unsigne return nCount; } +static unsigned long crc32Table[256]; +static void InitCrc32Table() +{ + if(crc32Table[1] == 0){ + // This is the official polynomial used by CRC32 in PKZip. + // Often times the polynomial shown reversed as 0x04C11DB7. + static const unsigned long dwPolynomial = 0xEDB88320; + + for(int i = 0; i < 256; i++) + { + unsigned long dwCrc = i; + for(int j = 8; j > 0; j--) + { + if(dwCrc & 1) + dwCrc = (dwCrc >> 1) ^ dwPolynomial; + else + dwCrc >>= 1; + } + crc32Table[i] = dwCrc; + } + } +} + +inline void CalcCrc32(const unsigned char byte, unsigned long &crc32) +{ + crc32 = ((crc32) >> 8) ^ crc32Table[(byte) ^ ((crc32) & 0x000000FF)]; +} + + const unsigned long pa_crc32(const char *in, size_t in_size){ unsigned long crc32=0xFFFFFFFF; @@ -1307,12 +1349,6 @@ const unsigned long pa_crc32(const char return ~crc32; } -const unsigned long pa_crc32(const String& file_spec){ - unsigned long crc32=0xFFFFFFFF; - file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32); - return ~crc32; -} - static void file_crc32_file_action( struct stat& finfo, int f, @@ -1330,3 +1366,145 @@ static void file_crc32_file_action( } } +const unsigned long pa_crc32(const String& file_spec){ + unsigned long crc32=0xFFFFFFFF; + file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32); + return ~crc32; +} + +// content-type: xxx; charset=WE-NEED-THIS +// content-type: xxx; charset="WE-NEED-THIS" +// content-type: xxx; charset="WE-NEED-THIS"; +Charset* detect_charset(const char* content_type){ + if(content_type){ + char* CONTENT_TYPE=pa_strdup(content_type); + + for(char *p=CONTENT_TYPE; *p; p++) + *p=(char)toupper((unsigned char)*p); + + if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){ + begin+=8; // skip "CHARSET=" + char* end=0; + if(*begin && (*begin=='"' || *begin =='\'')){ + char quote=*begin; + begin++; + end=(char*)strchr(begin, quote); + } + if(!end) + end=(char*)strchr(begin, ';'); + + if(end) + *end=0; // terminator + + return *begin?&charsets.get(begin):0; + } + } + return 0; +} + + +static bool is_latin(const char *in){ + for(; *in; in++){ + if ((unsigned char)(*in) > 0x7F) + return false; + } + return true; +} + +#define MAX_IDNA_LENGTH 256 + +const char *pa_idna_encode(const char *in, Charset &source_charset){ + if(!in || is_latin(in)) + return in; + + uint32_t utf32[MAX_IDNA_LENGTH]; + uint32_t *utf32_end=utf32; + + String::C sIn(in,strlen(in)); + + if(!source_charset.isUTF8()) + sIn=Charset::transcode(sIn, source_charset, UTF8_charset); + + int status=pa_convertUTF8toUTF32((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf32_end, utf32+MAX_IDNA_LENGTH-1, strictConversion); + if(status != conversionOK) + throw Exception("idna encode", new String(in), "utf conversion failed (%d)", status); + + *utf32_end=0; + + char *result = (char *)pa_malloc(MAX_IDNA_LENGTH); + status=pa_idna_to_ascii_4z(utf32, result, MAX_IDNA_LENGTH, 0); + if(status != IDNA_SUCCESS) + throw Exception("idna encode", new String(in), "encode failed: %s", pa_idna_strerror(status)); + + return result; +} + +const char *pa_idna_decode(const char *in, Charset &asked_charset){ + if(!in || !(*in)) + return in; + + uint32_t utf32[MAX_IDNA_LENGTH]; + const uint32_t *utf32_start=utf32; + uint32_t *utf32_end; + + int status=pa_idna_to_unicode_4z(in, utf32, MAX_IDNA_LENGTH, 0); + if(status != IDNA_SUCCESS) + throw Exception("idna decode", new String(in), "decode failed: %s", pa_idna_strerror(status)); + + for(utf32_end=utf32; *utf32_end; utf32_end++); + + char *result = (char *)pa_malloc(MAX_IDNA_LENGTH); + char *result_end = result; + + status=pa_convertUTF32toUTF8(&utf32_start, utf32_end, (UTF8**)&result_end, (UTF8*)(result+MAX_IDNA_LENGTH-1), strictConversion); + + if(status != conversionOK) + throw Exception("idna decode", new String(in), "utf conversion failed (%d)", status); + + *result_end='\0'; + + if(!asked_charset.isUTF8()) + result = (char *)Charset::transcode(result, UTF8_charset, asked_charset).cstr(); + + return result; +} + + +static bool isLeap(int year) { + return !( + (year % 4) || ((year % 400) && !(year % 100)) + ); +} + +int getMonthDays(int year, int month) { + static int monthDays[]={ + 31, + 28, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31 + }; + return (month == 1 /* january -- 0 */ && isLeap(year)) ? 29 : monthDays[month]; +} + +String::C date_gmt_string(tm* tms) { + /// http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 + static const char month_names[12][4]={ + "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}; + static const char days[7][4]={ + "Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; + + char *buf=new(PointerFreeGC) char[MAX_STRING]; + return String::C(buf, + snprintf(buf, MAX_STRING, "%s, %.2d %s %.4d %.2d:%.2d:%.2d GMT", + days[tms->tm_wday], + tms->tm_mday,month_names[tms->tm_mon],tms->tm_year+1900, + tms->tm_hour,tms->tm_min,tms->tm_sec)); +}