--- parser3/src/main/pa_common.C 2009/09/28 11:39:30 1.252 +++ parser3/src/main/pa_common.C 2015/04/02 22:04:41 1.281 @@ -1,7 +1,7 @@ /** @file Parser: commonly functions. - Copyright(c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com) + Copyright (c) 2000-2012 Art. Lebedev Studio (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) * BASE64 part @@ -26,8 +26,6 @@ * */ -static const char * const IDENT_COMMON_C="$Date: 2009/09/28 11:39:30 $"; - #include "pa_common.h" #include "pa_exception.h" #include "pa_hash.h" @@ -38,6 +36,19 @@ static const char * const IDENT_COMMON_C #include "pcre.h" #include "pa_request.h" +#ifdef _MSC_VER +#include +#include +#endif + +#ifdef _MSC_VER +#define pa_mkdir(path, mode) _mkdir(path) +#else +#define pa_mkdir(path, mode) mkdir(path, mode) +#endif + +volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.281 2015/04/02 22:04:41 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H; + // some maybe-undefined constants #ifndef _O_TEXT @@ -67,61 +78,13 @@ const String file_status_name(FILE_STATU // functions -const char* capitalize(const char* s){ - char* result=pa_strdup(s); - if(result){ - bool upper=true; - for(char* c=result; *c; c++){ - *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c); - upper=strchr("-_ ", *c) != 0; - } - } - return (const char*)result; -} - -void fix_line_breaks(char *str, size_t& length) { - //_asm int 3; - const char* const eob=str+length; - char* dest=str; - // fix DOS: \r\n -> \n - // fix Macintosh: \r -> \n - char* bol=str; - while(char* eol=(char*)memchr(bol, '\r', eob -bol)) { - size_t len=eol-bol; - if(dest!=bol) - memcpy(dest, bol, len); - dest+=len; - *dest++='\n'; - - if(&eol[1](context); size_t to_read_size=info.count; if(!to_read_size) to_read_size=(size_t)finfo.st_size; assert( !(info.buf && as_text) ); - if(to_read_size) { + if(to_read_size) { if(info.offset) lseek(f, info.offset, SEEK_SET); - *info.data=info.buf - ? info.buf - : new(PointerFreeGC) char[to_read_size+(as_text?1:0)]; - *info.data_size=(size_t)read(f, *info.data, to_read_size); - - if(ssize_t(*info.data_size)<0 || *info.data_size>to_read_size) - throw Exception("file.read", - &file_spec, - "read failed: actually read %u bytes count not in [0..%u] valid range", - *info.data_size, to_read_size); + *info.data=info.buf ? info.buf : (char *)pa_malloc_atomic(to_read_size+1); + ssize_t result=read(f, *info.data, to_read_size); + if(result<0) + throw Exception("file.read", &file_spec, "read failed: %s (%d)", strerror(errno), errno); + *info.data_size=result; } else { // empty file // for both, text and binary: for text we need that terminator, for binary we need nonzero pointer to be able to save such files - *info.data=new(PointerFreeGC) char[1]; + *info.data=(char *)pa_malloc_atomic(1); *(char*)(*info.data)=0; *info.data_size=0; return; @@ -187,9 +142,7 @@ File_read_result file_read(Request_chars if(params){ int valid_options=pa_get_valid_file_options_count(*params); if(valid_options!=params->count()) - throw Exception(PARSER_RUNTIME, - 0, - INVALID_OPTION_PASSED); + throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION); } File_read_action_info info={&result.str, &result.length, buf, offset, count}; @@ -200,23 +153,24 @@ File_read_result file_read(Request_chars if(as_text){ if(result.success){ + Charset* asked_charset=0; if(result.length>=3 && strncmp(result.str, "\xEF\xBB\xBF", 3)==0){ // skip UTF-8 signature (BOM code) result.str+=3; result.length-=3; + asked_charset=&UTF8_charset; } - if(result.length && transcode_text_result && params){ // must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below - if(Value* vcharset_name=params->get(PA_CHARSET_NAME)){ - Charset asked_charset=::charsets.get(vcharset_name->as_string(). - change_case(charsets.source(), String::CC_UPPER)); - - String::C body=String::C(result.str, result.length); - body=Charset::transcode(body, asked_charset, charsets.source()); + if(params) + if(Value* vcharset_name=params->get(PA_CHARSET_NAME)) + asked_charset=&::charsets.get(vcharset_name->as_string().change_case(charsets.source(), String::CC_UPPER)); + + if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below + String::C body=String::C(result.str, result.length); + body=Charset::transcode(body, *asked_charset, charsets.source()); - result.str=const_cast(body.str); // hacking a little - result.length=body.length; - } + result.str=const_cast(body.str); // hacking a little + result.length=body.length; } } if(result.length) @@ -251,8 +205,9 @@ File_read_result file_load(Request& r, c return result; } + #ifdef PA_SAFE_MODE -void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) { +void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) { if(finfo.st_uid/*foreign?*/!=geteuid() && finfo.st_gid/*foreign?*/!=getegid()) throw Exception(PARSER_RUNTIME, @@ -264,9 +219,14 @@ void check_safe_mode(struct stat finfo, "fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)", fname, finfo.st_uid, geteuid(), - finfo.st_gid, getegid()); -} -#endif + finfo.st_gid, getegid()); +} +#else +void check_safe_mode(struct stat, const String&, const char*) { +} +#endif + + bool file_read_action_under_lock(const String& file_spec, const char* action_name, File_read_action action, void *context, @@ -293,15 +253,13 @@ bool file_read_action_under_lock(const S strerror(errno), errno, fname); struct stat finfo; - if(stat(fname, &finfo)!=0) + if(fstat(f, &finfo)!=0) throw Exception("file.missing", // hardly possible: we just opened it OK &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); -#ifdef PA_SAFE_MODE check_safe_mode(finfo, file_spec, fname); -#endif action(finfo, f, file_spec, fname, as_text, context); } catch(...) { @@ -315,7 +273,8 @@ bool file_read_action_under_lock(const S return true; } else { if(fail_on_read_problem) - throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, + throw Exception(errno==EACCES?"file.access" + :(errno==ENOENT || errno==ENOTDIR || errno==ENODEV)?"file.missing":0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname); @@ -327,7 +286,7 @@ void create_dir_for_file(const String& f size_t pos_after=1; size_t pos_before; while((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) { - mkdir(file_spec.mid(0, pos_before).taint_cstr(String::L_FILE_SPEC), 0775); + pa_mkdir(file_spec.mid(0, pos_before).taint_cstr(String::L_FILE_SPEC), 0775); pos_after=pos_before+1; } } @@ -362,7 +321,12 @@ bool file_write_action_under_lock( } try { - action(f, context); +#if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE)) + struct stat finfo; + if(fstat(f, &finfo)==0 && finfo.st_mode & 0111) + fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any +#endif + action(f, context); } catch(...) { #ifdef HAVE_FTRUNCATE if(!do_append) @@ -392,16 +356,18 @@ struct File_write_action_info { size_t length; }; #endif + static void file_write_action(int f, void *context) { File_write_action_info& info=*static_cast(context); if(info.length) { - int written=write(f, info.str, info.length); + ssize_t written=write(f, info.str, info.length); if(written<0) - throw Exception("file.access", - 0, - "write failed: %s (%d)", strerror(errno), errno); + throw Exception("file.write", 0, "write failed: %s (%d)", strerror(errno), errno); + if((size_t)written!=info.length) + throw Exception("file.write", 0, "write failed: %u of %u bytes written", written, info.length); } } + void file_write( Request_charsets& charsets, const String& file_spec, @@ -429,16 +395,65 @@ void file_write( do_append); } +static size_t get_dir(char* fname, size_t helper_length){ + bool dir=false; + size_t pos=0; + for(pos=helper_length; pos; pos--){ + char c=fname[pos-1]; + if(c=='/' || c=='\\'){ + fname[pos-1]=0; + dir=true; + } else if(dir) break; + } + return pos; +} + +static bool entry_readable(char* fname, bool need_dir) { + if(need_dir){ + size_t size=strlen(fname); + while(size) { + char c=fname[size-1]; + if(c=='/' || c=='\\') + fname[--size]=0; + else + break; + } + } + + struct stat finfo; + if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { + bool is_dir=(finfo.st_mode&S_IFDIR) != 0; + return is_dir==need_dir; + } + return false; +} + +static bool entry_readable(const String& file_spec, bool need_dir) { + return entry_readable(file_spec.taint_cstrm(String::L_FILE_SPEC), need_dir); +} + // throws nothing! [this is required in file_move & file_delete] static void rmdir(const String& file_spec, size_t pos_after) { - size_t pos_before; - if((pos_before=file_spec.pos('/', pos_after))!=STRING_NOT_FOUND) - rmdir(file_spec, pos_before+1); - - rmdir(file_spec.mid(0, pos_after-1/* / */).taint_cstr(String::L_FILE_SPEC)); + char* dir_spec=file_spec.taint_cstrm(String::L_FILE_SPEC); + size_t length=strlen(dir_spec); + while( (length=get_dir(dir_spec, length)) && (length > pos_after) ){ +#ifdef _MSC_VER + if(!entry_readable(dir_spec, true)) + break; + DWORD attrs=GetFileAttributes(dir_spec); + if( + (attrs==INVALID_FILE_ATTRIBUTES) + || !(attrs & FILE_ATTRIBUTE_DIRECTORY) + || (attrs & FILE_ATTRIBUTE_REPARSE_POINT) + ) + break; +#endif + if( rmdir(dir_spec) ) + break; + }; } -bool file_delete(const String& file_spec, bool fail_on_problem) { +bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) { const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); if(unlink(fname)!=0) if(fail_on_problem) @@ -449,11 +464,13 @@ bool file_delete(const String& file_spec else return false; - rmdir(file_spec, 1); + if(!keep_empty_dirs) + rmdir(file_spec, 1); + return true; } -void file_move(const String& old_spec, const String& new_spec) { +void file_move(const String& old_spec, const String& new_spec, bool keep_empty_dirs) { const char* old_spec_cstr=old_spec.taint_cstr(String::L_FILE_SPEC); const char* new_spec_cstr=new_spec.taint_cstr(String::L_FILE_SPEC); @@ -465,7 +482,8 @@ void file_move(const String& old_spec, c "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr); - rmdir(old_spec, 1); + if(!keep_empty_dirs) + rmdir(old_spec, 1); } @@ -482,26 +500,6 @@ bool entry_exists(const String& file_spe return entry_exists(fname, 0); } -static bool entry_readable(const String& file_spec, bool need_dir) { - char* fname=file_spec.taint_cstrm(String::L_FILE_SPEC); - if(need_dir) { - size_t size=strlen(fname); - while(size) { - char c=fname[size-1]; - if(c=='/' || c=='\\') - fname[--size]=0; - else - break; - } - } - struct stat finfo; - if(access(fname, R_OK)==0 && entry_exists(fname, &finfo)) { - bool is_dir=(finfo.st_mode&S_IFDIR) != 0; - return is_dir==need_dir; - } - return false; -} - bool file_exist(const String& file_spec) { return entry_readable(file_spec, false); } @@ -512,7 +510,8 @@ bool dir_exists(const String& file_spec) const String* file_exist(const String& path, const String& name) { String& result=*new String(path); - result << "/"; + if(path.last_char() != '/') + result << "/"; result << name; return file_exist(result)?&result:0; } @@ -544,6 +543,68 @@ bool file_stat(const String& file_spec, return true; } +/** + String related functions +*/ + +bool capitalized(const char* s){ + bool upper=true; + for(const char* c=s; *c; c++){ + if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c))) + return false; + upper=strchr("-_ ", *c) != 0; + } + return true; +} + +const char* capitalize(const char* s){ + if(!s || capitalized(s)) + return s; + + char* result=pa_strdup(s); + if(result){ + bool upper=true; + for(char* c=result; *c; c++){ + *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c); + upper=strchr("-_ ", *c) != 0; + } + } + return (const char*)result; +} + +void fix_line_breaks(char *str, size_t& length) { + //_asm int 3; + const char* const eob=str+length; + char* dest=str; + // fix DOS: \r\n -> \n + // fix Macintosh: \r -> \n + char* bol=str; + while(char* eol=(char*)memchr(bol, '\r', eob -bol)) { + size_t len=eol-bol; + if(dest!=bol) + memmove(dest, bol, len); + dest+=len; + *dest++='\n'; + + if(&eol[1]hh*/+1/*for zero-teminator*/]; + unsigned char *src=bytes; + unsigned char *end=bytes+size; + char *dest=bytes_hex; + + const char *hex=upcase? hex_digits : "0123456789abcdef"; + + for(; src> 16); - *outptr++ = (unsigned char)(saved >> 8); - *outptr++ = (unsigned char)(saved); - i = 0; - } + switch(c) { + case 0xff: // non-base64 and non-whitespace chars. not allowed in strict mode + if(strict) + throw Exception(BASE64_FORMAT, 0, "Invalid base64 char on position %d is detected", inptr-in-1); + case 0xfe: // whitespace chars 0x09, 0x0A, 0x0D, 0x20 are allowed in any mode + break; + default: + saved = (saved << 6) | c; + i++; + if (i == 4) { + *outptr++ = (unsigned char)(saved >> 16); + *outptr++ = (unsigned char)(saved >> 8); + *outptr++ = (unsigned char)(saved); + i = 0; + } } } @@ -1186,7 +1209,7 @@ g_mime_utils_base64_decode_step (const u i = 2; while (inptr > in && i) { inptr--; - if (gmime_base64_rank[*inptr] != 0xff) { + if (gmime_base64_rank[*inptr] <= 0xfe) { if (*inptr == '=' && outptr > out) outptr--; i--; @@ -1215,17 +1238,9 @@ char* pa_base64_encode(const char *in, s return result; } - -char* pa_base64_encode(const String& file_spec){ - unsigned char* base64=0; - File_base64_action_info info={&base64}; - - file_read_action_under_lock(file_spec, - "pa_base64_encode", file_base64_file_action, &info); - - return (char*)base64; -} - +struct File_base64_action_info { + unsigned char** base64; +}; static void file_base64_file_action( struct stat& finfo, @@ -1252,16 +1267,32 @@ static void file_base64_file_action( } } -void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size) { - /* wont go to more than had (overly conservative) */ - result=new(PointerFreeGC) char[in_size+1/*terminator*/]; +char* pa_base64_encode(const String& file_spec){ + unsigned char* base64=0; + File_base64_action_info info={&base64}; + + file_read_action_under_lock(file_spec, + "pa_base64_encode", file_base64_file_action, &info); + + return (char*)base64; +} + +void pa_base64_decode(const char *in, size_t in_size, char*& result, size_t& result_size, bool strict) { + // every 4 base64 bytes are converted into 3 normal bytes + // not full set (tail) of 4-bytes set is ignored + size_t new_size=in_size/4*3; + result=new(PointerFreeGC) char[new_size+1/*terminator*/]; + int state=0; int save=0; result_size= - g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, - (unsigned char*)result, &state, &save); - assert(result_size <= in_size); + g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, + (unsigned char*)result, &state, &save, strict); + assert(result_size <= new_size); result[result_size]=0; // for text files + + if(strict && state!=0) + throw Exception(BASE64_FORMAT, 0, "Unexpected end of chars"); } @@ -1274,6 +1305,35 @@ int file_block_read(const int f, unsigne return nCount; } +static unsigned long crc32Table[256]; +static void InitCrc32Table() +{ + if(crc32Table[1] == 0){ + // This is the official polynomial used by CRC32 in PKZip. + // Often times the polynomial shown reversed as 0x04C11DB7. + static const unsigned long dwPolynomial = 0xEDB88320; + + for(int i = 0; i < 256; i++) + { + unsigned long dwCrc = i; + for(int j = 8; j > 0; j--) + { + if(dwCrc & 1) + dwCrc = (dwCrc >> 1) ^ dwPolynomial; + else + dwCrc >>= 1; + } + crc32Table[i] = dwCrc; + } + } +} + +inline void CalcCrc32(const unsigned char byte, unsigned long &crc32) +{ + crc32 = ((crc32) >> 8) ^ crc32Table[(byte) ^ ((crc32) & 0x000000FF)]; +} + + const unsigned long pa_crc32(const char *in, size_t in_size){ unsigned long crc32=0xFFFFFFFF; @@ -1284,12 +1344,6 @@ const unsigned long pa_crc32(const char return ~crc32; } -const unsigned long pa_crc32(const String& file_spec){ - unsigned long crc32=0xFFFFFFFF; - file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32); - return ~crc32; -} - static void file_crc32_file_action( struct stat& finfo, int f, @@ -1307,3 +1361,78 @@ static void file_crc32_file_action( } } +const unsigned long pa_crc32(const String& file_spec){ + unsigned long crc32=0xFFFFFFFF; + file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32); + return ~crc32; +} + +// content-type: xxx; charset=WE-NEED-THIS +// content-type: xxx; charset="WE-NEED-THIS" +// content-type: xxx; charset="WE-NEED-THIS"; +Charset* detect_charset(const char* content_type){ + if(content_type){ + char* CONTENT_TYPE=pa_strdup(content_type); + + for(char *p=CONTENT_TYPE; *p; p++) + *p=(char)toupper((unsigned char)*p); + + if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){ + begin+=8; // skip "CHARSET=" + char* end=0; + if(*begin && (*begin=='"' || *begin =='\'')){ + char quote=*begin; + begin++; + end=(char*)strchr(begin, quote); + } + if(!end) + end=(char*)strchr(begin, ';'); + + if(end) + *end=0; // terminator + + return *begin?&charsets.get(begin):0; + } + } + return 0; +} + + +static bool isLeap(int year) { + return !( + (year % 4) || ((year % 400) && !(year % 100)) + ); +} + +int getMonthDays(int year, int month) { + static int monthDays[]={ + 31, + 28, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31 + }; + return (month == 1 /* january -- 0 */ && isLeap(year)) ? 29 : monthDays[month]; +} + +String::C date_gmt_string(tm* tms) { + /// http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 + static const char month_names[12][4]={ + "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}; + static const char days[7][4]={ + "Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; + + char *buf=new(PointerFreeGC) char[MAX_STRING]; + return String::C(buf, + snprintf(buf, MAX_STRING, "%s, %.2d %s %.4d %.2d:%.2d:%.2d GMT", + days[tms->tm_wday], + tms->tm_mday,month_names[tms->tm_mon],tms->tm_year+1900, + tms->tm_hour,tms->tm_min,tms->tm_sec)); +}