--- parser3/src/main/pa_common.C 2013/10/14 19:45:13 1.278 +++ parser3/src/main/pa_common.C 2017/01/23 09:33:02 1.300 @@ -1,7 +1,7 @@ /** @file Parser: commonly functions. - Copyright (c) 2000-2012 Art. Lebedev Studio (http://www.artlebedev.com) + Copyright (c) 2000-2015 Art. Lebedev Studio (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) * BASE64 part @@ -36,6 +36,9 @@ #include "pcre.h" #include "pa_request.h" +#include "pa_idna.h" +#include "pa_convert_utf.h" + #ifdef _MSC_VER #include #include @@ -47,7 +50,7 @@ #define pa_mkdir(path, mode) mkdir(path, mode) #endif -volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.278 2013/10/14 19:45:13 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H; +volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.300 2017/01/23 09:33:02 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H; // some maybe-undefined constants @@ -111,12 +114,11 @@ struct File_read_action_info { }; #endif -static void file_read_action(struct stat& finfo, int f, const String& file_spec, const char* /*fname*/, bool as_text, void *context) { +static void file_read_action(struct stat& finfo, int f, const String& file_spec, void *context) { File_read_action_info& info=*static_cast(context); size_t to_read_size=info.count; if(!to_read_size) - to_read_size=(size_t)finfo.st_size; - assert( !(info.buf && as_text) ); + to_read_size=check_file_size(finfo.st_size, file_spec); if(to_read_size) { if(info.offset) lseek(f, info.offset, SEEK_SET); @@ -147,23 +149,16 @@ File_read_result file_read(Request_chars File_read_action_info info={&result.str, &result.length, buf, offset, count}; - result.success=file_read_action_under_lock(file_spec, - "read", file_read_action, &info, - as_text, fail_on_read_problem); + result.success=file_read_action_under_lock(file_spec, "read", file_read_action, &info, as_text, fail_on_read_problem); if(as_text){ if(result.success){ Charset* asked_charset=0; - if(result.length>=3 && strncmp(result.str, "\xEF\xBB\xBF", 3)==0){ - // skip UTF-8 signature (BOM code) - result.str+=3; - result.length-=3; - asked_charset=&UTF8_charset; - } - if(params) if(Value* vcharset_name=params->get(PA_CHARSET_NAME)) - asked_charset=&::charsets.get(vcharset_name->as_string().change_case(charsets.source(), String::CC_UPPER)); + asked_charset=&pa_charsets.get(vcharset_name->as_string()); + + asked_charset=pa_charsets.checkBOM(result.str, result.length, asked_charset); if(result.length && transcode_text_result && asked_charset){ // length must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below String::C body=String::C(result.str, result.length); @@ -188,9 +183,7 @@ File_read_result file_load(Request& r, c File_read_result result={false, 0, 0, 0}; if(file_spec.starts_with("http://")) { if(offset || count) - throw Exception(PARSER_RUNTIME, - 0, - "offset and load options are not supported for HTTP:// file load"); + throw Exception(PARSER_RUNTIME, 0, "offset and load options are not supported for HTTP:// file load"); // fail on read problem File_read_http_result http=pa_internal_file_read_http(r, file_spec, as_text, params, transcode_text_result); @@ -210,16 +203,13 @@ File_read_result file_load(Request& r, c void check_safe_mode(struct stat finfo, const String& file_spec, const char* fname) { if(finfo.st_uid/*foreign?*/!=geteuid() && finfo.st_gid/*foreign?*/!=getegid()) - throw Exception(PARSER_RUNTIME, - &file_spec, - "parser is in safe mode: " - "reading files of foreign group and user disabled " - "[recompile parser with --disable-safe-mode configure option], " - "actual filename '%s', " - "fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)", - fname, - finfo.st_uid, geteuid(), - finfo.st_gid, getegid()); + throw Exception(PARSER_RUNTIME, + &file_spec, + "parser is in safe mode: reading files of foreign group and user disabled " + "[recompile parser with --disable-safe-mode configure option], " + "actual filename '%s', fuid(%d)!=euid(%d) or fgid(%d)!=egid(%d)", + fname, finfo.st_uid, geteuid(), finfo.st_gid, getegid() + ); } #else void check_safe_mode(struct stat, const String&, const char*) { @@ -247,37 +237,29 @@ bool file_read_action_under_lock(const S if((f=open(fname, O_RDONLY|(as_text?_O_TEXT:_O_BINARY)))>=0) { try { if(pa_lock_shared_blocking(f)!=0) - throw Exception("file.lock", - &file_spec, - "shared lock failed: %s (%d), actual filename '%s'", - strerror(errno), errno, fname); + throw Exception("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); struct stat finfo; - if(fstat(f, &finfo)!=0) + if(pa_fstat(f, &finfo)!=0) throw Exception("file.missing", // hardly possible: we just opened it OK - &file_spec, - "stat failed: %s (%d), actual filename '%s'", - strerror(errno), errno, fname); + &file_spec, "stat failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); check_safe_mode(finfo, file_spec, fname); - action(finfo, f, file_spec, fname, as_text, context); + action(finfo, f, file_spec, context); } catch(...) { pa_unlock(f);close(f); if(fail_on_read_problem) rethrow; - return false; + return false; } pa_unlock(f);close(f); return true; } else { if(fail_on_read_problem) - throw Exception(errno==EACCES?"file.access" - :(errno==ENOENT || errno==ENOTDIR || errno==ENODEV)?"file.missing":0, - &file_spec, - "%s failed: %s (%d), actual filename '%s'", - action_name, strerror(errno), errno, fname); + throw Exception(errno==EACCES ? "file.access" : (errno==ENOENT || errno==ENOTDIR || errno==ENODEV) ? "file.missing" : 0, + &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname); return false; } } @@ -310,10 +292,7 @@ bool file_write_action_under_lock( |(as_text?_O_TEXT:_O_BINARY) |(do_append?O_APPEND:PA_O_TRUNC), 0664))>=0) { if((do_block?pa_lock_exclusive_blocking(f):pa_lock_exclusive_nonblocking(f))!=0) { - Exception e("file.lock", - &file_spec, - "shared lock failed: %s (%d), actual filename '%s'", - strerror(errno), errno, fname); + Exception e("file.lock", &file_spec, "shared lock failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); close(f); if(fail_on_lock_problem) throw e; @@ -323,7 +302,7 @@ bool file_write_action_under_lock( try { #if (defined(HAVE_FCHMOD) && defined(PA_SAFE_MODE)) struct stat finfo; - if(fstat(f, &finfo)==0 && finfo.st_mode & 0111) + if(pa_fstat(f, &finfo)==0 && finfo.st_mode & 0111) fchmod(f, finfo.st_mode & 0666/*clear executable bits*/); // backward: ignore errors if any #endif action(f, context); @@ -343,10 +322,7 @@ bool file_write_action_under_lock( pa_unlock(f);close(f); return true; } else - throw Exception(errno==EACCES?"file.access":0, - &file_spec, - "%s failed: %s (%d), actual filename '%s'", - action_name, strerror(errno), errno, fname); + throw Exception(errno==EACCES ? "file.access" : 0, &file_spec, "%s failed: %s (%d), actual filename '%s'", action_name, strerror(errno), errno, fname); // here should be nothing, see rethrow above } @@ -455,14 +431,13 @@ static void rmdir(const String& file_spe bool file_delete(const String& file_spec, bool fail_on_problem, bool keep_empty_dirs) { const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); - if(unlink(fname)!=0) + if(unlink(fname)!=0) { if(fail_on_problem) - throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, - &file_spec, - "unlink failed: %s (%d), actual filename '%s'", - strerror(errno), errno, fname); + throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, + &file_spec, "unlink failed: %s (%d), actual filename '%s'", strerror(errno), errno, fname); else return false; + } if(!keep_empty_dirs) rmdir(file_spec, 1); @@ -477,10 +452,8 @@ void file_move(const String& old_spec, c create_dir_for_file(new_spec); if(rename(old_spec_cstr, new_spec_cstr)!=0) - throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, - &old_spec, - "rename failed: %s (%d), actual filename '%s' to '%s'", - strerror(errno), errno, old_spec_cstr, new_spec_cstr); + throw Exception(errno==EACCES ? "file.access" : errno==ENOENT ? "file.missing" : 0, + &old_spec, "rename failed: %s (%d), actual filename '%s' to '%s'", strerror(errno), errno, old_spec_cstr, new_spec_cstr); if(!keep_empty_dirs) rmdir(old_spec, 1); @@ -489,7 +462,7 @@ void file_move(const String& old_spec, c bool entry_exists(const char* fname, struct stat *afinfo) { struct stat lfinfo; - bool result=stat(fname, &lfinfo)==0; + bool result=pa_stat(fname, &lfinfo)==0; if(afinfo) *afinfo=lfinfo; return result; @@ -520,29 +493,28 @@ bool file_executable(const String& file_ return access(file_spec.taint_cstr(String::L_FILE_SPEC), X_OK)==0; } -bool file_stat(const String& file_spec, - size_t& rsize, - time_t& ratime, - time_t& rmtime, - time_t& rctime, - bool fail_on_read_problem) { +bool file_stat(const String& file_spec, uint64_t& rsize, time_t& ratime, time_t& rmtime, time_t& rctime, bool fail_on_read_problem) { const char* fname=file_spec.taint_cstr(String::L_FILE_SPEC); struct stat finfo; - if(stat(fname, &finfo)!=0) + if(pa_stat(fname, &finfo)!=0) { if(fail_on_read_problem) - throw Exception("file.missing", - &file_spec, - "getting file size failed: %s (%d), real filename '%s'", - strerror(errno), errno, fname); + throw Exception("file.missing", &file_spec, "getting file size failed: %s (%d), real filename '%s'", strerror(errno), errno, fname); else return false; + } rsize=finfo.st_size; - ratime=finfo.st_atime; - rmtime=finfo.st_mtime; - rctime=finfo.st_ctime; + ratime=(time_t)finfo.st_atime; + rmtime=(time_t)finfo.st_mtime; + rctime=(time_t)finfo.st_ctime; return true; } +size_t check_file_size(uint64_t size, const String& file_spec){ + if(size > pa_file_size_limit) + throw Exception(PARSER_RUNTIME, &file_spec, "content size of %.15g bytes exceeds the limit (%.15g bytes)", (double)size, (double)pa_file_size_limit); + return (size_t)size; +} + /** String related functions */ @@ -572,6 +544,20 @@ const char* capitalize(const char* s){ return (const char*)result; } +char *str_lower(const char *s, size_t helper_length){ + char *result=pa_strdup(s, helper_length); + for(char* c=result; *c; c++) + *c=(char)tolower((unsigned char)*c); + return result; +} + +char *str_upper(const char *s, size_t helper_length){ + char *result=pa_strdup(s, helper_length); + for(char* c=result; *c; c++) + *c=(char)toupper((unsigned char)*c); + return result; +} + void fix_line_breaks(char *str, size_t& length) { //_asm int 3; const char* const eob=str+length; @@ -727,17 +713,13 @@ const char* format(double value, const c size=snprintf(local_buf, sizeof(local_buf), fmt, (uint)value); break; case FormatInvalid: - throw Exception(PARSER_RUNTIME, - 0, - "Incorrect format string '%s' was specified.", fmt); + throw Exception(PARSER_RUNTIME, 0, "Incorrect format string '%s' was specified.", fmt); } } else size=snprintf(local_buf, sizeof(local_buf), "%d", (int)value); if(size < 0 || size >= MAX_NUMBER-1){ // on win32 we manually reduce max size while printing - throw Exception(PARSER_RUNTIME, - 0, - "Error occure white executing snprintf with format string '%s'.", fmt); + throw Exception(PARSER_RUNTIME, 0, "Error occure white executing snprintf with format string '%s'.", fmt); } return pa_strdup(local_buf, (size_t)size); @@ -782,7 +764,7 @@ char* unescape_chars(const char* cp, int } else { switch(escapeState) { case EscapeRest: - if(!js && c=='+'){ + if(c=='+' && !js){ *dst++=' '; } else { *dst++=c; @@ -863,26 +845,6 @@ void back_slashes_to_slashes(char* s) { } #endif -bool StrStartFromNC(const char* str, const char* substr, bool equal){ - while(true) { - if(!(*substr)){ - if(!(*str)) - return true; - else - return !equal; - } - if(!(*str)) - return false; - if(isalpha((unsigned char)*str)) { - if(tolower((unsigned char)*str)!=tolower((unsigned char)*substr)) - return false; - } else if((*str) != (*substr)) - return false; - str++; - substr++; - } -} - size_t strpos(const char *str, const char *substr) { const char *p = strstr(str, substr); return (p==0)?STRING_NOT_FOUND:p-str; @@ -915,13 +877,15 @@ int remove_crlf(char* start, char* end) return to-start; } +const char* hex_digits="0123456789ABCDEF"; + const char* hex_string(unsigned char* bytes, size_t size, bool upcase) { char *bytes_hex=new(PointerFreeGC) char [size*2/*byte->hh*/+1/*for zero-teminator*/]; unsigned char *src=bytes; unsigned char *end=bytes+size; char *dest=bytes_hex; - const char *hex=upcase?"0123456789ABCDEF":"0123456789abcdef"; + const char *hex=upcase? hex_digits : "0123456789abcdef"; for(; srcs) - r=s; -#endif - b[r]=0; - return r; -} - -int __snprintf(char* b, size_t s, const char* f, ...) { - va_list l; - va_start(l, f); - int r=__vsnprintf(b, s, f, l); - va_end(l); - return r; -} - /* mime64 functions are from libgmime[http://spruce.sourceforge.net/gmime/] lib */ /* * Authors: Michael Zucchi @@ -1260,15 +1169,11 @@ struct File_base64_action_info { unsigned char** base64; }; -static void file_base64_file_action( - struct stat& finfo, - int f, - const String&, const char* /*fname*/, bool, - void *context) { +static void file_base64_file_action(struct stat& finfo, int f, const String& file_spec, void *context) { if(finfo.st_size) { File_base64_action_info& info=*static_cast(context); - *info.base64=new(PointerFreeGC) unsigned char[finfo.st_size * 2 + 6]; + *info.base64=new(PointerFreeGC) unsigned char[check_file_size(finfo.st_size, file_spec) * 2 + 6]; unsigned char* base64 = *info.base64; int state=0; int save=0; @@ -1289,8 +1194,7 @@ char* pa_base64_encode(const String& fil unsigned char* base64=0; File_base64_action_info info={&base64}; - file_read_action_under_lock(file_spec, - "pa_base64_encode", file_base64_file_action, &info); + file_read_action_under_lock(file_spec, "pa_base64_encode", file_base64_file_action, &info); return (char*)base64; } @@ -1303,9 +1207,7 @@ void pa_base64_decode(const char *in, si int state=0; int save=0; - result_size= - g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, - (unsigned char*)result, &state, &save, strict); + result_size=g_mime_utils_base64_decode_step ((const unsigned char*)in, in_size, (unsigned char*)result, &state, &save, strict); assert(result_size <= new_size); result[result_size]=0; // for text files @@ -1317,9 +1219,7 @@ void pa_base64_decode(const char *in, si int file_block_read(const int f, unsigned char* buffer, const size_t size){ int nCount = read(f, buffer, size); if (nCount < 0) - throw Exception("file.read", - 0, - "read failed: %s (%d)", strerror(errno), errno); + throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno); return nCount; } @@ -1352,7 +1252,7 @@ inline void CalcCrc32(const unsigned cha } -const unsigned long pa_crc32(const char *in, size_t in_size){ +unsigned long pa_crc32(const char *in, size_t in_size){ unsigned long crc32=0xFFFFFFFF; InitCrc32Table(); @@ -1362,11 +1262,7 @@ const unsigned long pa_crc32(const char return ~crc32; } -static void file_crc32_file_action( - struct stat& finfo, - int f, - const String&, const char* /*fname*/, bool, - void *context) { +static void file_crc32_file_action(struct stat& finfo, int f, const String&, void *context) { unsigned long& crc32=*static_cast(context); if(finfo.st_size) { InitCrc32Table(); @@ -1379,7 +1275,7 @@ static void file_crc32_file_action( } } -const unsigned long pa_crc32(const String& file_spec){ +unsigned long pa_crc32(const String& file_spec){ unsigned long crc32=0xFFFFFFFF; file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32); return ~crc32; @@ -1390,10 +1286,7 @@ const unsigned long pa_crc32(const Strin // content-type: xxx; charset="WE-NEED-THIS"; Charset* detect_charset(const char* content_type){ if(content_type){ - char* CONTENT_TYPE=pa_strdup(content_type); - - for(char *p=CONTENT_TYPE; *p; p++) - *p=(char)toupper((unsigned char)*p); + char* CONTENT_TYPE=str_upper(content_type); if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){ begin+=8; // skip "CHARSET=" @@ -1409,48 +1302,125 @@ Charset* detect_charset(const char* cont if(end) *end=0; // terminator - return *begin?&charsets.get(begin):0; + return *begin ? &pa_charsets.get_direct(begin) : 0; } } return 0; } -static bool isLeap(int year) { - return !( - (year % 4) || ((year % 400) && !(year % 100)) - ); -} - -int getMonthDays(int year, int month) { - static int monthDays[]={ - 31, - 28, - 31, - 30, - 31, - 30, - 31, - 31, - 30, - 31, - 30, - 31 - }; - return (month == 1 /* january -- 0 */ && isLeap(year)) ? 29 : monthDays[month]; -} - -String::C date_gmt_string(tm* tms) { - /// http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 - static const char month_names[12][4]={ - "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}; - static const char days[7][4]={ - "Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; - - char *buf=new(PointerFreeGC) char[MAX_STRING]; - return String::C(buf, - snprintf(buf, MAX_STRING, "%s, %.2d %s %.4d %.2d:%.2d:%.2d GMT", - days[tms->tm_wday], - tms->tm_mday,month_names[tms->tm_mon],tms->tm_year+1900, - tms->tm_hour,tms->tm_min,tms->tm_sec)); +static bool is_latin(const char *in){ + for(; *in; in++){ + if ((unsigned char)(*in) > 0x7F) + return false; + } + return true; +} + +#define MAX_IDNA_LENGTH 256 + +const char *pa_idna_encode(const char *in, Charset &source_charset){ + if(!in || is_latin(in)) + return in; + + uint32_t utf32[MAX_IDNA_LENGTH]; + uint32_t *utf32_end=utf32; + + String::C sIn(in,strlen(in)); + + if(!source_charset.isUTF8()) + sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset); + + int status=pa_convertUTF8toUTF32((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf32_end, utf32+MAX_IDNA_LENGTH-1, strictConversion); + if(status != conversionOK) + throw Exception("idna encode", new String(in), "utf conversion failed (%d)", status); + + *utf32_end=0; + + char *result = (char *)pa_malloc(MAX_IDNA_LENGTH); + status=pa_idna_to_ascii_4z(utf32, result, MAX_IDNA_LENGTH, 0); + if(status != IDNA_SUCCESS) + throw Exception("idna encode", new String(in), "encode failed: %s", pa_idna_strerror(status)); + + return result; +} + +const char *pa_idna_decode(const char *in, Charset &asked_charset){ + if(!in || !(*in)) + return in; + + uint32_t utf32[MAX_IDNA_LENGTH]; + const uint32_t *utf32_start=utf32; + uint32_t *utf32_end; + + int status=pa_idna_to_unicode_4z(in, utf32, MAX_IDNA_LENGTH, 0); + if(status != IDNA_SUCCESS) + throw Exception("idna decode", new String(in), "decode failed: %s", pa_idna_strerror(status)); + + for(utf32_end=utf32; *utf32_end; utf32_end++); + + char *result = (char *)pa_malloc(MAX_IDNA_LENGTH); + char *result_end = result; + + status=pa_convertUTF32toUTF8(&utf32_start, utf32_end, (UTF8**)&result_end, (UTF8*)(result+MAX_IDNA_LENGTH-1), strictConversion); + + if(status != conversionOK) + throw Exception("idna decode", new String(in), "utf conversion failed (%d)", status); + + *result_end='\0'; + + if(!asked_charset.isUTF8()) + result = (char *)Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr(); + + return result; +} +/// must be last in this file +#undef vsnprintf +int pa_vsnprintf(char* b, size_t s, const char* f, va_list l) { + if(!s) + return 0; + + int r; + // note: on win32 & maybe somewhere else + // vsnprintf do not writes terminating 0 in 'buffer full' case, reducing + // http://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010 + --s; + + // clients do not check for negative 's', feature: ignore such prints + if((ssize_t)s<0) + return 0; + +#ifdef _MSC_VER + // win32: if the number of bytes to write exceeds buffer, then count bytes are written and -1 is returned + r=_vsnprintf(b, s, f, l); + if(r<0) + r=s; +#else + r=vsnprintf(b, s, f, l); + /* + solaris: man vsnprintf + + The snprintf() function returns the number of characters + formatted, that is, the number of characters that would have + been written to the buffer if it were large enough. If the + value of n is 0 on a call to snprintf(), an unspecified + value less than 1 is returned. + */ + + if(r<0) + r=0; + else if((size_t)r>s) + r=s; +#endif + b[r]=0; + return r; +} + +int pa_snprintf(char* b, size_t s, const char* f, ...) { + va_list l; + va_start(l, f); + int r=pa_vsnprintf(b, s, f, l); + va_end(l); + return r; } +