Annotation of parser3/src/main/pa_common.C, revision 1.350
1.15 paf 1: /** @file
1.16 paf 2: Parser: commonly functions.
3:
1.343 moko 4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
1.327 moko 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
1.306 moko 6: */
1.210 paf 7:
1.1 paf 8: #include "pa_common.h"
1.4 paf 9: #include "pa_exception.h"
1.342 moko 10: #include "pa_inline_hash.h"
1.14 paf 11: #include "pa_globals.h"
1.154 paf 12: #include "pa_charsets.h"
1.223 misha 13: #include "pa_request_charsets.h"
1.347 moko 14: #include "pa_file.h"
15: #include "pa_stack.h"
1.98 paf 16:
1.283 moko 17: #include "pa_idna.h"
18: #include "pa_convert_utf.h"
19:
1.350 ! moko 20: volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.349 2026/10/01 11:48:37 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_INLINE_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H;
1.176 paf 21:
1.154 paf 22: // defines for globals
23:
24: #define FILE_STATUS_NAME "status"
25:
26: // globals
27:
28: const String file_status_name(FILE_STATUS_NAME);
29:
1.305 moko 30: String sql_bind_name(SQL_BIND_NAME);
31: String sql_limit_name(PA_SQL_LIMIT_NAME);
32: String sql_offset_name(PA_SQL_OFFSET_NAME);
33: String sql_default_name(SQL_DEFAULT_NAME);
34: String sql_distinct_name(SQL_DISTINCT_NAME);
35: String sql_value_type_name(SQL_VALUE_TYPE_NAME);
36:
1.278 moko 37: bool capitalized(const char* s){
38: bool upper=true;
39: for(const char* c=s; *c; c++){
40: if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c)))
41: return false;
42: upper=strchr("-_ ", *c) != 0;
43: }
44: return true;
45: }
46:
47: const char* capitalize(const char* s){
48: if(!s || capitalized(s))
49: return s;
50:
51: char* result=pa_strdup(s);
52: if(result){
53: bool upper=true;
54: for(char* c=result; *c; c++){
55: *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c);
56: upper=strchr("-_ ", *c) != 0;
57: }
58: }
59: return (const char*)result;
60: }
61:
1.323 moko 62: char *str_lower(const char *s, size_t length){
63: char *result=pa_strdup(s, length);
1.290 moko 64: for(char* c=result; *c; c++)
65: *c=(char)tolower((unsigned char)*c);
66: return result;
67: }
68:
1.323 moko 69: char *str_upper(const char *s, size_t length){
70: char *result=pa_strdup(s, length);
1.290 moko 71: for(char* c=result; *c; c++)
72: *c=(char)toupper((unsigned char)*c);
73: return result;
74: }
75:
1.278 moko 76: void fix_line_breaks(char *str, size_t& length) {
77: //_asm int 3;
78: const char* const eob=str+length;
79: char* dest=str;
80: // fix DOS: \r\n -> \n
81: // fix Macintosh: \r -> \n
82: char* bol=str;
83: while(char* eol=(char*)memchr(bol, '\r', eob -bol)) {
84: size_t len=eol-bol;
85: if(dest!=bol)
1.347 moko 86: memmove(dest, bol, len);
1.278 moko 87: dest+=len;
88: *dest++='\n';
89:
90: if(&eol[1]<eob && eol[1]=='\n') { // \r, \n = DOS
91: bol=eol+2;
92: length--;
93: } else // \r, not \n = Macintosh
94: bol=eol+1;
95: }
96: // last piece without \r
97: if(dest!=bol)
98: memmove(dest, bol, eob-bol);
99: str[length]=0; // terminating
100: }
101:
102: /**
1.347 moko 103: scans for @a delim[default \n] in @a *row_ref,
1.278 moko 104: @return piece of line before it or end of string, if no @a delim found
105: assigns @a *row_ref to point right after delimiter if there were one
106: or to zero if no @a delim were found.
107: */
108:
1.126 paf 109: char* getrow(char* *row_ref, char delim) {
1.229 misha 110: char* result=*row_ref;
111: if(result) {
1.126 paf 112: *row_ref=strchr(result, delim);
1.8 paf 113: if(*row_ref)
114: *((*row_ref)++)=0;
115: else if(!*result)
116: return 0;
1.229 misha 117: }
118: return result;
1.8 paf 119: }
120:
1.344 moko 121: const char* strrpbrk(const char* string, const char* chars) {
122: if(string) {
123: const char* pos=string+strlen(string);
124: while(pos!=string) {
125: if(strchr(chars, *--pos))
126: return pos;
127: }
128: }
129: return NULL;
130: }
131:
1.126 paf 132: char* lsplit(char* string, char delim) {
1.229 misha 133: if(string) {
1.344 moko 134: char* v=strchr(string, delim);
1.8 paf 135: if(v) {
136: *v=0;
137: return v+1;
138: }
1.229 misha 139: }
140: return 0;
1.8 paf 141: }
142:
1.126 paf 143: char* lsplit(char* *string_ref, char delim) {
1.229 misha 144: char* result=*string_ref;
1.344 moko 145: char* next=lsplit(*string_ref, delim);
1.229 misha 146: *string_ref=next;
147: return result;
1.9 paf 148: }
149:
1.126 paf 150: char* rsplit(char* string, char delim) {
1.229 misha 151: if(string) {
1.344 moko 152: char* v=strrchr(string, delim);
1.18 paf 153: if(v) {
1.9 paf 154: *v=0;
155: return v+1;
156: }
1.229 misha 157: }
1.314 moko 158: return NULL;
159: }
160:
1.344 moko 161: char* rsplit(char* string, const char* delims) {
162: if(char* v=strrpbrk(string, delims)) {
163: *v=0;
164: return v+1;
165: }
166: return NULL;
167: }
168:
1.348 moko 169: void pa_strncpy(char *dst, const char *src, size_t dst_size){
170: size_t left = dst_size;
1.325 moko 171:
172: if (left != 0 && src) {
173: while (--left != 0) {
174: if ((*dst++ = *src++) == '\0')
175: return;
176: }
177: }
1.348 moko 178: if (dst_size != 0)
1.325 moko 179: *dst = '\0';
180: }
181:
1.314 moko 182: #define STRCAT_STEP(str, len) if(str) { memcpy(ptr, str, len); ptr += len; }
183:
184: char *pa_strcat(const char *a, const char *b, const char *c) {
185: size_t len_a = a ? strlen(a) : 0;
1.315 moko 186: size_t len_b = b ? strlen(b) : 0;
187: size_t len_c = c ? strlen(c) : 0;
1.314 moko 188: char *result=new(PointerFreeGC) char[len_a + len_b + len_c +1/*0*/];
189: char *ptr=result;
190: STRCAT_STEP(a, len_a);
191: STRCAT_STEP(b, len_b);
192: STRCAT_STEP(c, len_c);
193: *ptr='\0';
194: return result;
1.10 paf 195: }
196:
1.229 misha 197:
1.36 paf 198: size_t stdout_write(const void *buf, size_t size) {
1.12 paf 199: #ifdef WIN32
1.187 paf 200: size_t to_write = size;
1.12 paf 201: do{
1.316 moko 202: int chunk_written=fwrite(buf, 1, min((size_t)8*0x400, size), stdout);
1.12 paf 203: if(chunk_written<=0)
204: break;
205: size-=chunk_written;
1.36 paf 206: buf=((const char*)buf)+chunk_written;
1.316 moko 207: } while(size>0);
1.12 paf 208:
1.316 moko 209: fflush(stdout);
1.187 paf 210: return to_write-size;
1.12 paf 211: #else
1.316 moko 212: size_t result=fwrite(buf, 1, size, stdout);
213: fflush(stdout);
214: return result;
1.12 paf 215: #endif
1.2 paf 216: }
1.14 paf 217:
1.229 misha 218: enum EscapeState {
1.341 moko 219: EscapeRest,
220: EscapeFirst,
1.229 misha 221: EscapeSecond,
222: EscapeUnicode
223: };
224:
1.236 misha 225: // @todo prescan for reduce required size (unescaped sting in 1 byte charset requires less memory usually)
1.258 misha 226: char* unescape_chars(const char* cp, int len, Charset* charset, bool js){
1.236 misha 227: char* s=new(PointerFreeGC) char[len+1]; // must be enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes)
1.336 moko 228: XMLByte* dst=(XMLByte *)s;
1.229 misha 229: EscapeState escapeState=EscapeRest;
230: uint escapedValue=0;
231: int srcPos=0;
1.230 misha 232: short int jsCnt=0;
1.236 misha 233: while(srcPos<len){
1.229 misha 234: uchar c=(uchar)cp[srcPos];
1.258 misha 235: if(c=='%' || (c=='\\' && js)){
1.229 misha 236: escapeState=EscapeFirst;
237: } else {
238: switch(escapeState) {
239: case EscapeRest:
1.286 moko 240: if(c=='+' && !js){
1.230 misha 241: *dst++=' ';
1.229 misha 242: } else {
1.230 misha 243: *dst++=c;
1.229 misha 244: }
245: break;
246: case EscapeFirst:
1.232 misha 247: if(charset && c=='u'){
1.229 misha 248: // escaped unicode value: %u0430
249: jsCnt=0;
250: escapedValue=0;
251: escapeState=EscapeUnicode;
252: } else {
1.231 misha 253: if(isxdigit(c)){
1.229 misha 254: escapedValue=hex_value[c] << 4;
255: escapeState=EscapeSecond;
256: } else {
1.230 misha 257: *dst++=c;
1.229 misha 258: escapeState=EscapeRest;
259: }
260: }
261: break;
262: case EscapeSecond:
1.231 misha 263: if(isxdigit(c)){
1.229 misha 264: escapedValue+=hex_value[c];
1.230 misha 265: *dst++=(char)escapedValue;
1.229 misha 266: }
267: escapeState=EscapeRest;
268: break;
269: case EscapeUnicode:
1.231 misha 270: if(isxdigit(c)){
1.229 misha 271: escapedValue=(escapedValue << 4) + hex_value[c];
272: if(++jsCnt==4){
1.230 misha 273: // transcode utf8 char to client charset (we can lost some chars here)
1.336 moko 274: charset->store_Char(dst, (XMLCh)escapedValue, '?');
1.229 misha 275: escapeState=EscapeRest;
276: }
277: } else {
278: // not full unicode value
279: escapeState=EscapeRest;
280: }
281: break;
282: }
283: }
284:
285: srcPos++;
286: }
287:
1.230 misha 288: *dst=0; // zero-termination
1.229 misha 289: return s;
290: }
1.24 paf 291:
1.268 misha 292: char *search_stop(char*& current, char cstop_at) {
293: // sanity check
294: if(!current)
295: return 0;
296:
297: // skip leading WS
298: while(*current==' ' || *current=='\t')
299: current++;
300: if(!*current)
301: return current=0;
302:
303: char *result=current;
304: if(char *pstop_at=strchr(current, cstop_at)) {
305: *pstop_at=0;
306: current=pstop_at+1;
307: } else
308: current=0;
309: return result;
310: }
311:
1.348 moko 312: char* backslashes_to_slashes(const char* path) {
313: if(!path)
314: return 0;
315: char* result=pa_strdup(path);
316: for(char* s=result; *s; s++)
317: if(*s=='\\')
318: *s='/';
319: return result;
320: }
321:
322: const String& backslashes_to_slashes(const String& path) {
1.349 moko 323: return path.pos('\\')==STRING_NOT_FOUND ? path : path.change_char('\\', '/');
1.24 paf 324: }
1.41 paf 325:
1.232 misha 326: size_t strpos(const char *str, const char *substr) {
327: const char *p = strstr(str, substr);
328: return (p==0)?STRING_NOT_FOUND:p-str;
329: }
330:
1.326 moko 331: size_t remove_crlf(char* start, char* end) {
1.226 misha 332: char* from=start;
333: char* to=start;
334: bool skip=false;
335: while(from < end){
336: switch(*from){
337: case '\n':
338: case '\r':
339: case '\t':
340: case ' ':
341: if(!skip){
342: *to=' ';
343: to++;
344: skip=true;
345: }
346: break;
347: default:
348: if(from != to)
349: *to=*from;
350: to++;
351: skip=false;
1.69 parser 352: }
1.226 misha 353: from++;
354: }
355: return to-start;
1.91 paf 356: }
357:
1.279 moko 358: const char* hex_digits="0123456789ABCDEF";
359:
1.278 moko 360: const char* hex_string(unsigned char* bytes, size_t size, bool upcase) {
361: char *bytes_hex=new(PointerFreeGC) char [size*2/*byte->hh*/+1/*for zero-teminator*/];
362: unsigned char *src=bytes;
363: unsigned char *end=bytes+size;
364: char *dest=bytes_hex;
365:
1.279 moko 366: const char *hex=upcase? hex_digits : "0123456789abcdef";
1.278 moko 367:
368: for(; src<end; src++) {
369: *dest++=hex[*src/0x10];
370: *dest++=hex[*src%0x10];
371: }
372: *dest=0;
373:
374: return bytes_hex;
375: }
1.91 paf 376:
1.321 moko 377: ssize_t file_block_read(const int f, void* buffer, const size_t size){
378: ssize_t nCount = read(f, buffer, size);
1.221 misha 379: if (nCount < 0)
1.289 moko 380: throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno);
1.221 misha 381: return nCount;
382: }
383:
1.278 moko 384: static unsigned long crc32Table[256];
385: static void InitCrc32Table()
386: {
387: if(crc32Table[1] == 0){
388: // This is the official polynomial used by CRC32 in PKZip.
389: // Often times the polynomial shown reversed as 0x04C11DB7.
390: static const unsigned long dwPolynomial = 0xEDB88320;
391:
392: for(int i = 0; i < 256; i++)
393: {
394: unsigned long dwCrc = i;
395: for(int j = 8; j > 0; j--)
396: {
397: if(dwCrc & 1)
398: dwCrc = (dwCrc >> 1) ^ dwPolynomial;
399: else
400: dwCrc >>= 1;
401: }
402: crc32Table[i] = dwCrc;
403: }
404: }
405: }
406:
407: inline void CalcCrc32(const unsigned char byte, unsigned long &crc32)
408: {
409: crc32 = ((crc32) >> 8) ^ crc32Table[(byte) ^ ((crc32) & 0x000000FF)];
410: }
411:
412:
1.297 moko 413: unsigned long pa_crc32(const char *in, size_t in_size){
1.218 misha 414: unsigned long crc32=0xFFFFFFFF;
1.220 misha 415:
1.240 misha 416: InitCrc32Table();
1.239 misha 417: for(size_t i = 0; i<in_size; i++)
418: CalcCrc32(in[i], crc32);
1.220 misha 419:
1.218 misha 420: return ~crc32;
421: }
422:
1.289 moko 423: static void file_crc32_file_action(struct stat& finfo, int f, const String&, void *context) {
1.218 misha 424: unsigned long& crc32=*static_cast<unsigned long *>(context);
425: if(finfo.st_size) {
426: InitCrc32Table();
1.220 misha 427: int nCount=0;
1.218 misha 428: do {
1.221 misha 429: unsigned char buffer[FILE_BUFFER_SIZE];
430: nCount = file_block_read(f, buffer, sizeof(buffer));
1.220 misha 431: for(int i = 0; i < nCount; i++) CalcCrc32(buffer[i], crc32);
432: } while(nCount > 0);
1.218 misha 433: }
434: }
435:
1.297 moko 436: unsigned long pa_crc32(const String& file_spec){
1.278 moko 437: unsigned long crc32=0xFFFFFFFF;
438: file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32);
439: return ~crc32;
440: }
441:
442: // content-type: xxx; charset=WE-NEED-THIS
443: // content-type: xxx; charset="WE-NEED-THIS"
444: // content-type: xxx; charset="WE-NEED-THIS";
445: Charset* detect_charset(const char* content_type){
446: if(content_type){
1.291 moko 447: char* CONTENT_TYPE=str_upper(content_type);
1.278 moko 448:
449: if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){
450: begin+=8; // skip "CHARSET="
451: char* end=0;
452: if(*begin && (*begin=='"' || *begin =='\'')){
453: char quote=*begin;
454: begin++;
455: end=(char*)strchr(begin, quote);
456: }
457: if(!end)
458: end=(char*)strchr(begin, ';');
459:
460: if(end)
461: *end=0; // terminator
462:
1.295 moko 463: return *begin ? &pa_charsets.get_direct(begin) : 0;
1.278 moko 464: }
465: }
466: return 0;
467: }
468:
1.301 moko 469: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset){
1.302 moko 470: if(!in)
471: return 0;
1.301 moko 472:
1.302 moko 473: String::C sIn(in,strlen(in));
1.301 moko 474:
1.302 moko 475: UTF16* utf16=(UTF16*)pa_malloc_atomic(sIn.length*2+2);
476: UTF16* utf16_end=utf16;
1.301 moko 477:
1.302 moko 478: if(!source_charset.isUTF8())
479: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.301 moko 480:
1.302 moko 481: int status=pa_convertUTF8toUTF16((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf16_end, utf16+sIn.length, strictConversion);
482: if(status != conversionOK)
1.350 ! moko 483: return 0; // without throw, callers treat 0 as an invalid file name
1.301 moko 484:
1.302 moko 485: *utf16_end=0;
1.301 moko 486:
1.302 moko 487: return utf16;
1.301 moko 488: }
489:
490: const char* pa_utf16_decode(const UTF16* in, Charset& asked_charset){
491: if(!in)
492: return 0;
493:
494: const UTF16* utf16_start=in;
495: const UTF16* utf16_end;
496:
497: for(utf16_end=in; *utf16_end; utf16_end++);
498:
499: char *result = (char *)pa_malloc_atomic((utf16_end-in)*6+1);
500: char *result_end = result;
501:
502: int status=pa_convertUTF16toUTF8(&utf16_start, utf16_end, (UTF8**)&result_end, (UTF8*)(result+(utf16_end-in)*6), strictConversion);
503:
504: if(status != conversionOK)
505: throw Exception("utf-16 decode", 0, "utf conversion failed (%d)", status);
506:
507: *result_end='\0';
508:
1.302 moko 509: if(asked_charset.isUTF8())
510: return result;
1.301 moko 511:
1.302 moko 512: return Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.301 moko 513: }
1.278 moko 514:
1.283 moko 515: static bool is_latin(const char *in){
516: for(; *in; in++){
517: if ((unsigned char)(*in) > 0x7F)
518: return false;
519: }
520: return true;
521: }
522:
523: #define MAX_IDNA_LENGTH 256
524:
1.301 moko 525: const char *pa_idna_encode(const char *in, Charset& source_charset){
1.283 moko 526: if(!in || is_latin(in))
527: return in;
528:
529: uint32_t utf32[MAX_IDNA_LENGTH];
530: uint32_t *utf32_end=utf32;
531:
532: String::C sIn(in,strlen(in));
533:
534: if(!source_charset.isUTF8())
1.295 moko 535: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.283 moko 536:
537: int status=pa_convertUTF8toUTF32((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf32_end, utf32+MAX_IDNA_LENGTH-1, strictConversion);
538: if(status != conversionOK)
539: throw Exception("idna encode", new String(in), "utf conversion failed (%d)", status);
540:
541: *utf32_end=0;
542:
543: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
544: status=pa_idna_to_ascii_4z(utf32, result, MAX_IDNA_LENGTH, 0);
545: if(status != IDNA_SUCCESS)
546: throw Exception("idna encode", new String(in), "encode failed: %s", pa_idna_strerror(status));
547:
548: return result;
549: }
550:
551: const char *pa_idna_decode(const char *in, Charset &asked_charset){
552: if(!in || !(*in))
553: return in;
554:
555: uint32_t utf32[MAX_IDNA_LENGTH];
556: const uint32_t *utf32_start=utf32;
557: uint32_t *utf32_end;
558:
559: int status=pa_idna_to_unicode_4z(in, utf32, MAX_IDNA_LENGTH, 0);
560: if(status != IDNA_SUCCESS)
561: throw Exception("idna decode", new String(in), "decode failed: %s", pa_idna_strerror(status));
562:
563: for(utf32_end=utf32; *utf32_end; utf32_end++);
564:
565: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
566: char *result_end = result;
567:
568: status=pa_convertUTF32toUTF8(&utf32_start, utf32_end, (UTF8**)&result_end, (UTF8*)(result+MAX_IDNA_LENGTH-1), strictConversion);
569:
570: if(status != conversionOK)
571: throw Exception("idna decode", new String(in), "utf conversion failed (%d)", status);
572:
573: *result_end='\0';
574:
575: if(!asked_charset.isUTF8())
1.295 moko 576: result = (char *)Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.283 moko 577:
578: return result;
579: }
1.292 moko 580: /// must be last in this file
581: #undef vsnprintf
582: int pa_vsnprintf(char* b, size_t s, const char* f, va_list l) {
583: if(!s)
584: return 0;
585:
586: int r;
587: // note: on win32 & maybe somewhere else
588: // vsnprintf do not writes terminating 0 in 'buffer full' case, reducing
589: // http://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010
590: --s;
591:
592: // clients do not check for negative 's', feature: ignore such prints
593: if((ssize_t)s<0)
594: return 0;
595:
596: #ifdef _MSC_VER
597: // win32: if the number of bytes to write exceeds buffer, then count bytes are written and -1 is returned
598: r=_vsnprintf(b, s, f, l);
599: if(r<0)
600: r=s;
601: #else
602: r=vsnprintf(b, s, f, l);
603: /*
604: solaris: man vsnprintf
605:
606: The snprintf() function returns the number of characters
607: formatted, that is, the number of characters that would have
608: been written to the buffer if it were large enough. If the
609: value of n is 0 on a call to snprintf(), an unspecified
610: value less than 1 is returned.
611: */
612:
613: if(r<0)
614: r=0;
615: else if((size_t)r>s)
616: r=s;
617: #endif
618: b[r]=0;
619: return r;
620: }
621:
622: int pa_snprintf(char* b, size_t s, const char* f, ...) {
623: va_list l;
624: va_start(l, f);
625: int r=pa_vsnprintf(b, s, f, l);
626: va_end(l);
627: return r;
628: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_common.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main