Annotation of parser3/src/main/pa_common.C, revision 1.348
1.15 paf 1: /** @file
1.16 paf 2: Parser: commonly functions.
3:
1.343 moko 4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
1.327 moko 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
1.306 moko 6: */
1.210 paf 7:
1.1 paf 8: #include "pa_common.h"
1.4 paf 9: #include "pa_exception.h"
1.342 moko 10: #include "pa_inline_hash.h"
1.14 paf 11: #include "pa_globals.h"
1.154 paf 12: #include "pa_charsets.h"
1.223 misha 13: #include "pa_request_charsets.h"
1.347 moko 14: #include "pa_file.h"
15: #include "pa_stack.h"
1.98 paf 16:
1.283 moko 17: #include "pa_idna.h"
18: #include "pa_convert_utf.h"
19:
1.348 ! moko 20: volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.347 2026/09/27 17:13:40 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_INLINE_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H;
1.176 paf 21:
1.154 paf 22: // defines for globals
23:
24: #define FILE_STATUS_NAME "status"
25:
26: // globals
27:
28: const String file_status_name(FILE_STATUS_NAME);
29:
1.305 moko 30: String sql_bind_name(SQL_BIND_NAME);
31: String sql_limit_name(PA_SQL_LIMIT_NAME);
32: String sql_offset_name(PA_SQL_OFFSET_NAME);
33: String sql_default_name(SQL_DEFAULT_NAME);
34: String sql_distinct_name(SQL_DISTINCT_NAME);
35: String sql_value_type_name(SQL_VALUE_TYPE_NAME);
36:
1.278 moko 37: bool capitalized(const char* s){
38: bool upper=true;
39: for(const char* c=s; *c; c++){
40: if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c)))
41: return false;
42: upper=strchr("-_ ", *c) != 0;
43: }
44: return true;
45: }
46:
47: const char* capitalize(const char* s){
48: if(!s || capitalized(s))
49: return s;
50:
51: char* result=pa_strdup(s);
52: if(result){
53: bool upper=true;
54: for(char* c=result; *c; c++){
55: *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c);
56: upper=strchr("-_ ", *c) != 0;
57: }
58: }
59: return (const char*)result;
60: }
61:
1.323 moko 62: char *str_lower(const char *s, size_t length){
63: char *result=pa_strdup(s, length);
1.290 moko 64: for(char* c=result; *c; c++)
65: *c=(char)tolower((unsigned char)*c);
66: return result;
67: }
68:
1.323 moko 69: char *str_upper(const char *s, size_t length){
70: char *result=pa_strdup(s, length);
1.290 moko 71: for(char* c=result; *c; c++)
72: *c=(char)toupper((unsigned char)*c);
73: return result;
74: }
75:
1.278 moko 76: void fix_line_breaks(char *str, size_t& length) {
77: //_asm int 3;
78: const char* const eob=str+length;
79: char* dest=str;
80: // fix DOS: \r\n -> \n
81: // fix Macintosh: \r -> \n
82: char* bol=str;
83: while(char* eol=(char*)memchr(bol, '\r', eob -bol)) {
84: size_t len=eol-bol;
85: if(dest!=bol)
1.347 moko 86: memmove(dest, bol, len);
1.278 moko 87: dest+=len;
88: *dest++='\n';
89:
90: if(&eol[1]<eob && eol[1]=='\n') { // \r, \n = DOS
91: bol=eol+2;
92: length--;
93: } else // \r, not \n = Macintosh
94: bol=eol+1;
95: }
96: // last piece without \r
97: if(dest!=bol)
98: memmove(dest, bol, eob-bol);
99: str[length]=0; // terminating
100: }
101:
102: /**
1.347 moko 103: scans for @a delim[default \n] in @a *row_ref,
1.278 moko 104: @return piece of line before it or end of string, if no @a delim found
105: assigns @a *row_ref to point right after delimiter if there were one
106: or to zero if no @a delim were found.
107: */
108:
1.126 paf 109: char* getrow(char* *row_ref, char delim) {
1.229 misha 110: char* result=*row_ref;
111: if(result) {
1.126 paf 112: *row_ref=strchr(result, delim);
1.8 paf 113: if(*row_ref)
114: *((*row_ref)++)=0;
115: else if(!*result)
116: return 0;
1.229 misha 117: }
118: return result;
1.8 paf 119: }
120:
1.344 moko 121: const char* strrpbrk(const char* string, const char* chars) {
122: if(string) {
123: const char* pos=string+strlen(string);
124: while(pos!=string) {
125: if(strchr(chars, *--pos))
126: return pos;
127: }
128: }
129: return NULL;
130: }
131:
1.126 paf 132: char* lsplit(char* string, char delim) {
1.229 misha 133: if(string) {
1.344 moko 134: char* v=strchr(string, delim);
1.8 paf 135: if(v) {
136: *v=0;
137: return v+1;
138: }
1.229 misha 139: }
140: return 0;
1.8 paf 141: }
142:
1.126 paf 143: char* lsplit(char* *string_ref, char delim) {
1.229 misha 144: char* result=*string_ref;
1.344 moko 145: char* next=lsplit(*string_ref, delim);
1.229 misha 146: *string_ref=next;
147: return result;
1.9 paf 148: }
149:
1.126 paf 150: char* rsplit(char* string, char delim) {
1.229 misha 151: if(string) {
1.344 moko 152: char* v=strrchr(string, delim);
1.18 paf 153: if(v) {
1.9 paf 154: *v=0;
155: return v+1;
156: }
1.229 misha 157: }
1.314 moko 158: return NULL;
159: }
160:
1.344 moko 161: char* rsplit(char* string, const char* delims) {
162: if(char* v=strrpbrk(string, delims)) {
163: *v=0;
164: return v+1;
165: }
166: return NULL;
167: }
168:
1.348 ! moko 169: void pa_strncpy(char *dst, const char *src, size_t dst_size){
! 170: size_t left = dst_size;
1.325 moko 171:
172: if (left != 0 && src) {
173: while (--left != 0) {
174: if ((*dst++ = *src++) == '\0')
175: return;
176: }
177: }
1.348 ! moko 178: if (dst_size != 0)
1.325 moko 179: *dst = '\0';
180: }
181:
1.314 moko 182: #define STRCAT_STEP(str, len) if(str) { memcpy(ptr, str, len); ptr += len; }
183:
184: char *pa_strcat(const char *a, const char *b, const char *c) {
185: size_t len_a = a ? strlen(a) : 0;
1.315 moko 186: size_t len_b = b ? strlen(b) : 0;
187: size_t len_c = c ? strlen(c) : 0;
1.314 moko 188: char *result=new(PointerFreeGC) char[len_a + len_b + len_c +1/*0*/];
189: char *ptr=result;
190: STRCAT_STEP(a, len_a);
191: STRCAT_STEP(b, len_b);
192: STRCAT_STEP(c, len_c);
193: *ptr='\0';
194: return result;
1.10 paf 195: }
196:
1.229 misha 197:
1.36 paf 198: size_t stdout_write(const void *buf, size_t size) {
1.12 paf 199: #ifdef WIN32
1.187 paf 200: size_t to_write = size;
1.12 paf 201: do{
1.316 moko 202: int chunk_written=fwrite(buf, 1, min((size_t)8*0x400, size), stdout);
1.12 paf 203: if(chunk_written<=0)
204: break;
205: size-=chunk_written;
1.36 paf 206: buf=((const char*)buf)+chunk_written;
1.316 moko 207: } while(size>0);
1.12 paf 208:
1.316 moko 209: fflush(stdout);
1.187 paf 210: return to_write-size;
1.12 paf 211: #else
1.316 moko 212: size_t result=fwrite(buf, 1, size, stdout);
213: fflush(stdout);
214: return result;
1.12 paf 215: #endif
1.2 paf 216: }
1.14 paf 217:
1.229 misha 218: enum EscapeState {
1.341 moko 219: EscapeRest,
220: EscapeFirst,
1.229 misha 221: EscapeSecond,
222: EscapeUnicode
223: };
224:
1.236 misha 225: // @todo prescan for reduce required size (unescaped sting in 1 byte charset requires less memory usually)
1.258 misha 226: char* unescape_chars(const char* cp, int len, Charset* charset, bool js){
1.236 misha 227: char* s=new(PointerFreeGC) char[len+1]; // must be enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes)
1.336 moko 228: XMLByte* dst=(XMLByte *)s;
1.229 misha 229: EscapeState escapeState=EscapeRest;
230: uint escapedValue=0;
231: int srcPos=0;
1.230 misha 232: short int jsCnt=0;
1.236 misha 233: while(srcPos<len){
1.229 misha 234: uchar c=(uchar)cp[srcPos];
1.258 misha 235: if(c=='%' || (c=='\\' && js)){
1.229 misha 236: escapeState=EscapeFirst;
237: } else {
238: switch(escapeState) {
239: case EscapeRest:
1.286 moko 240: if(c=='+' && !js){
1.230 misha 241: *dst++=' ';
1.229 misha 242: } else {
1.230 misha 243: *dst++=c;
1.229 misha 244: }
245: break;
246: case EscapeFirst:
1.232 misha 247: if(charset && c=='u'){
1.229 misha 248: // escaped unicode value: %u0430
249: jsCnt=0;
250: escapedValue=0;
251: escapeState=EscapeUnicode;
252: } else {
1.231 misha 253: if(isxdigit(c)){
1.229 misha 254: escapedValue=hex_value[c] << 4;
255: escapeState=EscapeSecond;
256: } else {
1.230 misha 257: *dst++=c;
1.229 misha 258: escapeState=EscapeRest;
259: }
260: }
261: break;
262: case EscapeSecond:
1.231 misha 263: if(isxdigit(c)){
1.229 misha 264: escapedValue+=hex_value[c];
1.230 misha 265: *dst++=(char)escapedValue;
1.229 misha 266: }
267: escapeState=EscapeRest;
268: break;
269: case EscapeUnicode:
1.231 misha 270: if(isxdigit(c)){
1.229 misha 271: escapedValue=(escapedValue << 4) + hex_value[c];
272: if(++jsCnt==4){
1.230 misha 273: // transcode utf8 char to client charset (we can lost some chars here)
1.336 moko 274: charset->store_Char(dst, (XMLCh)escapedValue, '?');
1.229 misha 275: escapeState=EscapeRest;
276: }
277: } else {
278: // not full unicode value
279: escapeState=EscapeRest;
280: }
281: break;
282: }
283: }
284:
285: srcPos++;
286: }
287:
1.230 misha 288: *dst=0; // zero-termination
1.229 misha 289: return s;
290: }
1.24 paf 291:
1.268 misha 292: char *search_stop(char*& current, char cstop_at) {
293: // sanity check
294: if(!current)
295: return 0;
296:
297: // skip leading WS
298: while(*current==' ' || *current=='\t')
299: current++;
300: if(!*current)
301: return current=0;
302:
303: char *result=current;
304: if(char *pstop_at=strchr(current, cstop_at)) {
305: *pstop_at=0;
306: current=pstop_at+1;
307: } else
308: current=0;
309: return result;
310: }
311:
1.348 ! moko 312: char* backslashes_to_slashes(const char* path) {
! 313: if(!path)
! 314: return 0;
! 315: char* result=pa_strdup(path);
! 316: for(char* s=result; *s; s++)
! 317: if(*s=='\\')
! 318: *s='/';
! 319: return result;
! 320: }
! 321:
! 322: const String& backslashes_to_slashes(const String& path) {
! 323: size_t pos=path.pos('\\');
! 324: if(pos==STRING_NOT_FOUND)
! 325: return path;
! 326: String& result=*new String;
! 327: size_t from=0;
! 328: do {
! 329: result << path.mid(from, pos) << "/";
! 330: from=pos+1;
! 331: } while((pos=path.pos('\\', from))!=STRING_NOT_FOUND);
! 332: result << path.mid(from, path.length());
! 333: return result;
1.24 paf 334: }
1.41 paf 335:
1.232 misha 336: size_t strpos(const char *str, const char *substr) {
337: const char *p = strstr(str, substr);
338: return (p==0)?STRING_NOT_FOUND:p-str;
339: }
340:
1.326 moko 341: size_t remove_crlf(char* start, char* end) {
1.226 misha 342: char* from=start;
343: char* to=start;
344: bool skip=false;
345: while(from < end){
346: switch(*from){
347: case '\n':
348: case '\r':
349: case '\t':
350: case ' ':
351: if(!skip){
352: *to=' ';
353: to++;
354: skip=true;
355: }
356: break;
357: default:
358: if(from != to)
359: *to=*from;
360: to++;
361: skip=false;
1.69 parser 362: }
1.226 misha 363: from++;
364: }
365: return to-start;
1.91 paf 366: }
367:
1.279 moko 368: const char* hex_digits="0123456789ABCDEF";
369:
1.278 moko 370: const char* hex_string(unsigned char* bytes, size_t size, bool upcase) {
371: char *bytes_hex=new(PointerFreeGC) char [size*2/*byte->hh*/+1/*for zero-teminator*/];
372: unsigned char *src=bytes;
373: unsigned char *end=bytes+size;
374: char *dest=bytes_hex;
375:
1.279 moko 376: const char *hex=upcase? hex_digits : "0123456789abcdef";
1.278 moko 377:
378: for(; src<end; src++) {
379: *dest++=hex[*src/0x10];
380: *dest++=hex[*src%0x10];
381: }
382: *dest=0;
383:
384: return bytes_hex;
385: }
1.91 paf 386:
1.321 moko 387: ssize_t file_block_read(const int f, void* buffer, const size_t size){
388: ssize_t nCount = read(f, buffer, size);
1.221 misha 389: if (nCount < 0)
1.289 moko 390: throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno);
1.221 misha 391: return nCount;
392: }
393:
1.278 moko 394: static unsigned long crc32Table[256];
395: static void InitCrc32Table()
396: {
397: if(crc32Table[1] == 0){
398: // This is the official polynomial used by CRC32 in PKZip.
399: // Often times the polynomial shown reversed as 0x04C11DB7.
400: static const unsigned long dwPolynomial = 0xEDB88320;
401:
402: for(int i = 0; i < 256; i++)
403: {
404: unsigned long dwCrc = i;
405: for(int j = 8; j > 0; j--)
406: {
407: if(dwCrc & 1)
408: dwCrc = (dwCrc >> 1) ^ dwPolynomial;
409: else
410: dwCrc >>= 1;
411: }
412: crc32Table[i] = dwCrc;
413: }
414: }
415: }
416:
417: inline void CalcCrc32(const unsigned char byte, unsigned long &crc32)
418: {
419: crc32 = ((crc32) >> 8) ^ crc32Table[(byte) ^ ((crc32) & 0x000000FF)];
420: }
421:
422:
1.297 moko 423: unsigned long pa_crc32(const char *in, size_t in_size){
1.218 misha 424: unsigned long crc32=0xFFFFFFFF;
1.220 misha 425:
1.240 misha 426: InitCrc32Table();
1.239 misha 427: for(size_t i = 0; i<in_size; i++)
428: CalcCrc32(in[i], crc32);
1.220 misha 429:
1.218 misha 430: return ~crc32;
431: }
432:
1.289 moko 433: static void file_crc32_file_action(struct stat& finfo, int f, const String&, void *context) {
1.218 misha 434: unsigned long& crc32=*static_cast<unsigned long *>(context);
435: if(finfo.st_size) {
436: InitCrc32Table();
1.220 misha 437: int nCount=0;
1.218 misha 438: do {
1.221 misha 439: unsigned char buffer[FILE_BUFFER_SIZE];
440: nCount = file_block_read(f, buffer, sizeof(buffer));
1.220 misha 441: for(int i = 0; i < nCount; i++) CalcCrc32(buffer[i], crc32);
442: } while(nCount > 0);
1.218 misha 443: }
444: }
445:
1.297 moko 446: unsigned long pa_crc32(const String& file_spec){
1.278 moko 447: unsigned long crc32=0xFFFFFFFF;
448: file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32);
449: return ~crc32;
450: }
451:
452: // content-type: xxx; charset=WE-NEED-THIS
453: // content-type: xxx; charset="WE-NEED-THIS"
454: // content-type: xxx; charset="WE-NEED-THIS";
455: Charset* detect_charset(const char* content_type){
456: if(content_type){
1.291 moko 457: char* CONTENT_TYPE=str_upper(content_type);
1.278 moko 458:
459: if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){
460: begin+=8; // skip "CHARSET="
461: char* end=0;
462: if(*begin && (*begin=='"' || *begin =='\'')){
463: char quote=*begin;
464: begin++;
465: end=(char*)strchr(begin, quote);
466: }
467: if(!end)
468: end=(char*)strchr(begin, ';');
469:
470: if(end)
471: *end=0; // terminator
472:
1.295 moko 473: return *begin ? &pa_charsets.get_direct(begin) : 0;
1.278 moko 474: }
475: }
476: return 0;
477: }
478:
1.301 moko 479: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset){
1.302 moko 480: if(!in)
481: return 0;
1.301 moko 482:
1.302 moko 483: String::C sIn(in,strlen(in));
1.301 moko 484:
1.302 moko 485: UTF16* utf16=(UTF16*)pa_malloc_atomic(sIn.length*2+2);
486: UTF16* utf16_end=utf16;
1.301 moko 487:
1.302 moko 488: if(!source_charset.isUTF8())
489: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.301 moko 490:
1.302 moko 491: int status=pa_convertUTF8toUTF16((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf16_end, utf16+sIn.length, strictConversion);
492: if(status != conversionOK)
493: throw Exception("utf-16 encode", new String(in), "utf-16 conversion failed (%d)", status);
1.301 moko 494:
1.302 moko 495: *utf16_end=0;
1.301 moko 496:
1.302 moko 497: return utf16;
1.301 moko 498: }
499:
500: const char* pa_utf16_decode(const UTF16* in, Charset& asked_charset){
501: if(!in)
502: return 0;
503:
504: const UTF16* utf16_start=in;
505: const UTF16* utf16_end;
506:
507: for(utf16_end=in; *utf16_end; utf16_end++);
508:
509: char *result = (char *)pa_malloc_atomic((utf16_end-in)*6+1);
510: char *result_end = result;
511:
512: int status=pa_convertUTF16toUTF8(&utf16_start, utf16_end, (UTF8**)&result_end, (UTF8*)(result+(utf16_end-in)*6), strictConversion);
513:
514: if(status != conversionOK)
515: throw Exception("utf-16 decode", 0, "utf conversion failed (%d)", status);
516:
517: *result_end='\0';
518:
1.302 moko 519: if(asked_charset.isUTF8())
520: return result;
1.301 moko 521:
1.302 moko 522: return Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.301 moko 523: }
1.278 moko 524:
1.283 moko 525: static bool is_latin(const char *in){
526: for(; *in; in++){
527: if ((unsigned char)(*in) > 0x7F)
528: return false;
529: }
530: return true;
531: }
532:
533: #define MAX_IDNA_LENGTH 256
534:
1.301 moko 535: const char *pa_idna_encode(const char *in, Charset& source_charset){
1.283 moko 536: if(!in || is_latin(in))
537: return in;
538:
539: uint32_t utf32[MAX_IDNA_LENGTH];
540: uint32_t *utf32_end=utf32;
541:
542: String::C sIn(in,strlen(in));
543:
544: if(!source_charset.isUTF8())
1.295 moko 545: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.283 moko 546:
547: int status=pa_convertUTF8toUTF32((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf32_end, utf32+MAX_IDNA_LENGTH-1, strictConversion);
548: if(status != conversionOK)
549: throw Exception("idna encode", new String(in), "utf conversion failed (%d)", status);
550:
551: *utf32_end=0;
552:
553: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
554: status=pa_idna_to_ascii_4z(utf32, result, MAX_IDNA_LENGTH, 0);
555: if(status != IDNA_SUCCESS)
556: throw Exception("idna encode", new String(in), "encode failed: %s", pa_idna_strerror(status));
557:
558: return result;
559: }
560:
561: const char *pa_idna_decode(const char *in, Charset &asked_charset){
562: if(!in || !(*in))
563: return in;
564:
565: uint32_t utf32[MAX_IDNA_LENGTH];
566: const uint32_t *utf32_start=utf32;
567: uint32_t *utf32_end;
568:
569: int status=pa_idna_to_unicode_4z(in, utf32, MAX_IDNA_LENGTH, 0);
570: if(status != IDNA_SUCCESS)
571: throw Exception("idna decode", new String(in), "decode failed: %s", pa_idna_strerror(status));
572:
573: for(utf32_end=utf32; *utf32_end; utf32_end++);
574:
575: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
576: char *result_end = result;
577:
578: status=pa_convertUTF32toUTF8(&utf32_start, utf32_end, (UTF8**)&result_end, (UTF8*)(result+MAX_IDNA_LENGTH-1), strictConversion);
579:
580: if(status != conversionOK)
581: throw Exception("idna decode", new String(in), "utf conversion failed (%d)", status);
582:
583: *result_end='\0';
584:
585: if(!asked_charset.isUTF8())
1.295 moko 586: result = (char *)Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.283 moko 587:
588: return result;
589: }
1.292 moko 590: /// must be last in this file
591: #undef vsnprintf
592: int pa_vsnprintf(char* b, size_t s, const char* f, va_list l) {
593: if(!s)
594: return 0;
595:
596: int r;
597: // note: on win32 & maybe somewhere else
598: // vsnprintf do not writes terminating 0 in 'buffer full' case, reducing
599: // http://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010
600: --s;
601:
602: // clients do not check for negative 's', feature: ignore such prints
603: if((ssize_t)s<0)
604: return 0;
605:
606: #ifdef _MSC_VER
607: // win32: if the number of bytes to write exceeds buffer, then count bytes are written and -1 is returned
608: r=_vsnprintf(b, s, f, l);
609: if(r<0)
610: r=s;
611: #else
612: r=vsnprintf(b, s, f, l);
613: /*
614: solaris: man vsnprintf
615:
616: The snprintf() function returns the number of characters
617: formatted, that is, the number of characters that would have
618: been written to the buffer if it were large enough. If the
619: value of n is 0 on a call to snprintf(), an unspecified
620: value less than 1 is returned.
621: */
622:
623: if(r<0)
624: r=0;
625: else if((size_t)r>s)
626: r=s;
627: #endif
628: b[r]=0;
629: return r;
630: }
631:
632: int pa_snprintf(char* b, size_t s, const char* f, ...) {
633: va_list l;
634: va_start(l, f);
635: int r=pa_vsnprintf(b, s, f, l);
636: va_end(l);
637: return r;
638: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_common.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main