Annotation of parser3/src/main/pa_common.C, revision 1.347
1.15 paf 1: /** @file
1.16 paf 2: Parser: commonly functions.
3:
1.343 moko 4: Copyright (c) 2000-2026 Art. Lebedev Studio (https://www.artlebedev.com)
1.327 moko 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
1.306 moko 6: */
1.210 paf 7:
1.1 paf 8: #include "pa_common.h"
1.4 paf 9: #include "pa_exception.h"
1.342 moko 10: #include "pa_inline_hash.h"
1.14 paf 11: #include "pa_globals.h"
1.154 paf 12: #include "pa_charsets.h"
1.223 misha 13: #include "pa_request_charsets.h"
1.347 ! moko 14: #include "pa_file.h"
! 15: #include "pa_stack.h"
1.98 paf 16:
1.283 moko 17: #include "pa_idna.h"
18: #include "pa_convert_utf.h"
19:
1.347 ! moko 20: volatile const char * IDENT_PA_COMMON_C="$Id: pa_common.C,v 1.346 2026/09/25 13:53:43 moko Exp $" IDENT_PA_COMMON_H IDENT_PA_HASH_H IDENT_PA_INLINE_HASH_H IDENT_PA_ARRAY_H IDENT_PA_STACK_H;
1.176 paf 21:
1.154 paf 22: // defines for globals
23:
24: #define FILE_STATUS_NAME "status"
25:
26: // globals
27:
28: const String file_status_name(FILE_STATUS_NAME);
29:
1.305 moko 30: String sql_bind_name(SQL_BIND_NAME);
31: String sql_limit_name(PA_SQL_LIMIT_NAME);
32: String sql_offset_name(PA_SQL_OFFSET_NAME);
33: String sql_default_name(SQL_DEFAULT_NAME);
34: String sql_distinct_name(SQL_DISTINCT_NAME);
35: String sql_value_type_name(SQL_VALUE_TYPE_NAME);
36:
1.278 moko 37: bool capitalized(const char* s){
38: bool upper=true;
39: for(const char* c=s; *c; c++){
40: if(*c != (upper ? toupper((unsigned char)*c) : tolower((unsigned char)*c)))
41: return false;
42: upper=strchr("-_ ", *c) != 0;
43: }
44: return true;
45: }
46:
47: const char* capitalize(const char* s){
48: if(!s || capitalized(s))
49: return s;
50:
51: char* result=pa_strdup(s);
52: if(result){
53: bool upper=true;
54: for(char* c=result; *c; c++){
55: *c=upper ? (char)toupper((unsigned char)*c) : (char)tolower((unsigned char)*c);
56: upper=strchr("-_ ", *c) != 0;
57: }
58: }
59: return (const char*)result;
60: }
61:
1.323 moko 62: char *str_lower(const char *s, size_t length){
63: char *result=pa_strdup(s, length);
1.290 moko 64: for(char* c=result; *c; c++)
65: *c=(char)tolower((unsigned char)*c);
66: return result;
67: }
68:
1.323 moko 69: char *str_upper(const char *s, size_t length){
70: char *result=pa_strdup(s, length);
1.290 moko 71: for(char* c=result; *c; c++)
72: *c=(char)toupper((unsigned char)*c);
73: return result;
74: }
75:
1.278 moko 76: void fix_line_breaks(char *str, size_t& length) {
77: //_asm int 3;
78: const char* const eob=str+length;
79: char* dest=str;
80: // fix DOS: \r\n -> \n
81: // fix Macintosh: \r -> \n
82: char* bol=str;
83: while(char* eol=(char*)memchr(bol, '\r', eob -bol)) {
84: size_t len=eol-bol;
85: if(dest!=bol)
1.347 ! moko 86: memmove(dest, bol, len);
1.278 moko 87: dest+=len;
88: *dest++='\n';
89:
90: if(&eol[1]<eob && eol[1]=='\n') { // \r, \n = DOS
91: bol=eol+2;
92: length--;
93: } else // \r, not \n = Macintosh
94: bol=eol+1;
95: }
96: // last piece without \r
97: if(dest!=bol)
98: memmove(dest, bol, eob-bol);
99: str[length]=0; // terminating
100: }
101:
102: /**
1.347 ! moko 103: scans for @a delim[default \n] in @a *row_ref,
1.278 moko 104: @return piece of line before it or end of string, if no @a delim found
105: assigns @a *row_ref to point right after delimiter if there were one
106: or to zero if no @a delim were found.
107: */
108:
1.126 paf 109: char* getrow(char* *row_ref, char delim) {
1.229 misha 110: char* result=*row_ref;
111: if(result) {
1.126 paf 112: *row_ref=strchr(result, delim);
1.8 paf 113: if(*row_ref)
114: *((*row_ref)++)=0;
115: else if(!*result)
116: return 0;
1.229 misha 117: }
118: return result;
1.8 paf 119: }
120:
1.344 moko 121: const char* strrpbrk(const char* string, const char* chars) {
122: if(string) {
123: const char* pos=string+strlen(string);
124: while(pos!=string) {
125: if(strchr(chars, *--pos))
126: return pos;
127: }
128: }
129: return NULL;
130: }
131:
1.126 paf 132: char* lsplit(char* string, char delim) {
1.229 misha 133: if(string) {
1.344 moko 134: char* v=strchr(string, delim);
1.8 paf 135: if(v) {
136: *v=0;
137: return v+1;
138: }
1.229 misha 139: }
140: return 0;
1.8 paf 141: }
142:
1.126 paf 143: char* lsplit(char* *string_ref, char delim) {
1.229 misha 144: char* result=*string_ref;
1.344 moko 145: char* next=lsplit(*string_ref, delim);
1.229 misha 146: *string_ref=next;
147: return result;
1.9 paf 148: }
149:
1.126 paf 150: char* rsplit(char* string, char delim) {
1.229 misha 151: if(string) {
1.344 moko 152: char* v=strrchr(string, delim);
1.18 paf 153: if(v) {
1.9 paf 154: *v=0;
155: return v+1;
156: }
1.229 misha 157: }
1.314 moko 158: return NULL;
159: }
160:
1.344 moko 161: char* rsplit(char* string, const char* delims) {
162: if(char* v=strrpbrk(string, delims)) {
163: *v=0;
164: return v+1;
165: }
166: return NULL;
167: }
168:
1.325 moko 169: void pa_strncpy(char *dst, const char *src, size_t n){
170: size_t left = n;
171:
172: if (left != 0 && src) {
173: while (--left != 0) {
174: if ((*dst++ = *src++) == '\0')
175: return;
176: }
177: }
178: if (n != 0)
179: *dst = '\0';
180: }
181:
1.314 moko 182: #define STRCAT_STEP(str, len) if(str) { memcpy(ptr, str, len); ptr += len; }
183:
184: char *pa_strcat(const char *a, const char *b, const char *c) {
185: size_t len_a = a ? strlen(a) : 0;
1.315 moko 186: size_t len_b = b ? strlen(b) : 0;
187: size_t len_c = c ? strlen(c) : 0;
1.314 moko 188: char *result=new(PointerFreeGC) char[len_a + len_b + len_c +1/*0*/];
189: char *ptr=result;
190: STRCAT_STEP(a, len_a);
191: STRCAT_STEP(b, len_b);
192: STRCAT_STEP(c, len_c);
193: *ptr='\0';
194: return result;
1.10 paf 195: }
196:
1.229 misha 197:
1.36 paf 198: size_t stdout_write(const void *buf, size_t size) {
1.12 paf 199: #ifdef WIN32
1.187 paf 200: size_t to_write = size;
1.12 paf 201: do{
1.316 moko 202: int chunk_written=fwrite(buf, 1, min((size_t)8*0x400, size), stdout);
1.12 paf 203: if(chunk_written<=0)
204: break;
205: size-=chunk_written;
1.36 paf 206: buf=((const char*)buf)+chunk_written;
1.316 moko 207: } while(size>0);
1.12 paf 208:
1.316 moko 209: fflush(stdout);
1.187 paf 210: return to_write-size;
1.12 paf 211: #else
1.316 moko 212: size_t result=fwrite(buf, 1, size, stdout);
213: fflush(stdout);
214: return result;
1.12 paf 215: #endif
1.2 paf 216: }
1.14 paf 217:
1.229 misha 218: enum EscapeState {
1.341 moko 219: EscapeRest,
220: EscapeFirst,
1.229 misha 221: EscapeSecond,
222: EscapeUnicode
223: };
224:
1.236 misha 225: // @todo prescan for reduce required size (unescaped sting in 1 byte charset requires less memory usually)
1.258 misha 226: char* unescape_chars(const char* cp, int len, Charset* charset, bool js){
1.236 misha 227: char* s=new(PointerFreeGC) char[len+1]; // must be enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes)
1.336 moko 228: XMLByte* dst=(XMLByte *)s;
1.229 misha 229: EscapeState escapeState=EscapeRest;
230: uint escapedValue=0;
231: int srcPos=0;
1.230 misha 232: short int jsCnt=0;
1.236 misha 233: while(srcPos<len){
1.229 misha 234: uchar c=(uchar)cp[srcPos];
1.258 misha 235: if(c=='%' || (c=='\\' && js)){
1.229 misha 236: escapeState=EscapeFirst;
237: } else {
238: switch(escapeState) {
239: case EscapeRest:
1.286 moko 240: if(c=='+' && !js){
1.230 misha 241: *dst++=' ';
1.229 misha 242: } else {
1.230 misha 243: *dst++=c;
1.229 misha 244: }
245: break;
246: case EscapeFirst:
1.232 misha 247: if(charset && c=='u'){
1.229 misha 248: // escaped unicode value: %u0430
249: jsCnt=0;
250: escapedValue=0;
251: escapeState=EscapeUnicode;
252: } else {
1.231 misha 253: if(isxdigit(c)){
1.229 misha 254: escapedValue=hex_value[c] << 4;
255: escapeState=EscapeSecond;
256: } else {
1.230 misha 257: *dst++=c;
1.229 misha 258: escapeState=EscapeRest;
259: }
260: }
261: break;
262: case EscapeSecond:
1.231 misha 263: if(isxdigit(c)){
1.229 misha 264: escapedValue+=hex_value[c];
1.230 misha 265: *dst++=(char)escapedValue;
1.229 misha 266: }
267: escapeState=EscapeRest;
268: break;
269: case EscapeUnicode:
1.231 misha 270: if(isxdigit(c)){
1.229 misha 271: escapedValue=(escapedValue << 4) + hex_value[c];
272: if(++jsCnt==4){
1.230 misha 273: // transcode utf8 char to client charset (we can lost some chars here)
1.336 moko 274: charset->store_Char(dst, (XMLCh)escapedValue, '?');
1.229 misha 275: escapeState=EscapeRest;
276: }
277: } else {
278: // not full unicode value
279: escapeState=EscapeRest;
280: }
281: break;
282: }
283: }
284:
285: srcPos++;
286: }
287:
1.230 misha 288: *dst=0; // zero-termination
1.229 misha 289: return s;
290: }
1.24 paf 291:
1.268 misha 292: char *search_stop(char*& current, char cstop_at) {
293: // sanity check
294: if(!current)
295: return 0;
296:
297: // skip leading WS
298: while(*current==' ' || *current=='\t')
299: current++;
300: if(!*current)
301: return current=0;
302:
303: char *result=current;
304: if(char *pstop_at=strchr(current, cstop_at)) {
305: *pstop_at=0;
306: current=pstop_at+1;
307: } else
308: current=0;
309: return result;
310: }
311:
1.24 paf 312: #ifdef WIN32
1.126 paf 313: void back_slashes_to_slashes(char* s) {
1.24 paf 314: if(s)
315: for(; *s; s++)
316: if(*s=='\\')
1.126 paf 317: *s='/';
1.24 paf 318: }
319: #endif
1.41 paf 320:
1.232 misha 321: size_t strpos(const char *str, const char *substr) {
322: const char *p = strstr(str, substr);
323: return (p==0)?STRING_NOT_FOUND:p-str;
324: }
325:
1.326 moko 326: size_t remove_crlf(char* start, char* end) {
1.226 misha 327: char* from=start;
328: char* to=start;
329: bool skip=false;
330: while(from < end){
331: switch(*from){
332: case '\n':
333: case '\r':
334: case '\t':
335: case ' ':
336: if(!skip){
337: *to=' ';
338: to++;
339: skip=true;
340: }
341: break;
342: default:
343: if(from != to)
344: *to=*from;
345: to++;
346: skip=false;
1.69 parser 347: }
1.226 misha 348: from++;
349: }
350: return to-start;
1.91 paf 351: }
352:
1.279 moko 353: const char* hex_digits="0123456789ABCDEF";
354:
1.278 moko 355: const char* hex_string(unsigned char* bytes, size_t size, bool upcase) {
356: char *bytes_hex=new(PointerFreeGC) char [size*2/*byte->hh*/+1/*for zero-teminator*/];
357: unsigned char *src=bytes;
358: unsigned char *end=bytes+size;
359: char *dest=bytes_hex;
360:
1.279 moko 361: const char *hex=upcase? hex_digits : "0123456789abcdef";
1.278 moko 362:
363: for(; src<end; src++) {
364: *dest++=hex[*src/0x10];
365: *dest++=hex[*src%0x10];
366: }
367: *dest=0;
368:
369: return bytes_hex;
370: }
1.91 paf 371:
1.321 moko 372: ssize_t file_block_read(const int f, void* buffer, const size_t size){
373: ssize_t nCount = read(f, buffer, size);
1.221 misha 374: if (nCount < 0)
1.289 moko 375: throw Exception("file.read", 0, "read failed: %s (%d)", strerror(errno), errno);
1.221 misha 376: return nCount;
377: }
378:
1.278 moko 379: static unsigned long crc32Table[256];
380: static void InitCrc32Table()
381: {
382: if(crc32Table[1] == 0){
383: // This is the official polynomial used by CRC32 in PKZip.
384: // Often times the polynomial shown reversed as 0x04C11DB7.
385: static const unsigned long dwPolynomial = 0xEDB88320;
386:
387: for(int i = 0; i < 256; i++)
388: {
389: unsigned long dwCrc = i;
390: for(int j = 8; j > 0; j--)
391: {
392: if(dwCrc & 1)
393: dwCrc = (dwCrc >> 1) ^ dwPolynomial;
394: else
395: dwCrc >>= 1;
396: }
397: crc32Table[i] = dwCrc;
398: }
399: }
400: }
401:
402: inline void CalcCrc32(const unsigned char byte, unsigned long &crc32)
403: {
404: crc32 = ((crc32) >> 8) ^ crc32Table[(byte) ^ ((crc32) & 0x000000FF)];
405: }
406:
407:
1.297 moko 408: unsigned long pa_crc32(const char *in, size_t in_size){
1.218 misha 409: unsigned long crc32=0xFFFFFFFF;
1.220 misha 410:
1.240 misha 411: InitCrc32Table();
1.239 misha 412: for(size_t i = 0; i<in_size; i++)
413: CalcCrc32(in[i], crc32);
1.220 misha 414:
1.218 misha 415: return ~crc32;
416: }
417:
1.289 moko 418: static void file_crc32_file_action(struct stat& finfo, int f, const String&, void *context) {
1.218 misha 419: unsigned long& crc32=*static_cast<unsigned long *>(context);
420: if(finfo.st_size) {
421: InitCrc32Table();
1.220 misha 422: int nCount=0;
1.218 misha 423: do {
1.221 misha 424: unsigned char buffer[FILE_BUFFER_SIZE];
425: nCount = file_block_read(f, buffer, sizeof(buffer));
1.220 misha 426: for(int i = 0; i < nCount; i++) CalcCrc32(buffer[i], crc32);
427: } while(nCount > 0);
1.218 misha 428: }
429: }
430:
1.297 moko 431: unsigned long pa_crc32(const String& file_spec){
1.278 moko 432: unsigned long crc32=0xFFFFFFFF;
433: file_read_action_under_lock(file_spec, "crc32", file_crc32_file_action, &crc32);
434: return ~crc32;
435: }
436:
437: // content-type: xxx; charset=WE-NEED-THIS
438: // content-type: xxx; charset="WE-NEED-THIS"
439: // content-type: xxx; charset="WE-NEED-THIS";
440: Charset* detect_charset(const char* content_type){
441: if(content_type){
1.291 moko 442: char* CONTENT_TYPE=str_upper(content_type);
1.278 moko 443:
444: if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){
445: begin+=8; // skip "CHARSET="
446: char* end=0;
447: if(*begin && (*begin=='"' || *begin =='\'')){
448: char quote=*begin;
449: begin++;
450: end=(char*)strchr(begin, quote);
451: }
452: if(!end)
453: end=(char*)strchr(begin, ';');
454:
455: if(end)
456: *end=0; // terminator
457:
1.295 moko 458: return *begin ? &pa_charsets.get_direct(begin) : 0;
1.278 moko 459: }
460: }
461: return 0;
462: }
463:
1.301 moko 464: const UTF16* pa_utf16_encode(const char* in, Charset& source_charset){
1.302 moko 465: if(!in)
466: return 0;
1.301 moko 467:
1.302 moko 468: String::C sIn(in,strlen(in));
1.301 moko 469:
1.302 moko 470: UTF16* utf16=(UTF16*)pa_malloc_atomic(sIn.length*2+2);
471: UTF16* utf16_end=utf16;
1.301 moko 472:
1.302 moko 473: if(!source_charset.isUTF8())
474: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.301 moko 475:
1.302 moko 476: int status=pa_convertUTF8toUTF16((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf16_end, utf16+sIn.length, strictConversion);
477: if(status != conversionOK)
478: throw Exception("utf-16 encode", new String(in), "utf-16 conversion failed (%d)", status);
1.301 moko 479:
1.302 moko 480: *utf16_end=0;
1.301 moko 481:
1.302 moko 482: return utf16;
1.301 moko 483: }
484:
485: const char* pa_utf16_decode(const UTF16* in, Charset& asked_charset){
486: if(!in)
487: return 0;
488:
489: const UTF16* utf16_start=in;
490: const UTF16* utf16_end;
491:
492: for(utf16_end=in; *utf16_end; utf16_end++);
493:
494: char *result = (char *)pa_malloc_atomic((utf16_end-in)*6+1);
495: char *result_end = result;
496:
497: int status=pa_convertUTF16toUTF8(&utf16_start, utf16_end, (UTF8**)&result_end, (UTF8*)(result+(utf16_end-in)*6), strictConversion);
498:
499: if(status != conversionOK)
500: throw Exception("utf-16 decode", 0, "utf conversion failed (%d)", status);
501:
502: *result_end='\0';
503:
1.302 moko 504: if(asked_charset.isUTF8())
505: return result;
1.301 moko 506:
1.302 moko 507: return Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.301 moko 508: }
1.278 moko 509:
1.283 moko 510: static bool is_latin(const char *in){
511: for(; *in; in++){
512: if ((unsigned char)(*in) > 0x7F)
513: return false;
514: }
515: return true;
516: }
517:
518: #define MAX_IDNA_LENGTH 256
519:
1.301 moko 520: const char *pa_idna_encode(const char *in, Charset& source_charset){
1.283 moko 521: if(!in || is_latin(in))
522: return in;
523:
524: uint32_t utf32[MAX_IDNA_LENGTH];
525: uint32_t *utf32_end=utf32;
526:
527: String::C sIn(in,strlen(in));
528:
529: if(!source_charset.isUTF8())
1.295 moko 530: sIn=Charset::transcode(sIn, source_charset, pa_UTF8_charset);
1.283 moko 531:
532: int status=pa_convertUTF8toUTF32((const UTF8**)&sIn.str, (const UTF8*)(sIn.str+sIn.length), &utf32_end, utf32+MAX_IDNA_LENGTH-1, strictConversion);
533: if(status != conversionOK)
534: throw Exception("idna encode", new String(in), "utf conversion failed (%d)", status);
535:
536: *utf32_end=0;
537:
538: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
539: status=pa_idna_to_ascii_4z(utf32, result, MAX_IDNA_LENGTH, 0);
540: if(status != IDNA_SUCCESS)
541: throw Exception("idna encode", new String(in), "encode failed: %s", pa_idna_strerror(status));
542:
543: return result;
544: }
545:
546: const char *pa_idna_decode(const char *in, Charset &asked_charset){
547: if(!in || !(*in))
548: return in;
549:
550: uint32_t utf32[MAX_IDNA_LENGTH];
551: const uint32_t *utf32_start=utf32;
552: uint32_t *utf32_end;
553:
554: int status=pa_idna_to_unicode_4z(in, utf32, MAX_IDNA_LENGTH, 0);
555: if(status != IDNA_SUCCESS)
556: throw Exception("idna decode", new String(in), "decode failed: %s", pa_idna_strerror(status));
557:
558: for(utf32_end=utf32; *utf32_end; utf32_end++);
559:
560: char *result = (char *)pa_malloc(MAX_IDNA_LENGTH);
561: char *result_end = result;
562:
563: status=pa_convertUTF32toUTF8(&utf32_start, utf32_end, (UTF8**)&result_end, (UTF8*)(result+MAX_IDNA_LENGTH-1), strictConversion);
564:
565: if(status != conversionOK)
566: throw Exception("idna decode", new String(in), "utf conversion failed (%d)", status);
567:
568: *result_end='\0';
569:
570: if(!asked_charset.isUTF8())
1.295 moko 571: result = (char *)Charset::transcode(result, pa_UTF8_charset, asked_charset).cstr();
1.283 moko 572:
573: return result;
574: }
1.292 moko 575: /// must be last in this file
576: #undef vsnprintf
577: int pa_vsnprintf(char* b, size_t s, const char* f, va_list l) {
578: if(!s)
579: return 0;
580:
581: int r;
582: // note: on win32 & maybe somewhere else
583: // vsnprintf do not writes terminating 0 in 'buffer full' case, reducing
584: // http://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010
585: --s;
586:
587: // clients do not check for negative 's', feature: ignore such prints
588: if((ssize_t)s<0)
589: return 0;
590:
591: #ifdef _MSC_VER
592: // win32: if the number of bytes to write exceeds buffer, then count bytes are written and -1 is returned
593: r=_vsnprintf(b, s, f, l);
594: if(r<0)
595: r=s;
596: #else
597: r=vsnprintf(b, s, f, l);
598: /*
599: solaris: man vsnprintf
600:
601: The snprintf() function returns the number of characters
602: formatted, that is, the number of characters that would have
603: been written to the buffer if it were large enough. If the
604: value of n is 0 on a call to snprintf(), an unspecified
605: value less than 1 is returned.
606: */
607:
608: if(r<0)
609: r=0;
610: else if((size_t)r>s)
611: r=s;
612: #endif
613: b[r]=0;
614: return r;
615: }
616:
617: int pa_snprintf(char* b, size_t s, const char* f, ...) {
618: va_list l;
619: va_start(l, f);
620: int r=pa_vsnprintf(b, s, f, l);
621: va_end(l);
622: return r;
623: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_common.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main