Annotation of parser3/src/main/pa_charset.C, revision 1.98
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.90 moko 4: Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.96 moko 11: // we are using some pcre_internal.h stuff as well
12: #include "../lib/pcre/pa_pcre_internal.h"
13:
1.98 ! moko 14: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.97 2015/04/06 22:27:26 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 15:
1.1 paf 16: #ifdef XML
1.8 paf 17: #include "libxml/encoding.h"
1.1 paf 18: #endif
19:
1.46 paf 20: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 21:
22: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 23: #define PRECALCULATE_DEST_LENGTH
1.46 paf 24:
1.38 paf 25: // globals
26:
27: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
28: #include "utf8-to-upper.inc"
29: };
30: Charset::UTF8CaseTable UTF8CaseToUpper={
31: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
32: UTF8CaseToUpperRecords};
33:
34: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
35: #include "utf8-to-lower.inc"
36: };
37: Charset::UTF8CaseTable UTF8CaseToLower={
38: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
39: UTF8CaseToLowerRecords};
40:
1.1 paf 41: // helpers
42:
43: inline void prepare_case_tables(unsigned char *tables) {
44: unsigned char *lcc_table=tables+lcc_offset;
45: unsigned char *fcc_table=tables+fcc_offset;
46: for(int i=0; i<0x100; i++)
1.53 paf 47: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 48: }
49: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
50: unsigned char bit) {
51: unsigned char *ctypes_table=tables+ctypes_offset;
52: ctypes_table[0]=bit;
53: for(; *cstr; cstr++) {
54: unsigned char c=*cstr;
55: ctypes_table[c]|=bit;
56: }
57: }
1.35 paf 58: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 59: if(!cstr || !*cstr)
60: return 0;
61: if(cstr[1]==0)
1.4 paf 62: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 63:
1.91 moko 64: return pa_atoui(cstr,0);
1.1 paf 65: }
1.35 paf 66: inline bool to_bool(const char* cstr) {
1.1 paf 67: return cstr && *cstr!=0;
68: }
69: static void element2ctypes(unsigned char c, bool belongs,
70: unsigned char *tables, unsigned char bit, int group_offset=-1) {
71: if(!belongs)
72: return;
73:
74: unsigned char *ctypes_table=tables+ctypes_offset;
75:
76: ctypes_table[c]|=bit;
77: if(group_offset>=0)
1.4 paf 78: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 79: }
80: static void element2case(unsigned char from, unsigned char to,
81: unsigned char *tables) {
82: if(!to)
83: return;
84:
85: unsigned char *lcc_table=tables+lcc_offset;
86: unsigned char *fcc_table=tables+fcc_offset;
87: lcc_table[from]=to;
88: fcc_table[from]=to; fcc_table[to]=from;
89: }
90:
1.95 moko 91: inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
1.93 moko 92: if(prefix) {
1.95 moko 93: strcpy((char *)dest, prefix);
1.93 moko 94: dest+=strlen(prefix);
95: }
96: *dest++=hex_digits[c >> 4];
97: *dest++=hex_digits[c & 0x0F];
1.95 moko 98: return dest;
1.93 moko 99: }
100:
1.95 moko 101: inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
1.93 moko 102: if(prefix) {
1.95 moko 103: strcpy((char *)dest, prefix);
1.93 moko 104: dest+=strlen(prefix);
105: }
106: *dest++=hex_digits[(c >> 12) & 0x0F];
107: *dest++=hex_digits[(c >> 8) & 0x0F];
108: *dest++=hex_digits[(c >> 4) & 0x0F];
109: *dest++=hex_digits[(c) & 0x0F];
1.95 moko 110: return dest;
1.93 moko 111: }
112:
1.1 paf 113: // methods
114:
1.37 paf 115: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 116: FNAME(ANAME),
117: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 118:
1.35 paf 119: if(afile_spec) {
1.1 paf 120: fisUTF8=false;
1.35 paf 121: load_definition(*charsets, *afile_spec);
1.1 paf 122: #ifdef XML
1.35 paf 123: addEncoding(FNAME_CSTR);
1.1 paf 124: #endif
125: } else {
126: fisUTF8=true;
1.4 paf 127: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.96 moko 128: memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
1.1 paf 129: }
130:
131: #ifdef XML
1.35 paf 132: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 133: #endif
134: }
135:
1.35 paf 136: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 137: // pcre_tables
138: // lowcase, flipcase, bits digit+word+whitespace, masks
139:
140: // must not move this inside of prepare_case_tables
141: // don't know the size there
142: memset(pcre_tables, 0, sizeof(pcre_tables));
143: prepare_case_tables(pcre_tables);
1.4 paf 144: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 145:
146: // charset
1.35 paf 147: memset(&tables, 0, sizeof(tables));
1.1 paf 148:
149: // loading text
1.35 paf 150: char *data=file_read_text(charsets, afile_spec);
1.1 paf 151:
152: // ignore header
153: getrow(&data);
154:
155: // parse cells
156: char *row;
1.42 paf 157: while((row=getrow(&data))) {
1.1 paf 158: // remove empty&comment lines
159: if(!*row || *row=='#')
160: continue;
161:
162: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 163: unsigned char c=0;
1.1 paf 164: char *cell;
1.42 paf 165: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 166: switch(column) {
1.53 paf 167: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 168: // pcre_tables
169: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
170: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
171: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
172: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
173: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 174: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 175: case 7:
176: case 8:
177: // charset
1.10 paf 178: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 179: throw Exception(PARSER_RUNTIME,
1.35 paf 180: &afile_spec,
1.1 paf 181: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
182:
183: XMLCh unicode=(XMLCh)to_wchar_code(cell);
184: if(!unicode && column==7/*unicode1 column*/)
185: unicode=(XMLCh)c;
186: if(unicode) {
1.10 paf 187: if(!tables.fromTable[c])
188: tables.fromTable[c]=unicode;
189: tables.toTable[tables.toTableSize].intCh=unicode;
190: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
191: tables.toTableSize++;
1.1 paf 192: }
193: break;
194: }
195: }
196: };
197:
1.87 moko 198: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
199: for(uint i=0; i<0x20; i++)
200: if(!tables.fromTable[i]){
201: tables.fromTable[i]=i;
202: tables.toTable[tables.toTableSize].intCh=i;
203: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
204: tables.toTableSize++;
205: }
206:
1.1 paf 207: // sort by the Unicode code point
208: sort_ToTable();
209: }
210:
211: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
212: return
1.38 paf 213: static_cast<const Charset::Tables::Rec *>(a)->intCh-
214: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 215: }
216:
217: void Charset::sort_ToTable() {
1.92 moko 218: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 219: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 220: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 221: //fclose(f);
222: }
223:
1.60 misha 224: // @todo: precache for spedup searching
1.10 paf 225: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 226: const Charset::Tables& tables,
227: XMLByte not_found) {
1.80 misha 228: int lo = 0;
229: int hi = tables.toTableSize - 1;
1.39 paf 230: while(lo<=hi) {
1.35 paf 231: // Calc the mid point of the low and high offset.
1.39 paf 232: const unsigned int i = (lo + hi) / 2;
233:
234: XMLCh cur=tables.toTable[i].intCh;
235: if(toXlat==cur)
236: return tables.toTable[i].extCh;
237: if(toXlat>cur)
238: lo = i+1;
1.1 paf 239: else
1.39 paf 240: hi = i-1;
241: }
1.35 paf 242:
243: return not_found;
1.1 paf 244: }
245:
1.35 paf 246: String::C Charset::transcode(const String::C src,
247: const Charset& source_charset,
248: const Charset& dest_charset) {
249: if(!src.length)
250: return String::C("", 0);
1.4 paf 251:
1.1 paf 252: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
253: default: // 0x00
1.35 paf 254: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 255: case 0x01:
1.35 paf 256: return source_charset.transcodeToUTF8(src);
1.1 paf 257: case 0x10:
1.35 paf 258: return dest_charset.transcodeFromUTF8(src);
1.1 paf 259: case 0x11:
1.35 paf 260: return src;
1.1 paf 261: }
262: }
263:
264: // ---------------------------------------------------------------------------
265: // Local static data
266: //
267: // gUTFBytes
268: // A list of counts of trailing bytes for each initial byte in the input.
269: //
270: // gUTFOffsets
271: // A list of values to offset each result char type, according to how
272: // many source bytes when into making it.
273: //
274: // gFirstByteMark
275: // A list of values to mask onto the first byte of an encoded sequence,
276: // indexed by the number of bytes used to create the sequence.
277: // ---------------------------------------------------------------------------
278: static const XMLByte gUTFBytes[0x100] = {
279: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
280: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
281: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
282: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
283: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
284: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
285: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
287: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
288: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
289: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
290: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
291: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
292: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
293: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
294: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
295: };
296:
297: static const uint gUTFOffsets[6] = {
1.80 misha 298: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 299: };
300:
301: static const XMLByte gFirstByteMark[7] = {
1.80 misha 302: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 303: };
304:
1.71 misha 305: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
306: XMLByte *toFill, int& toFillLen,
307: const Charset::Tables& tables) {
1.11 paf 308: const XMLByte* srcPtr=srcData;
309: const XMLByte* srcEnd=srcData+srcLen;
310: XMLByte* outPtr=toFill;
311: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 312:
1.35 paf 313: while(srcPtr<srcEnd) {
314: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 315: if(!curVal) {
1.35 paf 316: // use the replacement character
317: *outPtr++= '?';
318: srcPtr++;
319: continue;
320: }
1.1 paf 321:
1.35 paf 322: // Figure out how many bytes we need
323: unsigned int encodedBytes;
324: if(curVal<0x80)
325: encodedBytes = 1;
326: else if(curVal<0x800)
327: encodedBytes = 2;
328: else if(curVal<0x10000)
329: encodedBytes = 3;
330: else if(curVal<0x200000)
331: encodedBytes = 4;
332: else if(curVal<0x4000000)
333: encodedBytes = 5;
334: else if(curVal<= 0x7FFFFFFF)
335: encodedBytes = 6;
336: else {
337: // use the replacement character
338: *outPtr++= '?';
339: srcPtr++;
340: continue;
341: }
1.11 paf 342:
1.35 paf 343: // If we cannot fully get this char into the output buffer
344: if (outPtr + encodedBytes > outEnd)
345: break;
346:
347: // We can do it, so update the source index
348: srcPtr++;
349:
350: // And spit out the bytes. We spit them out in reverse order
351: // here, so bump up the output pointer and work down as we go.
352: outPtr+= encodedBytes;
353: switch(encodedBytes) {
1.60 misha 354: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
355: curVal>>= 6;
356: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
357: curVal>>= 6;
358: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
359: curVal>>= 6;
360: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
361: curVal>>= 6;
362: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
363: curVal>>= 6;
364: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 365: }
366:
367: // Add the encoded bytes back in again to indicate we've eaten them
368: outPtr+= encodedBytes;
369: }
370:
371: // Update the bytes eaten
372: srcLen = srcPtr - srcData;
373:
374: // Return the characters read
375: toFillLen = outPtr - toFill;
376:
1.29 paf 377: //return srcPtr==srcEnd?(int)toFillLen:-1;
378: /*
379: xmlCharEncodingInputFunc
380: Returns :
381: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
382: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
383: of ocetes consumed.
384: */
385: return 0;
1.1 paf 386: }
1.26 paf 387: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 388: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
389: XMLByte* toFill, int& toFillLen,
390: const Charset::Tables& tables) {
1.11 paf 391: const XMLByte* srcPtr=srcData;
392: const XMLByte* srcEnd=srcData+srcLen;
393: XMLByte* outPtr=toFill;
394: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 395:
1.35 paf 396: // We now loop until we either run out of input data, or room to store
397: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
398: // Get the next leading byte out
399: const XMLByte firstByte =* srcPtr;
400:
401: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 402: if(firstByte<=127) {
1.35 paf 403: *outPtr++= firstByte;
404: srcPtr++;
405: continue;
406: }
407:
408: // See how many trailing src bytes this sequence is going to require
409: const unsigned int trailingBytes = gUTFBytes[firstByte];
410:
411: // If there are not enough source bytes to do this one, then we
412: // are done. Note that we done>= here because we are implicitly
413: // counting the 1 byte we get no matter what.
414: if(srcPtr+trailingBytes>= srcEnd)
415: break;
416:
417: // Looks ok, so lets build up the value
418: uint tmpVal=0;
419: switch(trailingBytes) {
420: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
421: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
422: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
423: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
424: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
425: case 0: tmpVal+=*srcPtr++;
426: break;
427:
428: default:
429: throw Exception(0,
430: 0,
1.49 paf 431: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 432: }
433: tmpVal-=gUTFOffsets[trailingBytes];
434:
435: // If it will fit into a single char, then put it in. Otherwise
436: // fail [*encode it as a surrogate pair. If its not valid, use the
437: // replacement char.*]
438: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 439: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
440: *outPtr++=xlat;
1.49 paf 441: else {
1.50 paf 442: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 443: }
444: } else {
445: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
446: for(uint i=0; i<=trailingBytes; i++)
447: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
448: }
1.1 paf 449: }
1.35 paf 450:
451: // Update the bytes eaten
452: srcLen = srcPtr - srcData;
453:
454: // Return the characters read
455: toFillLen = outPtr - toFill;
1.11 paf 456:
1.29 paf 457: //return srcPtr==srcEnd?(int)toFillLen:-1;
458: /*
459: xmlCharEncodingOutputFunc
460: Returns :
461: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
462: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
463: of ocetes consumed.
464: */
465: return 0;
1.10 paf 466: }
467:
1.85 misha 468: static bool need_escape(XMLByte c){
1.60 misha 469: return
1.66 misha 470: !(
471: (c<=127)
472: && (
1.89 misha 473: pa_isalnum((unsigned char)c)
1.66 misha 474: || strchr("*@-_+./", c)!=0
475: )
476: );
1.60 misha 477: }
478:
1.70 misha 479: // read one UTF8 char and return length of this char (in bytes)
480: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 481: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
482: return 0;
483:
484: firstByte=*srcPtr;
485:
486: if(firstByte<=127){
487: UTF8Char=firstByte;
488: srcPtr++;
489: return 1;
490: }
491:
492: unsigned int trailingBytes=gUTFBytes[firstByte];
493:
494: if(srcPtr+trailingBytes>=srcEnd){
495: return 0; // not enough bytes in source string for reading
496: }
497:
498: uint tmpVal=0;
499: switch(trailingBytes){
500: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
501: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
502: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
503: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
504: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
505: case 0: tmpVal+=*srcPtr++;
506: }
507:
508: tmpVal-=gUTFOffsets[trailingBytes];
509: UTF8Char=tmpVal;
510:
511: return trailingBytes+1;
512: }
513:
1.70 misha 514: // skip UTF8 char and return length of this char (in bytes)
515: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 516: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
517: return 0;
518:
1.63 misha 519: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
520: srcPtr+=trailingBytes;
1.62 misha 521:
522: return trailingBytes;
1.61 misha 523: }
524:
1.85 misha 525: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 526: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 527: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
528: return 0;
529:
530: firstByte=*srcPtr++;
531: UTF8Char=tables.fromTable[firstByte];
532:
533: if(UTF8Char<0x80)
534: return 1;
535: else if(UTF8Char<0x800)
536: return 2;
537: else if(UTF8Char<0x10000)
538: return 3;
539: else if(UTF8Char<0x200000)
540: return 4;
541: else if(UTF8Char<0x4000000)
542: return 5;
543: else if(UTF8Char<= 0x7FFFFFFF)
544: return 6;
545:
546: // will use the replacement character '?'
547: firstByte=0;
548: return 1;
549: }
550:
1.85 misha 551: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
552: size_t dest_length=0;
553:
554: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
555: if(i.getCharSize()==1)
556: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
557: else
558: dest_length+=6; // %uXXXX
1.60 misha 559: }
560:
1.85 misha 561: return dest_length;
1.60 misha 562: }
563:
1.86 moko 564: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
565: const XMLByte* src_end=src+src_length;
566: XMLByte first_byte;
567: XMLCh UTF8_char;
1.85 misha 568: size_t dest_length=0;
569:
1.86 moko 570: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 571: if(char_size==1)
572: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
573: else
574: dest_length+=6; // %uXXXX
1.60 misha 575: }
576:
1.85 misha 577: return dest_length;
578: }
579:
580: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 581: if(!src.length)
1.85 misha 582: return 0;
583:
584: #ifdef PRECALCULATE_DEST_LENGTH
585: if(source_charset.isUTF8())
1.86 moko 586: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 587: else
1.86 moko 588: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 589: #else
590: return src_length*6; // enough for %uXXXX but too memory-hungry
591: #endif
592: }
593:
594: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
595: if(char_size==1) \
596: if(first_byte){ \
597: if(need_escape(first_byte)) \
1.95 moko 598: dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 599: else \
600: *dest_ptr++=first_byte; /*as is*/ \
601: } else \
602: *dest_ptr++='?'; /* replacement char '?' */ \
603: else \
1.95 moko 604: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 605:
606:
607: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
608: XMLByte* dest_ptr=dest;
609:
610: // loop until we either run out of input data
611: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
612: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 613:
1.85 misha 614: return dest_ptr - dest;
1.60 misha 615: }
616:
1.85 misha 617: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
618: const XMLByte* src_end=src+src_length;
619: XMLByte* dest_ptr=dest;
620:
621: XMLByte first_byte;
622: XMLCh UTF8_char;
623: uint char_size;
624:
1.86 moko 625: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 626: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
627:
628: return dest_ptr - dest;
629: }
1.60 misha 630:
631: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 632: if(!src.length)
1.60 misha 633: return String::C("", 0);
634:
1.85 misha 635: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
636: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
637:
638: size_t dest_length;
639: if(source_charset.isUTF8())
1.86 moko 640: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 641: else
1.86 moko 642: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 643:
644: if(dest_length>dest_calculated_length)
645: throw Exception(0, 0, "Charset::escape buffer overflow");
646:
647: dest_body[dest_length]=0; // terminator
648: return String::C((char*)dest_body, dest_length);
649: }
650:
651: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 652: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 653: return String::Body(dest.length ? dest.str:0);
654: }
655:
656: String& Charset::escape(const String& src, const Charset& source_charset) {
657: if(src.is_empty())
658: return *new String();
659:
660: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
661: }
662:
663: inline bool need_json_escape(unsigned char c){
664: return strchr("\n\"\\/\t\r\b\f", c)!=0;
665: }
666:
667: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
668: size_t dest_length=0;
669:
670: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 moko 671: if(i.getCharSize()==1){
672: XMLByte first_byte=i.getFirstByte();
673: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
674: } else
1.85 misha 675: dest_length+=6; // \uXXXX
676: }
677:
678: return dest_length;
679: }
680:
1.86 moko 681: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
682: const XMLByte* src_end=src+src_length;
1.85 misha 683: XMLByte first_byte;
684: XMLCh UTF8_char;
1.60 misha 685: size_t dest_length=0;
686:
1.86 moko 687: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 688: if(char_size==1)
1.93 moko 689: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 690: else
691: dest_length+=6; // \uXXXX
1.60 misha 692: }
1.85 misha 693:
694: return dest_length;
695: }
696:
697: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 698: if(!src.length)
1.85 misha 699: return 0;
700:
701: #ifdef PRECALCULATE_DEST_LENGTH
702: if(source_charset.isUTF8())
1.86 moko 703: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 704: else
1.86 moko 705: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 706: #else
1.85 misha 707: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 708: #endif
1.85 misha 709: }
710:
711: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
712: if(char_size==1) \
713: switch(first_byte){ \
714: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
715: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
716: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
717: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
718: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
719: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
720: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
721: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
722: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.95 moko 723: default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
1.93 moko 724: else *dest_ptr++=first_byte; \
1.85 misha 725: } \
726: else \
1.95 moko 727: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 728:
729:
730: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
731: XMLByte* dest_ptr=dest;
732:
733: // loop until we either run out of input data
734: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
735: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
736:
737: return dest_ptr - dest;
738: }
739:
740: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
741: const XMLByte* src_end=src+src_length;
742: XMLByte* dest_ptr=dest;
743:
744: XMLByte first_byte;
745: XMLCh UTF8_char;
746: uint char_size;
747:
1.86 moko 748: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 749: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
750:
751: return dest_ptr - dest;
752: }
1.60 misha 753:
1.85 misha 754: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 755: if(!src.length)
1.85 misha 756: return String::C("", 0);
1.60 misha 757:
1.85 misha 758: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
759: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
760:
761: size_t dest_length;
762: if(source_charset.isUTF8())
1.86 moko 763: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 764: else
1.86 moko 765: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 766:
1.85 misha 767: if(dest_length>dest_calculated_length)
768: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 769:
770: dest_body[dest_length]=0; // terminator
771: return String::C((char*)dest_body, dest_length);
772: }
1.85 misha 773:
774: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 775: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 776: return String::Body(dest.length ? dest.str:0);
1.64 misha 777: }
778:
1.85 misha 779: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 780: if(src.is_empty())
1.73 misha 781: return *new String();
1.64 misha 782:
1.85 misha 783: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 784: }
1.60 misha 785:
1.35 paf 786: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 787: int src_length=src.length;
1.60 misha 788:
789: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 790: int dest_length=0;
1.60 misha 791: const XMLByte* srcPtr=(XMLByte*)src.str;
792: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 793: XMLByte firstByte;
794: XMLCh UTF8Char;
795: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 796: dest_length+=charSize;
797: #else
1.85 misha 798: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 799: #endif
800:
1.35 paf 801: #ifndef NDEBUG
1.71 misha 802: int saved_dest_length=dest_length;
1.35 paf 803: #endif
804: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 805:
806: if(::transcodeToUTF8(
1.35 paf 807: (XMLByte *)src.str, src_length,
808: dest_body, dest_length,
1.11 paf 809: tables)<0)
1.43 paf 810: throw Exception(0,
1.10 paf 811: 0,
1.11 paf 812: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 813:
1.60 misha 814: assert(dest_length<=saved_dest_length);
815: dest_body[dest_length]=0; // terminator
1.35 paf 816: return String::C((char*)dest_body, dest_length);
1.10 paf 817: }
1.38 paf 818:
819: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 820: int lo = 0;
821: int hi = table.size - 1;
1.39 paf 822: while(lo<=hi) {
1.38 paf 823: // Calc the mid point of the low and high offset.
1.39 paf 824: const unsigned int i = (lo + hi) / 2;
825:
826: XMLCh cur=table.records[i].from;
827: if(src==cur)
828: return table.records[i].to;
829: if(src>cur)
830: lo = i+1;
1.38 paf 831: else
1.39 paf 832: hi = i-1;
833: }
834:
835: // not found
1.38 paf 836: return src;
837: }
838:
1.58 misha 839: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 840: if(!src) {
841: // use the replacement character
842: *outPtr++= '?';
843: return;
844: }
845:
846: // Figure out how many bytes we need
847: unsigned int encodedBytes;
848: if(src<0x80)
849: encodedBytes = 1;
850: else if(src<0x800)
851: encodedBytes = 2;
852: else if(src<0x10000)
853: encodedBytes = 3;
854: else if(src<0x200000)
855: encodedBytes = 4;
856: else if(src<0x4000000)
857: encodedBytes = 5;
858: else if(src<= 0x7FFFFFFF)
859: encodedBytes = 6;
860: else {
861: // use the replacement character
862: *outPtr++= '?';
863: return;
864: }
865:
866: // And spit out the bytes. We spit them out in reverse order
867: // here, so bump up the output pointer and work down as we go.
868: outPtr+= encodedBytes;
869: switch(encodedBytes) {
870: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
871: src>>= 6;
872: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
873: src>>= 6;
874: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
875: src>>= 6;
876: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
877: src>>= 6;
878: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
879: src>>= 6;
880: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
881: }
882:
883: // Add the encoded bytes back in again to indicate we've eaten them
884: outPtr+= encodedBytes;
885: }
886:
887: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
888: const Charset::UTF8CaseTable& table) {
889: store_UTF8(change_case_UTF8(src, table), outPtr);
1.98 ! moko 890: }
! 891:
1.44 paf 892: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
893: XMLByte* toFill, size_t toFillLen,
894: const Charset::UTF8CaseTable& table) {
1.38 paf 895: const XMLByte* srcPtr=srcData;
1.44 paf 896: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 897: XMLByte* outPtr=toFill;
1.44 paf 898: XMLByte* outEnd=toFill+toFillLen;
899:
900: // We now loop until we either run out of input data, or room to store
901: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
902: // Get the next leading byte out
903: const XMLByte firstByte =* srcPtr;
1.38 paf 904:
1.60 misha 905: if(firstByte<=127) {
1.38 paf 906: change_case_UTF8(firstByte, outPtr, table);
907: srcPtr++;
908: continue;
909: }
910:
911: // See how many trailing src bytes this sequence is going to require
912: const unsigned int trailingBytes = gUTFBytes[firstByte];
913:
914: // Looks ok, so lets build up the value
915: uint tmpVal=0;
916: switch(trailingBytes) {
917: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
918: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
919: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
920: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
921: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
922: case 0: tmpVal+=*srcPtr++;
923: break;
924:
925: default:
926: throw Exception(0,
927: 0,
928: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
929: }
930: tmpVal-=gUTFOffsets[trailingBytes];
931:
932: // If it will fit into a single char, then put it in. Otherwise
933: // fail [*encode it as a surrogate pair. If its not valid, use the
934: // replacement char.*]
935: if(!(tmpVal & 0xFFFF0000))
936: change_case_UTF8(tmpVal, outPtr, table);
937: else
938: throw Exception(0,
939: 0,
940: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
941: }
942:
943: if(srcPtr!=outPtr)
944: throw Exception(0,
945: 0,
946: "change_case_UTF8 error: end pointers do not match");
947: }
948:
1.60 misha 949: static size_t getDecNumLength(XMLCh UTF8Char){
950: return
951: (UTF8Char < 100)
952: ?2
953: :(UTF8Char < 1000)
954: ?3
955: :(UTF8Char < 10000)
956: ?4
957: :5;
958: }
1.38 paf 959:
1.35 paf 960: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 961: int src_length=src.length;
1.60 misha 962: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 963: int dest_length=0;
1.82 misha 964: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 965: dest_length += ( i.next() & 0xFFFF0000 )
966: ? 3*i.getCharSize() // %XX for each byte
967: : ( xlatOneTo(i.next(), tables, 0) != 0 )
968: ? 1 // can convert it to a single char
969: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 970: }
971: #else
972: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 973: int dest_length=src_length*6;
1.60 misha 974: #endif
975:
1.35 paf 976: #ifndef NDEBUG
1.71 misha 977: int saved_dest_length=dest_length;
1.35 paf 978: #endif
979: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 980:
981: if(::transcodeFromUTF8(
1.82 misha 982: (XMLByte *)src.str, src_length,
1.35 paf 983: dest_body, dest_length,
1.11 paf 984: tables)<0)
1.43 paf 985: throw Exception(0,
1.10 paf 986: 0,
1.35 paf 987: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 988:
1.60 misha 989: assert(dest_length<=saved_dest_length);
990: dest_body[dest_length]=0; // terminator
1.35 paf 991: return String::C((char*)dest_body, dest_length);
1.1 paf 992: }
993:
994: /// transcode using both charsets
1.35 paf 995: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 996: const Charset& dest_charset) const {
1.35 paf 997: if(&dest_charset==this)
998: return src;
999: else {
1000: size_t dest_length=src.length;
1001: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1002:
1003: XMLByte* output=dest_body;
1004: const XMLByte* input=(XMLByte *)src.str;
1005: while(XMLCh c=*input++) {
1006: XMLCh curVal = tables.fromTable[c];
1007: *output++=curVal?
1008: xlatOneTo(curVal, dest_charset.tables, '?') // OK
1009: :'?'; // use the replacement character
1.6 paf 1010: }
1.1 paf 1011:
1.35 paf 1012: dest_body[dest_length]=0; // terminator
1013: return String::C((char*)dest_body, dest_length);
1.6 paf 1014: }
1.1 paf 1015: }
1016:
1.58 misha 1017: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 1018: if(isUTF8())
1.58 misha 1019: store_UTF8(src, outPtr);
1.59 misha 1020: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1021: *outPtr++=ch;
1.57 misha 1022: }
1023:
1.1 paf 1024: #ifdef XML
1.10 paf 1025:
1.35 paf 1026: static const Charset::Tables* tables[MAX_CHARSETS];
1027:
1.46 paf 1028: #ifdef PA_PATCHED_LIBXML_BACKWARD
1029:
1030: #define declareXml256ioFuncs(i) \
1031: static int xml256CharEncodingInputFunc##i( \
1032: unsigned char *out, int *outlen, \
1033: const unsigned char *in, int *inlen, void*) { \
1034: return transcodeToUTF8( \
1.71 misha 1035: in, *inlen, \
1036: out, *outlen, \
1.46 paf 1037: *tables[i]); \
1038: } \
1039: static int xml256CharEncodingOutputFunc##i( \
1040: unsigned char *out, int *outlen, \
1041: const unsigned char *in, int *inlen, void*) { \
1042: return transcodeFromUTF8( \
1.71 misha 1043: in, *inlen, \
1044: out, *outlen, \
1.46 paf 1045: *tables[i]); \
1046: }
1047:
1048: #else
1049:
1.35 paf 1050: #define declareXml256ioFuncs(i) \
1051: static int xml256CharEncodingInputFunc##i( \
1052: unsigned char *out, int *outlen, \
1053: const unsigned char *in, int *inlen) { \
1054: return transcodeToUTF8( \
1.71 misha 1055: in, *inlen, \
1056: out, *outlen, \
1.35 paf 1057: *tables[i]); \
1058: } \
1059: static int xml256CharEncodingOutputFunc##i( \
1060: unsigned char *out, int *outlen, \
1061: const unsigned char *in, int *inlen) { \
1062: return transcodeFromUTF8( \
1.71 misha 1063: in, *inlen, \
1064: out, *outlen, \
1.35 paf 1065: *tables[i]); \
1066: }
1067:
1.46 paf 1068: #endif
1069:
1070:
1.35 paf 1071: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1072: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1073: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1074: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1075: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1076:
1077: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1078: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1079: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1080: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1081: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1082: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1083: };
1084: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1085: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1086: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1087: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1088: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1089: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1090: };
1091: static size_t handlers_count=0;
1.10 paf 1092:
1093: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1094: if(handlers_count==MAX_CHARSETS)
1095: throw Exception(0,
1096: 0,
1097: "already allocated %d handlers, no space for new encoding '%s'",
1098: MAX_CHARSETS, name_cstr);
1099:
1.97 moko 1100: xmlCharEncodingHandler* handler=new xmlCharEncodingHandler;
1.35 paf 1101: {
1102: handler->name=name_cstr;
1103: handler->input=inputFuncs[handlers_count];
1104: handler->output=outputFuncs[handlers_count];
1105: ::tables[handlers_count]=&tables;
1106: handlers_count++;
1107: }
1.10 paf 1108:
1109: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1110:
1.10 paf 1111: }
1112:
1.37 paf 1113: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1114: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1115: transcoder(NAME); // check right way
1.15 paf 1116: }
1117:
1.37 paf 1118: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1119: if(!ftranscoder)
1.56 misha 1120: throw Exception(PARSER_RUNTIME,
1.35 paf 1121: new String(NAME, String::L_TAINTED),
1.10 paf 1122: "unsupported encoding");
1.35 paf 1123: return *ftranscoder;
1.10 paf 1124: }
1125:
1.54 paf 1126: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1127: if(!s)
1.35 paf 1128: return String::C("", 0);
1.8 paf 1129:
1.35 paf 1130: int inlen=strlen((const char*)s);
1.51 paf 1131: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1132: #ifndef NDEBUG
1133: int saved_outlen=outlen;
1134: #endif
1135: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1136:
1.30 paf 1137: int error;
1.35 paf 1138: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1139: error=output(
1.17 paf 1140: (unsigned char*)out, &outlen,
1.46 paf 1141: (const unsigned char*)s, &inlen
1142: #ifdef PA_PATCHED_LIBXML_BACKWARD
1143: ,0
1144: #endif
1145: );
1.30 paf 1146: } else {
1147: memcpy(out, s, outlen=inlen);
1148: error=0;
1149: }
1150: if(error<0)
1.23 paf 1151: throw Exception(0,
1.8 paf 1152: 0,
1.30 paf 1153: "transcode_cstr failed (%d)", error);
1.8 paf 1154:
1.35 paf 1155: assert(outlen<=saved_outlen); out[outlen]=0;
1156: return String::C(out, outlen);
1.14 paf 1157: }
1.54 paf 1158: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1159: String::C cstr=transcode_cstr(s);
1.75 misha 1160: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1161: }
1162:
1.8 paf 1163: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1164: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1165: xmlChar* out;
1.30 paf 1166: int outlen;
1167: int error;
1.35 paf 1168: #ifndef NDEBUG
1169: int saved_outlen;
1170: #endif
1171: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1172: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1173: #ifndef NDEBUG
1174: saved_outlen=outlen;
1175: #endif
1.47 paf 1176: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1177: error=input(
1.17 paf 1178: out, &outlen,
1.46 paf 1179: (const unsigned char*)buf, (int*)&buf_size
1180: #ifdef PA_PATCHED_LIBXML_BACKWARD
1181: ,0
1182: #endif
1183: );
1.30 paf 1184: } else {
1185: outlen=buf_size;
1.35 paf 1186: #ifndef NDEBUG
1187: saved_outlen=outlen;
1188: #endif
1189: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1190: memcpy(out, buf, outlen);
1191: error=0;
1192: }
1.17 paf 1193:
1.30 paf 1194: if(error<0)
1.23 paf 1195: throw Exception(0,
1.8 paf 1196: 0,
1.30 paf 1197: "transcode_buf failed (%d)", error);
1.8 paf 1198:
1.35 paf 1199: assert(outlen<=saved_outlen); out[outlen]=0;
1200: return out;
1.24 paf 1201: }
1.1 paf 1202:
1.79 misha 1203: xmlChar* Charset::transcode(const String& s) {
1204: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1205: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1206: }
1.35 paf 1207:
1.79 misha 1208: xmlChar* Charset::transcode(const String::Body s) {
1209: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1210: }
1.36 paf 1211: #endif
1.34 paf 1212:
1.37 paf 1213: String::Body Charset::transcode(const String::Body src,
1.34 paf 1214: const Charset& source_transcoder,
1.35 paf 1215: const Charset& dest_transcoder) {
1.34 paf 1216:
1.35 paf 1217: const char *src_ptr=src.cstr();
1.83 misha 1218: size_t src_size=src.length();
1.34 paf 1219:
1.77 misha 1220: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1221:
1.77 misha 1222: return String::Body(dest.length ? dest.str:0);
1.35 paf 1223: }
1224:
1225: String& Charset::transcode(const String& src,
1226: const Charset& source_transcoder,
1227: const Charset& dest_transcoder) {
1.72 misha 1228: if(src.is_empty())
1.73 misha 1229: return *new String();
1.34 paf 1230:
1.37 paf 1231: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1232: }
1233:
1.35 paf 1234: void Charset::transcode(ArrayString& src,
1.34 paf 1235: const Charset& source_transcoder,
1.35 paf 1236: const Charset& dest_transcoder) {
1237: for(size_t i=0; i<src.count(); i++)
1238: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1239: }
1240:
1241: #ifndef DOXYGEN
1242: struct Transcode_pair_info {
1243: const Charset* source_transcoder;
1244: const Charset* dest_transcoder;
1245: };
1246: #endif
1.76 misha 1247: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1248: String::Body& avalue,
1.35 paf 1249: Transcode_pair_info* info) {
1250: avalue=Charset::transcode(avalue,
1251: *info->source_transcoder,
1252: *info->dest_transcoder);
1.34 paf 1253: }
1.61 misha 1254:
1.35 paf 1255: void Charset::transcode(HashStringString& src,
1.34 paf 1256: const Charset& source_transcoder,
1.35 paf 1257: const Charset& dest_transcoder) {
1258: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1259: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1260: }
1.61 misha 1261:
1262: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1263: const XMLByte* ptr=srcBegin;
1.70 misha 1264: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1265:
1266: return ptr-srcBegin;
1267: }
1268:
1269: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1270: size_t charPos=0;
1271: const XMLByte* ptr=srcBegin;
1272: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1273: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1274: if(ptr>ptrEnd)
1275: return charPos;
1276: charPos++;
1277: }
1278:
1279: // scan till end but position in bytes still too low
1280: throw Exception(0,
1281: 0,
1282: "Error convertion byte pos to char pos");
1283: }
1284:
1285: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1286: size_t size=0;
1.70 misha 1287: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1288: size++;
1289:
1290: return size;
1291: }
1.80 misha 1292:
1.84 misha 1293: unsigned int lengthUTF8Char(const XMLByte c){
1294: return gUTFBytes[c]+1;
1295: }
1296:
1.94 moko 1297: const char *fixUTF8(const char *src){
1298: if(src && *src){
1299: size_t length=strlen(src);
1300:
1301: int error_offset;
1.96 moko 1302: if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
1.94 moko 1303:
1304: char *result=(char *)pa_malloc_atomic(length+1);
1305: char *dst=result;
1306:
1307: do {
1308:
1309: if(error_offset){
1310: strncpy(dst, src, error_offset);
1311: dst+=error_offset;
1312:
1313: src+=error_offset;
1314: length-=error_offset;
1315:
1316: }
1317:
1318: *dst++='?';
1319: src++;
1320: length--;
1321:
1.96 moko 1322: } while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));
1.94 moko 1323:
1324: if(length){
1325: strcpy(dst, src);
1326: } else {
1327: *dst='\0';
1328: }
1329:
1330: return result;
1331: }
1332: }
1333: return src;
1334: }
1335:
1.80 misha 1336: bool UTF8_string_iterator::has_next(){
1337: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1338: return fcharSize!=0;
1339: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main