Annotation of parser3/src/main/pa_charset.C, revision 1.95
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.90 moko 4: Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.95 ! moko 11: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.94 2013/10/15 21:27:37 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 12:
1.1 paf 13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 18:
19: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 20: #define PRECALCULATE_DEST_LENGTH
1.46 paf 21:
1.38 paf 22: // globals
23:
24: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
25: #include "utf8-to-upper.inc"
26: };
27: Charset::UTF8CaseTable UTF8CaseToUpper={
28: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
29: UTF8CaseToUpperRecords};
30:
31: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
32: #include "utf8-to-lower.inc"
33: };
34: Charset::UTF8CaseTable UTF8CaseToLower={
35: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
36: UTF8CaseToLowerRecords};
37:
1.1 paf 38: // helpers
39:
40: inline void prepare_case_tables(unsigned char *tables) {
41: unsigned char *lcc_table=tables+lcc_offset;
42: unsigned char *fcc_table=tables+fcc_offset;
43: for(int i=0; i<0x100; i++)
1.53 paf 44: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 45: }
46: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
47: unsigned char bit) {
48: unsigned char *ctypes_table=tables+ctypes_offset;
49: ctypes_table[0]=bit;
50: for(; *cstr; cstr++) {
51: unsigned char c=*cstr;
52: ctypes_table[c]|=bit;
53: }
54: }
1.35 paf 55: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 56: if(!cstr || !*cstr)
57: return 0;
58: if(cstr[1]==0)
1.4 paf 59: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 60:
1.91 moko 61: return pa_atoui(cstr,0);
1.1 paf 62: }
1.35 paf 63: inline bool to_bool(const char* cstr) {
1.1 paf 64: return cstr && *cstr!=0;
65: }
66: static void element2ctypes(unsigned char c, bool belongs,
67: unsigned char *tables, unsigned char bit, int group_offset=-1) {
68: if(!belongs)
69: return;
70:
71: unsigned char *ctypes_table=tables+ctypes_offset;
72:
73: ctypes_table[c]|=bit;
74: if(group_offset>=0)
1.4 paf 75: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 76: }
77: static void element2case(unsigned char from, unsigned char to,
78: unsigned char *tables) {
79: if(!to)
80: return;
81:
82: unsigned char *lcc_table=tables+lcc_offset;
83: unsigned char *fcc_table=tables+fcc_offset;
84: lcc_table[from]=to;
85: fcc_table[from]=to; fcc_table[to]=from;
86: }
87:
1.95 ! moko 88: inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
1.93 moko 89: if(prefix) {
1.95 ! moko 90: strcpy((char *)dest, prefix);
1.93 moko 91: dest+=strlen(prefix);
92: }
93: *dest++=hex_digits[c >> 4];
94: *dest++=hex_digits[c & 0x0F];
1.95 ! moko 95: return dest;
1.93 moko 96: }
97:
1.95 ! moko 98: inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
1.93 moko 99: if(prefix) {
1.95 ! moko 100: strcpy((char *)dest, prefix);
1.93 moko 101: dest+=strlen(prefix);
102: }
103: *dest++=hex_digits[(c >> 12) & 0x0F];
104: *dest++=hex_digits[(c >> 8) & 0x0F];
105: *dest++=hex_digits[(c >> 4) & 0x0F];
106: *dest++=hex_digits[(c) & 0x0F];
1.95 ! moko 107: return dest;
1.93 moko 108: }
109:
1.1 paf 110: // methods
111:
1.37 paf 112: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 113: FNAME(ANAME),
114: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 115:
1.35 paf 116: if(afile_spec) {
1.1 paf 117: fisUTF8=false;
1.35 paf 118: load_definition(*charsets, *afile_spec);
1.1 paf 119: #ifdef XML
1.35 paf 120: addEncoding(FNAME_CSTR);
1.1 paf 121: #endif
122: } else {
123: fisUTF8=true;
1.4 paf 124: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 125: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 126: }
127:
128: #ifdef XML
1.35 paf 129: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 130: #endif
131: }
132:
1.35 paf 133: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 134: // pcre_tables
135: // lowcase, flipcase, bits digit+word+whitespace, masks
136:
137: // must not move this inside of prepare_case_tables
138: // don't know the size there
139: memset(pcre_tables, 0, sizeof(pcre_tables));
140: prepare_case_tables(pcre_tables);
1.4 paf 141: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 142:
143: // charset
1.35 paf 144: memset(&tables, 0, sizeof(tables));
1.1 paf 145:
146: // loading text
1.35 paf 147: char *data=file_read_text(charsets, afile_spec);
1.1 paf 148:
149: // ignore header
150: getrow(&data);
151:
152: // parse cells
153: char *row;
1.42 paf 154: while((row=getrow(&data))) {
1.1 paf 155: // remove empty&comment lines
156: if(!*row || *row=='#')
157: continue;
158:
159: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 160: unsigned char c=0;
1.1 paf 161: char *cell;
1.42 paf 162: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 163: switch(column) {
1.53 paf 164: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 165: // pcre_tables
166: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
167: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
168: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
169: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
170: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 171: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 172: case 7:
173: case 8:
174: // charset
1.10 paf 175: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 176: throw Exception(PARSER_RUNTIME,
1.35 paf 177: &afile_spec,
1.1 paf 178: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
179:
180: XMLCh unicode=(XMLCh)to_wchar_code(cell);
181: if(!unicode && column==7/*unicode1 column*/)
182: unicode=(XMLCh)c;
183: if(unicode) {
1.10 paf 184: if(!tables.fromTable[c])
185: tables.fromTable[c]=unicode;
186: tables.toTable[tables.toTableSize].intCh=unicode;
187: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
188: tables.toTableSize++;
1.1 paf 189: }
190: break;
191: }
192: }
193: };
194:
1.87 moko 195: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
196: for(uint i=0; i<0x20; i++)
197: if(!tables.fromTable[i]){
198: tables.fromTable[i]=i;
199: tables.toTable[tables.toTableSize].intCh=i;
200: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
201: tables.toTableSize++;
202: }
203:
1.1 paf 204: // sort by the Unicode code point
205: sort_ToTable();
206: }
207:
208: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
209: return
1.38 paf 210: static_cast<const Charset::Tables::Rec *>(a)->intCh-
211: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 212: }
213:
214: void Charset::sort_ToTable() {
1.92 moko 215: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 216: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 217: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 218: //fclose(f);
219: }
220:
1.60 misha 221: // @todo: precache for spedup searching
1.10 paf 222: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 223: const Charset::Tables& tables,
224: XMLByte not_found) {
1.80 misha 225: int lo = 0;
226: int hi = tables.toTableSize - 1;
1.39 paf 227: while(lo<=hi) {
1.35 paf 228: // Calc the mid point of the low and high offset.
1.39 paf 229: const unsigned int i = (lo + hi) / 2;
230:
231: XMLCh cur=tables.toTable[i].intCh;
232: if(toXlat==cur)
233: return tables.toTable[i].extCh;
234: if(toXlat>cur)
235: lo = i+1;
1.1 paf 236: else
1.39 paf 237: hi = i-1;
238: }
1.35 paf 239:
240: return not_found;
1.1 paf 241: }
242:
1.35 paf 243: String::C Charset::transcode(const String::C src,
244: const Charset& source_charset,
245: const Charset& dest_charset) {
246: if(!src.length)
247: return String::C("", 0);
1.4 paf 248:
1.1 paf 249: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
250: default: // 0x00
1.35 paf 251: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 252: case 0x01:
1.35 paf 253: return source_charset.transcodeToUTF8(src);
1.1 paf 254: case 0x10:
1.35 paf 255: return dest_charset.transcodeFromUTF8(src);
1.1 paf 256: case 0x11:
1.35 paf 257: return src;
1.1 paf 258: }
259: }
260:
261: // ---------------------------------------------------------------------------
262: // Local static data
263: //
264: // gUTFBytes
265: // A list of counts of trailing bytes for each initial byte in the input.
266: //
267: // gUTFOffsets
268: // A list of values to offset each result char type, according to how
269: // many source bytes when into making it.
270: //
271: // gFirstByteMark
272: // A list of values to mask onto the first byte of an encoded sequence,
273: // indexed by the number of bytes used to create the sequence.
274: // ---------------------------------------------------------------------------
275: static const XMLByte gUTFBytes[0x100] = {
276: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
277: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
278: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
279: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
280: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
281: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
282: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
283: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
284: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
285: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
287: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
288: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
289: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
290: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
291: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
292: };
293:
294: static const uint gUTFOffsets[6] = {
1.80 misha 295: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 296: };
297:
298: static const XMLByte gFirstByteMark[7] = {
1.80 misha 299: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 300: };
301:
1.71 misha 302: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
303: XMLByte *toFill, int& toFillLen,
304: const Charset::Tables& tables) {
1.11 paf 305: const XMLByte* srcPtr=srcData;
306: const XMLByte* srcEnd=srcData+srcLen;
307: XMLByte* outPtr=toFill;
308: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 309:
1.35 paf 310: while(srcPtr<srcEnd) {
311: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 312: if(!curVal) {
1.35 paf 313: // use the replacement character
314: *outPtr++= '?';
315: srcPtr++;
316: continue;
317: }
1.1 paf 318:
1.35 paf 319: // Figure out how many bytes we need
320: unsigned int encodedBytes;
321: if(curVal<0x80)
322: encodedBytes = 1;
323: else if(curVal<0x800)
324: encodedBytes = 2;
325: else if(curVal<0x10000)
326: encodedBytes = 3;
327: else if(curVal<0x200000)
328: encodedBytes = 4;
329: else if(curVal<0x4000000)
330: encodedBytes = 5;
331: else if(curVal<= 0x7FFFFFFF)
332: encodedBytes = 6;
333: else {
334: // use the replacement character
335: *outPtr++= '?';
336: srcPtr++;
337: continue;
338: }
1.11 paf 339:
1.35 paf 340: // If we cannot fully get this char into the output buffer
341: if (outPtr + encodedBytes > outEnd)
342: break;
343:
344: // We can do it, so update the source index
345: srcPtr++;
346:
347: // And spit out the bytes. We spit them out in reverse order
348: // here, so bump up the output pointer and work down as we go.
349: outPtr+= encodedBytes;
350: switch(encodedBytes) {
1.60 misha 351: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
352: curVal>>= 6;
353: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
354: curVal>>= 6;
355: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
356: curVal>>= 6;
357: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
358: curVal>>= 6;
359: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
360: curVal>>= 6;
361: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 362: }
363:
364: // Add the encoded bytes back in again to indicate we've eaten them
365: outPtr+= encodedBytes;
366: }
367:
368: // Update the bytes eaten
369: srcLen = srcPtr - srcData;
370:
371: // Return the characters read
372: toFillLen = outPtr - toFill;
373:
1.29 paf 374: //return srcPtr==srcEnd?(int)toFillLen:-1;
375: /*
376: xmlCharEncodingInputFunc
377: Returns :
378: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
379: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
380: of ocetes consumed.
381: */
382: return 0;
1.1 paf 383: }
1.26 paf 384: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 385: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
386: XMLByte* toFill, int& toFillLen,
387: const Charset::Tables& tables) {
1.11 paf 388: const XMLByte* srcPtr=srcData;
389: const XMLByte* srcEnd=srcData+srcLen;
390: XMLByte* outPtr=toFill;
391: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 392:
1.35 paf 393: // We now loop until we either run out of input data, or room to store
394: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
395: // Get the next leading byte out
396: const XMLByte firstByte =* srcPtr;
397:
398: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 399: if(firstByte<=127) {
1.35 paf 400: *outPtr++= firstByte;
401: srcPtr++;
402: continue;
403: }
404:
405: // See how many trailing src bytes this sequence is going to require
406: const unsigned int trailingBytes = gUTFBytes[firstByte];
407:
408: // If there are not enough source bytes to do this one, then we
409: // are done. Note that we done>= here because we are implicitly
410: // counting the 1 byte we get no matter what.
411: if(srcPtr+trailingBytes>= srcEnd)
412: break;
413:
414: // Looks ok, so lets build up the value
415: uint tmpVal=0;
416: switch(trailingBytes) {
417: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
418: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
419: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
420: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
421: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
422: case 0: tmpVal+=*srcPtr++;
423: break;
424:
425: default:
426: throw Exception(0,
427: 0,
1.49 paf 428: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 429: }
430: tmpVal-=gUTFOffsets[trailingBytes];
431:
432: // If it will fit into a single char, then put it in. Otherwise
433: // fail [*encode it as a surrogate pair. If its not valid, use the
434: // replacement char.*]
435: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 436: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
437: *outPtr++=xlat;
1.49 paf 438: else {
1.50 paf 439: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 440: }
441: } else {
442: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
443: for(uint i=0; i<=trailingBytes; i++)
444: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
445: }
1.1 paf 446: }
1.35 paf 447:
448: // Update the bytes eaten
449: srcLen = srcPtr - srcData;
450:
451: // Return the characters read
452: toFillLen = outPtr - toFill;
1.11 paf 453:
1.29 paf 454: //return srcPtr==srcEnd?(int)toFillLen:-1;
455: /*
456: xmlCharEncodingOutputFunc
457: Returns :
458: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
459: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
460: of ocetes consumed.
461: */
462: return 0;
1.10 paf 463: }
464:
1.85 misha 465: static bool need_escape(XMLByte c){
1.60 misha 466: return
1.66 misha 467: !(
468: (c<=127)
469: && (
1.89 misha 470: pa_isalnum((unsigned char)c)
1.66 misha 471: || strchr("*@-_+./", c)!=0
472: )
473: );
1.60 misha 474: }
475:
1.70 misha 476: // read one UTF8 char and return length of this char (in bytes)
477: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 478: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
479: return 0;
480:
481: firstByte=*srcPtr;
482:
483: if(firstByte<=127){
484: UTF8Char=firstByte;
485: srcPtr++;
486: return 1;
487: }
488:
489: unsigned int trailingBytes=gUTFBytes[firstByte];
490:
491: if(srcPtr+trailingBytes>=srcEnd){
492: return 0; // not enough bytes in source string for reading
493: }
494:
495: uint tmpVal=0;
496: switch(trailingBytes){
497: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
498: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
499: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
500: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
501: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
502: case 0: tmpVal+=*srcPtr++;
503: }
504:
505: tmpVal-=gUTFOffsets[trailingBytes];
506: UTF8Char=tmpVal;
507:
508: return trailingBytes+1;
509: }
510:
1.70 misha 511: // skip UTF8 char and return length of this char (in bytes)
512: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 513: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
514: return 0;
515:
1.63 misha 516: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
517: srcPtr+=trailingBytes;
1.62 misha 518:
519: return trailingBytes;
1.61 misha 520: }
521:
1.85 misha 522: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 523: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 524: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
525: return 0;
526:
527: firstByte=*srcPtr++;
528: UTF8Char=tables.fromTable[firstByte];
529:
530: if(UTF8Char<0x80)
531: return 1;
532: else if(UTF8Char<0x800)
533: return 2;
534: else if(UTF8Char<0x10000)
535: return 3;
536: else if(UTF8Char<0x200000)
537: return 4;
538: else if(UTF8Char<0x4000000)
539: return 5;
540: else if(UTF8Char<= 0x7FFFFFFF)
541: return 6;
542:
543: // will use the replacement character '?'
544: firstByte=0;
545: return 1;
546: }
547:
1.85 misha 548: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
549: size_t dest_length=0;
550:
551: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
552: if(i.getCharSize()==1)
553: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
554: else
555: dest_length+=6; // %uXXXX
1.60 misha 556: }
557:
1.85 misha 558: return dest_length;
1.60 misha 559: }
560:
1.86 moko 561: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
562: const XMLByte* src_end=src+src_length;
563: XMLByte first_byte;
564: XMLCh UTF8_char;
1.85 misha 565: size_t dest_length=0;
566:
1.86 moko 567: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 568: if(char_size==1)
569: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
570: else
571: dest_length+=6; // %uXXXX
1.60 misha 572: }
573:
1.85 misha 574: return dest_length;
575: }
576:
577: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 578: if(!src.length)
1.85 misha 579: return 0;
580:
581: #ifdef PRECALCULATE_DEST_LENGTH
582: if(source_charset.isUTF8())
1.86 moko 583: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 584: else
1.86 moko 585: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 586: #else
587: return src_length*6; // enough for %uXXXX but too memory-hungry
588: #endif
589: }
590:
591: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
592: if(char_size==1) \
593: if(first_byte){ \
594: if(need_escape(first_byte)) \
1.95 ! moko 595: dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 596: else \
597: *dest_ptr++=first_byte; /*as is*/ \
598: } else \
599: *dest_ptr++='?'; /* replacement char '?' */ \
600: else \
1.95 ! moko 601: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 602:
603:
604: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
605: XMLByte* dest_ptr=dest;
606:
607: // loop until we either run out of input data
608: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
609: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 610:
1.85 misha 611: return dest_ptr - dest;
1.60 misha 612: }
613:
1.85 misha 614: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
615: const XMLByte* src_end=src+src_length;
616: XMLByte* dest_ptr=dest;
617:
618: XMLByte first_byte;
619: XMLCh UTF8_char;
620: uint char_size;
621:
1.86 moko 622: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 623: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
624:
625: return dest_ptr - dest;
626: }
1.60 misha 627:
628: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 629: if(!src.length)
1.60 misha 630: return String::C("", 0);
631:
1.85 misha 632: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
633: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
634:
635: size_t dest_length;
636: if(source_charset.isUTF8())
1.86 moko 637: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 638: else
1.86 moko 639: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 640:
641: if(dest_length>dest_calculated_length)
642: throw Exception(0, 0, "Charset::escape buffer overflow");
643:
644: dest_body[dest_length]=0; // terminator
645: return String::C((char*)dest_body, dest_length);
646: }
647:
648: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 649: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 650: return String::Body(dest.length ? dest.str:0);
651: }
652:
653: String& Charset::escape(const String& src, const Charset& source_charset) {
654: if(src.is_empty())
655: return *new String();
656:
657: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
658: }
659:
660: inline bool need_json_escape(unsigned char c){
661: return strchr("\n\"\\/\t\r\b\f", c)!=0;
662: }
663:
664: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
665: size_t dest_length=0;
666:
667: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 moko 668: if(i.getCharSize()==1){
669: XMLByte first_byte=i.getFirstByte();
670: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
671: } else
1.85 misha 672: dest_length+=6; // \uXXXX
673: }
674:
675: return dest_length;
676: }
677:
1.86 moko 678: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
679: const XMLByte* src_end=src+src_length;
1.85 misha 680: XMLByte first_byte;
681: XMLCh UTF8_char;
1.60 misha 682: size_t dest_length=0;
683:
1.86 moko 684: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 685: if(char_size==1)
1.93 moko 686: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 687: else
688: dest_length+=6; // \uXXXX
1.60 misha 689: }
1.85 misha 690:
691: return dest_length;
692: }
693:
694: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 695: if(!src.length)
1.85 misha 696: return 0;
697:
698: #ifdef PRECALCULATE_DEST_LENGTH
699: if(source_charset.isUTF8())
1.86 moko 700: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 701: else
1.86 moko 702: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 703: #else
1.85 misha 704: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 705: #endif
1.85 misha 706: }
707:
708: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
709: if(char_size==1) \
710: switch(first_byte){ \
711: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
712: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
713: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
714: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
715: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
716: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
717: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
718: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
719: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.95 ! moko 720: default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
1.93 moko 721: else *dest_ptr++=first_byte; \
1.85 misha 722: } \
723: else \
1.95 ! moko 724: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 725:
726:
727: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
728: XMLByte* dest_ptr=dest;
729:
730: // loop until we either run out of input data
731: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
732: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
733:
734: return dest_ptr - dest;
735: }
736:
737: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
738: const XMLByte* src_end=src+src_length;
739: XMLByte* dest_ptr=dest;
740:
741: XMLByte first_byte;
742: XMLCh UTF8_char;
743: uint char_size;
744:
1.86 moko 745: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 746: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
747:
748: return dest_ptr - dest;
749: }
1.60 misha 750:
1.85 misha 751: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 752: if(!src.length)
1.85 misha 753: return String::C("", 0);
1.60 misha 754:
1.85 misha 755: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
756: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
757:
758: size_t dest_length;
759: if(source_charset.isUTF8())
1.86 moko 760: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 761: else
1.86 moko 762: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 763:
1.85 misha 764: if(dest_length>dest_calculated_length)
765: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 766:
767: dest_body[dest_length]=0; // terminator
768: return String::C((char*)dest_body, dest_length);
769: }
1.85 misha 770:
771: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 772: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 773: return String::Body(dest.length ? dest.str:0);
1.64 misha 774: }
775:
1.85 misha 776: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 777: if(src.is_empty())
1.73 misha 778: return *new String();
1.64 misha 779:
1.85 misha 780: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 781: }
1.60 misha 782:
1.35 paf 783: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 784: int src_length=src.length;
1.60 misha 785:
786: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 787: int dest_length=0;
1.60 misha 788: const XMLByte* srcPtr=(XMLByte*)src.str;
789: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 790: XMLByte firstByte;
791: XMLCh UTF8Char;
792: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 793: dest_length+=charSize;
794: #else
1.85 misha 795: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 796: #endif
797:
1.35 paf 798: #ifndef NDEBUG
1.71 misha 799: int saved_dest_length=dest_length;
1.35 paf 800: #endif
801: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 802:
803: if(::transcodeToUTF8(
1.35 paf 804: (XMLByte *)src.str, src_length,
805: dest_body, dest_length,
1.11 paf 806: tables)<0)
1.43 paf 807: throw Exception(0,
1.10 paf 808: 0,
1.11 paf 809: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 810:
1.60 misha 811: assert(dest_length<=saved_dest_length);
812: dest_body[dest_length]=0; // terminator
1.35 paf 813: return String::C((char*)dest_body, dest_length);
1.10 paf 814: }
1.38 paf 815:
816: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 817: int lo = 0;
818: int hi = table.size - 1;
1.39 paf 819: while(lo<=hi) {
1.38 paf 820: // Calc the mid point of the low and high offset.
1.39 paf 821: const unsigned int i = (lo + hi) / 2;
822:
823: XMLCh cur=table.records[i].from;
824: if(src==cur)
825: return table.records[i].to;
826: if(src>cur)
827: lo = i+1;
1.38 paf 828: else
1.39 paf 829: hi = i-1;
830: }
831:
832: // not found
1.38 paf 833: return src;
834: }
835:
1.58 misha 836: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 837: if(!src) {
838: // use the replacement character
839: *outPtr++= '?';
840: return;
841: }
842:
843: // Figure out how many bytes we need
844: unsigned int encodedBytes;
845: if(src<0x80)
846: encodedBytes = 1;
847: else if(src<0x800)
848: encodedBytes = 2;
849: else if(src<0x10000)
850: encodedBytes = 3;
851: else if(src<0x200000)
852: encodedBytes = 4;
853: else if(src<0x4000000)
854: encodedBytes = 5;
855: else if(src<= 0x7FFFFFFF)
856: encodedBytes = 6;
857: else {
858: // use the replacement character
859: *outPtr++= '?';
860: return;
861: }
862:
863: // And spit out the bytes. We spit them out in reverse order
864: // here, so bump up the output pointer and work down as we go.
865: outPtr+= encodedBytes;
866: switch(encodedBytes) {
867: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
868: src>>= 6;
869: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
870: src>>= 6;
871: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
872: src>>= 6;
873: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
874: src>>= 6;
875: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
876: src>>= 6;
877: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
878: }
879:
880: // Add the encoded bytes back in again to indicate we've eaten them
881: outPtr+= encodedBytes;
882: }
883:
884: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
885: const Charset::UTF8CaseTable& table) {
886: store_UTF8(change_case_UTF8(src, table), outPtr);
887: };
1.44 paf 888: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
889: XMLByte* toFill, size_t toFillLen,
890: const Charset::UTF8CaseTable& table) {
1.38 paf 891: const XMLByte* srcPtr=srcData;
1.44 paf 892: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 893: XMLByte* outPtr=toFill;
1.44 paf 894: XMLByte* outEnd=toFill+toFillLen;
895:
896: // We now loop until we either run out of input data, or room to store
897: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
898: // Get the next leading byte out
899: const XMLByte firstByte =* srcPtr;
1.38 paf 900:
1.60 misha 901: if(firstByte<=127) {
1.38 paf 902: change_case_UTF8(firstByte, outPtr, table);
903: srcPtr++;
904: continue;
905: }
906:
907: // See how many trailing src bytes this sequence is going to require
908: const unsigned int trailingBytes = gUTFBytes[firstByte];
909:
910: // Looks ok, so lets build up the value
911: uint tmpVal=0;
912: switch(trailingBytes) {
913: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
914: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
915: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
916: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
917: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
918: case 0: tmpVal+=*srcPtr++;
919: break;
920:
921: default:
922: throw Exception(0,
923: 0,
924: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
925: }
926: tmpVal-=gUTFOffsets[trailingBytes];
927:
928: // If it will fit into a single char, then put it in. Otherwise
929: // fail [*encode it as a surrogate pair. If its not valid, use the
930: // replacement char.*]
931: if(!(tmpVal & 0xFFFF0000))
932: change_case_UTF8(tmpVal, outPtr, table);
933: else
934: throw Exception(0,
935: 0,
936: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
937: }
938:
939: if(srcPtr!=outPtr)
940: throw Exception(0,
941: 0,
942: "change_case_UTF8 error: end pointers do not match");
943: }
944:
1.60 misha 945: static size_t getDecNumLength(XMLCh UTF8Char){
946: return
947: (UTF8Char < 100)
948: ?2
949: :(UTF8Char < 1000)
950: ?3
951: :(UTF8Char < 10000)
952: ?4
953: :5;
954: }
1.38 paf 955:
1.35 paf 956: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 957: int src_length=src.length;
1.60 misha 958: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 959: int dest_length=0;
1.82 misha 960: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 961: dest_length += ( i.next() & 0xFFFF0000 )
962: ? 3*i.getCharSize() // %XX for each byte
963: : ( xlatOneTo(i.next(), tables, 0) != 0 )
964: ? 1 // can convert it to a single char
965: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 966: }
967: #else
968: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 969: int dest_length=src_length*6;
1.60 misha 970: #endif
971:
1.35 paf 972: #ifndef NDEBUG
1.71 misha 973: int saved_dest_length=dest_length;
1.35 paf 974: #endif
975: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 976:
977: if(::transcodeFromUTF8(
1.82 misha 978: (XMLByte *)src.str, src_length,
1.35 paf 979: dest_body, dest_length,
1.11 paf 980: tables)<0)
1.43 paf 981: throw Exception(0,
1.10 paf 982: 0,
1.35 paf 983: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 984:
1.60 misha 985: assert(dest_length<=saved_dest_length);
986: dest_body[dest_length]=0; // terminator
1.35 paf 987: return String::C((char*)dest_body, dest_length);
1.1 paf 988: }
989:
990: /// transcode using both charsets
1.35 paf 991: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 992: const Charset& dest_charset) const {
1.35 paf 993: if(&dest_charset==this)
994: return src;
995: else {
996: size_t dest_length=src.length;
997: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
998:
999: XMLByte* output=dest_body;
1000: const XMLByte* input=(XMLByte *)src.str;
1001: while(XMLCh c=*input++) {
1002: XMLCh curVal = tables.fromTable[c];
1003: *output++=curVal?
1004: xlatOneTo(curVal, dest_charset.tables, '?') // OK
1005: :'?'; // use the replacement character
1.6 paf 1006: }
1.1 paf 1007:
1.35 paf 1008: dest_body[dest_length]=0; // terminator
1009: return String::C((char*)dest_body, dest_length);
1.6 paf 1010: }
1.1 paf 1011: }
1012:
1.58 misha 1013: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 1014: if(isUTF8())
1.58 misha 1015: store_UTF8(src, outPtr);
1.59 misha 1016: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1017: *outPtr++=ch;
1.57 misha 1018: }
1019:
1.1 paf 1020: #ifdef XML
1.10 paf 1021:
1.35 paf 1022: static const Charset::Tables* tables[MAX_CHARSETS];
1023:
1.46 paf 1024: #ifdef PA_PATCHED_LIBXML_BACKWARD
1025:
1026: #define declareXml256ioFuncs(i) \
1027: static int xml256CharEncodingInputFunc##i( \
1028: unsigned char *out, int *outlen, \
1029: const unsigned char *in, int *inlen, void*) { \
1030: return transcodeToUTF8( \
1.71 misha 1031: in, *inlen, \
1032: out, *outlen, \
1.46 paf 1033: *tables[i]); \
1034: } \
1035: static int xml256CharEncodingOutputFunc##i( \
1036: unsigned char *out, int *outlen, \
1037: const unsigned char *in, int *inlen, void*) { \
1038: return transcodeFromUTF8( \
1.71 misha 1039: in, *inlen, \
1040: out, *outlen, \
1.46 paf 1041: *tables[i]); \
1042: }
1043:
1044: #else
1045:
1.35 paf 1046: #define declareXml256ioFuncs(i) \
1047: static int xml256CharEncodingInputFunc##i( \
1048: unsigned char *out, int *outlen, \
1049: const unsigned char *in, int *inlen) { \
1050: return transcodeToUTF8( \
1.71 misha 1051: in, *inlen, \
1052: out, *outlen, \
1.35 paf 1053: *tables[i]); \
1054: } \
1055: static int xml256CharEncodingOutputFunc##i( \
1056: unsigned char *out, int *outlen, \
1057: const unsigned char *in, int *inlen) { \
1058: return transcodeFromUTF8( \
1.71 misha 1059: in, *inlen, \
1060: out, *outlen, \
1.35 paf 1061: *tables[i]); \
1062: }
1063:
1.46 paf 1064: #endif
1065:
1066:
1.35 paf 1067: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1068: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1069: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1070: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1071: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1072:
1073: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1074: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1075: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1076: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1077: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1078: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1079: };
1080: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1081: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1082: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1083: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1084: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1085: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1086: };
1087: static size_t handlers_count=0;
1.10 paf 1088:
1089: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1090: if(handlers_count==MAX_CHARSETS)
1091: throw Exception(0,
1092: 0,
1093: "already allocated %d handlers, no space for new encoding '%s'",
1094: MAX_CHARSETS, name_cstr);
1095:
1.45 paf 1096: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 1097: {
1098: handler->name=name_cstr;
1099: handler->input=inputFuncs[handlers_count];
1100: handler->output=outputFuncs[handlers_count];
1101: ::tables[handlers_count]=&tables;
1102: handlers_count++;
1103: }
1.10 paf 1104:
1105: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1106:
1.10 paf 1107: }
1108:
1.37 paf 1109: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1110: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1111: transcoder(NAME); // check right way
1.15 paf 1112: }
1113:
1.37 paf 1114: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1115: if(!ftranscoder)
1.56 misha 1116: throw Exception(PARSER_RUNTIME,
1.35 paf 1117: new String(NAME, String::L_TAINTED),
1.10 paf 1118: "unsupported encoding");
1.35 paf 1119: return *ftranscoder;
1.10 paf 1120: }
1121:
1.54 paf 1122: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1123: if(!s)
1.35 paf 1124: return String::C("", 0);
1.8 paf 1125:
1.35 paf 1126: int inlen=strlen((const char*)s);
1.51 paf 1127: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1128: #ifndef NDEBUG
1129: int saved_outlen=outlen;
1130: #endif
1131: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1132:
1.30 paf 1133: int error;
1.35 paf 1134: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1135: error=output(
1.17 paf 1136: (unsigned char*)out, &outlen,
1.46 paf 1137: (const unsigned char*)s, &inlen
1138: #ifdef PA_PATCHED_LIBXML_BACKWARD
1139: ,0
1140: #endif
1141: );
1.30 paf 1142: } else {
1143: memcpy(out, s, outlen=inlen);
1144: error=0;
1145: }
1146: if(error<0)
1.23 paf 1147: throw Exception(0,
1.8 paf 1148: 0,
1.30 paf 1149: "transcode_cstr failed (%d)", error);
1.8 paf 1150:
1.35 paf 1151: assert(outlen<=saved_outlen); out[outlen]=0;
1152: return String::C(out, outlen);
1.14 paf 1153: }
1.54 paf 1154: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1155: String::C cstr=transcode_cstr(s);
1.75 misha 1156: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1157: }
1158:
1.8 paf 1159: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1160: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1161: xmlChar* out;
1.30 paf 1162: int outlen;
1163: int error;
1.35 paf 1164: #ifndef NDEBUG
1165: int saved_outlen;
1166: #endif
1167: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1168: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1169: #ifndef NDEBUG
1170: saved_outlen=outlen;
1171: #endif
1.47 paf 1172: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1173: error=input(
1.17 paf 1174: out, &outlen,
1.46 paf 1175: (const unsigned char*)buf, (int*)&buf_size
1176: #ifdef PA_PATCHED_LIBXML_BACKWARD
1177: ,0
1178: #endif
1179: );
1.30 paf 1180: } else {
1181: outlen=buf_size;
1.35 paf 1182: #ifndef NDEBUG
1183: saved_outlen=outlen;
1184: #endif
1185: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1186: memcpy(out, buf, outlen);
1187: error=0;
1188: }
1.17 paf 1189:
1.30 paf 1190: if(error<0)
1.23 paf 1191: throw Exception(0,
1.8 paf 1192: 0,
1.30 paf 1193: "transcode_buf failed (%d)", error);
1.8 paf 1194:
1.35 paf 1195: assert(outlen<=saved_outlen); out[outlen]=0;
1196: return out;
1.24 paf 1197: }
1.1 paf 1198:
1.79 misha 1199: xmlChar* Charset::transcode(const String& s) {
1200: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1201: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1202: }
1.35 paf 1203:
1.79 misha 1204: xmlChar* Charset::transcode(const String::Body s) {
1205: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1206: }
1.36 paf 1207: #endif
1.34 paf 1208:
1.37 paf 1209: String::Body Charset::transcode(const String::Body src,
1.34 paf 1210: const Charset& source_transcoder,
1.35 paf 1211: const Charset& dest_transcoder) {
1.34 paf 1212:
1.35 paf 1213: const char *src_ptr=src.cstr();
1.83 misha 1214: size_t src_size=src.length();
1.34 paf 1215:
1.77 misha 1216: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1217:
1.77 misha 1218: return String::Body(dest.length ? dest.str:0);
1.35 paf 1219: }
1220:
1221: String& Charset::transcode(const String& src,
1222: const Charset& source_transcoder,
1223: const Charset& dest_transcoder) {
1.72 misha 1224: if(src.is_empty())
1.73 misha 1225: return *new String();
1.34 paf 1226:
1.37 paf 1227: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1228: }
1229:
1.35 paf 1230: void Charset::transcode(ArrayString& src,
1.34 paf 1231: const Charset& source_transcoder,
1.35 paf 1232: const Charset& dest_transcoder) {
1233: for(size_t i=0; i<src.count(); i++)
1234: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1235: }
1236:
1237: #ifndef DOXYGEN
1238: struct Transcode_pair_info {
1239: const Charset* source_transcoder;
1240: const Charset* dest_transcoder;
1241: };
1242: #endif
1.76 misha 1243: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1244: String::Body& avalue,
1.35 paf 1245: Transcode_pair_info* info) {
1246: avalue=Charset::transcode(avalue,
1247: *info->source_transcoder,
1248: *info->dest_transcoder);
1.34 paf 1249: }
1.61 misha 1250:
1.35 paf 1251: void Charset::transcode(HashStringString& src,
1.34 paf 1252: const Charset& source_transcoder,
1.35 paf 1253: const Charset& dest_transcoder) {
1254: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1255: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1256: }
1.61 misha 1257:
1258: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1259: const XMLByte* ptr=srcBegin;
1.70 misha 1260: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1261:
1262: return ptr-srcBegin;
1263: }
1264:
1265: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1266: size_t charPos=0;
1267: const XMLByte* ptr=srcBegin;
1268: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1269: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1270: if(ptr>ptrEnd)
1271: return charPos;
1272: charPos++;
1273: }
1274:
1275: // scan till end but position in bytes still too low
1276: throw Exception(0,
1277: 0,
1278: "Error convertion byte pos to char pos");
1279: }
1280:
1281: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1282: size_t size=0;
1.70 misha 1283: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1284: size++;
1285:
1286: return size;
1287: }
1.80 misha 1288:
1.84 misha 1289: unsigned int lengthUTF8Char(const XMLByte c){
1290: return gUTFBytes[c]+1;
1291: }
1292:
1.94 moko 1293: const char *fixUTF8(const char *src){
1294: if(src && *src){
1295: size_t length=strlen(src);
1296:
1297: int error_offset;
1298: if(_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
1299:
1300: char *result=(char *)pa_malloc_atomic(length+1);
1301: char *dst=result;
1302:
1303: do {
1304:
1305: if(error_offset){
1306: strncpy(dst, src, error_offset);
1307: dst+=error_offset;
1308:
1309: src+=error_offset;
1310: length-=error_offset;
1311:
1312: }
1313:
1314: *dst++='?';
1315: src++;
1316: length--;
1317:
1318: } while (length && _pcre_valid_utf((unsigned char *)src, length, &error_offset));
1319:
1320: if(length){
1321: strcpy(dst, src);
1322: } else {
1323: *dst='\0';
1324: }
1325:
1326: return result;
1327: }
1328: }
1329: return src;
1330: }
1331:
1.80 misha 1332: bool UTF8_string_iterator::has_next(){
1333: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1334: return fcharSize!=0;
1335: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main