Annotation of parser3/src/main/pa_charset.C, revision 1.91
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.90 moko 4: Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.91 ! moko 11: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.90 2012-03-16 09:24:13 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 12:
1.1 paf 13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 18:
19: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 20: #define PRECALCULATE_DEST_LENGTH
1.46 paf 21:
1.38 paf 22: // globals
23:
24: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
25: #include "utf8-to-upper.inc"
26: };
27: Charset::UTF8CaseTable UTF8CaseToUpper={
28: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
29: UTF8CaseToUpperRecords};
30:
31: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
32: #include "utf8-to-lower.inc"
33: };
34: Charset::UTF8CaseTable UTF8CaseToLower={
35: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
36: UTF8CaseToLowerRecords};
37:
1.1 paf 38: // helpers
39:
40: inline void prepare_case_tables(unsigned char *tables) {
41: unsigned char *lcc_table=tables+lcc_offset;
42: unsigned char *fcc_table=tables+fcc_offset;
43: for(int i=0; i<0x100; i++)
1.53 paf 44: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 45: }
46: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
47: unsigned char bit) {
48: unsigned char *ctypes_table=tables+ctypes_offset;
49: ctypes_table[0]=bit;
50: for(; *cstr; cstr++) {
51: unsigned char c=*cstr;
52: ctypes_table[c]|=bit;
53: }
54: }
1.35 paf 55: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 56: if(!cstr || !*cstr)
57: return 0;
58: if(cstr[1]==0)
1.4 paf 59: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 60:
1.91 ! moko 61: return pa_atoui(cstr,0);
1.1 paf 62: }
1.35 paf 63: inline bool to_bool(const char* cstr) {
1.1 paf 64: return cstr && *cstr!=0;
65: }
66: static void element2ctypes(unsigned char c, bool belongs,
67: unsigned char *tables, unsigned char bit, int group_offset=-1) {
68: if(!belongs)
69: return;
70:
71: unsigned char *ctypes_table=tables+ctypes_offset;
72:
73: ctypes_table[c]|=bit;
74: if(group_offset>=0)
1.4 paf 75: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 76: }
77: static void element2case(unsigned char from, unsigned char to,
78: unsigned char *tables) {
79: if(!to)
80: return;
81:
82: unsigned char *lcc_table=tables+lcc_offset;
83: unsigned char *fcc_table=tables+fcc_offset;
84: lcc_table[from]=to;
85: fcc_table[from]=to; fcc_table[to]=from;
86: }
87:
88: // methods
89:
1.37 paf 90: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 91: FNAME(ANAME),
92: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 93:
1.35 paf 94: if(afile_spec) {
1.1 paf 95: fisUTF8=false;
1.35 paf 96: load_definition(*charsets, *afile_spec);
1.1 paf 97: #ifdef XML
1.35 paf 98: addEncoding(FNAME_CSTR);
1.1 paf 99: #endif
100: } else {
101: fisUTF8=true;
1.4 paf 102: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 103: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 104: }
105:
106: #ifdef XML
1.35 paf 107: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 108: #endif
109: }
110:
1.35 paf 111: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 112: // pcre_tables
113: // lowcase, flipcase, bits digit+word+whitespace, masks
114:
115: // must not move this inside of prepare_case_tables
116: // don't know the size there
117: memset(pcre_tables, 0, sizeof(pcre_tables));
118: prepare_case_tables(pcre_tables);
1.4 paf 119: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 120:
121: // charset
1.35 paf 122: memset(&tables, 0, sizeof(tables));
1.1 paf 123:
124: // loading text
1.35 paf 125: char *data=file_read_text(charsets, afile_spec);
1.1 paf 126:
127: // ignore header
128: getrow(&data);
129:
130: // parse cells
131: char *row;
1.42 paf 132: while((row=getrow(&data))) {
1.1 paf 133: // remove empty&comment lines
134: if(!*row || *row=='#')
135: continue;
136:
137: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 138: unsigned char c=0;
1.1 paf 139: char *cell;
1.42 paf 140: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 141: switch(column) {
1.53 paf 142: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 143: // pcre_tables
144: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
145: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
146: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
147: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
148: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 149: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 150: case 7:
151: case 8:
152: // charset
1.10 paf 153: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 154: throw Exception(PARSER_RUNTIME,
1.35 paf 155: &afile_spec,
1.1 paf 156: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
157:
158: XMLCh unicode=(XMLCh)to_wchar_code(cell);
159: if(!unicode && column==7/*unicode1 column*/)
160: unicode=(XMLCh)c;
161: if(unicode) {
1.10 paf 162: if(!tables.fromTable[c])
163: tables.fromTable[c]=unicode;
164: tables.toTable[tables.toTableSize].intCh=unicode;
165: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
166: tables.toTableSize++;
1.1 paf 167: }
168: break;
169: }
170: }
171: };
172:
1.87 moko 173: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
174: for(uint i=0; i<0x20; i++)
175: if(!tables.fromTable[i]){
176: tables.fromTable[i]=i;
177: tables.toTable[tables.toTableSize].intCh=i;
178: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
179: tables.toTableSize++;
180: }
181:
1.1 paf 182: // sort by the Unicode code point
183: sort_ToTable();
184: }
185:
186: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
187: return
1.38 paf 188: static_cast<const Charset::Tables::Rec *>(a)->intCh-
189: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 190: }
191:
192: void Charset::sort_ToTable() {
1.10 paf 193: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 194: sort_cmp_Trans_rec_intCh);
195: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 196: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 197: //fclose(f);
198: }
199:
1.60 misha 200: // @todo: precache for spedup searching
1.10 paf 201: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 202: const Charset::Tables& tables,
203: XMLByte not_found) {
1.80 misha 204: int lo = 0;
205: int hi = tables.toTableSize - 1;
1.39 paf 206: while(lo<=hi) {
1.35 paf 207: // Calc the mid point of the low and high offset.
1.39 paf 208: const unsigned int i = (lo + hi) / 2;
209:
210: XMLCh cur=tables.toTable[i].intCh;
211: if(toXlat==cur)
212: return tables.toTable[i].extCh;
213: if(toXlat>cur)
214: lo = i+1;
1.1 paf 215: else
1.39 paf 216: hi = i-1;
217: }
1.35 paf 218:
219: return not_found;
1.1 paf 220: }
221:
1.35 paf 222: String::C Charset::transcode(const String::C src,
223: const Charset& source_charset,
224: const Charset& dest_charset) {
225: if(!src.length)
226: return String::C("", 0);
1.4 paf 227:
1.1 paf 228: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
229: default: // 0x00
1.35 paf 230: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 231: case 0x01:
1.35 paf 232: return source_charset.transcodeToUTF8(src);
1.1 paf 233: case 0x10:
1.35 paf 234: return dest_charset.transcodeFromUTF8(src);
1.1 paf 235: case 0x11:
1.35 paf 236: return src;
1.1 paf 237: }
238: }
239:
240: // ---------------------------------------------------------------------------
241: // Local static data
242: //
243: // gUTFBytes
244: // A list of counts of trailing bytes for each initial byte in the input.
245: //
246: // gUTFOffsets
247: // A list of values to offset each result char type, according to how
248: // many source bytes when into making it.
249: //
250: // gFirstByteMark
251: // A list of values to mask onto the first byte of an encoded sequence,
252: // indexed by the number of bytes used to create the sequence.
253: // ---------------------------------------------------------------------------
254: static const XMLByte gUTFBytes[0x100] = {
255: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
256: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
257: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
258: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
259: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
260: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
261: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
262: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
263: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
264: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
265: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
266: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
267: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
268: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
269: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
270: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
271: };
272:
273: static const uint gUTFOffsets[6] = {
1.80 misha 274: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 275: };
276:
277: static const XMLByte gFirstByteMark[7] = {
1.80 misha 278: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 279: };
280:
1.71 misha 281: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
282: XMLByte *toFill, int& toFillLen,
283: const Charset::Tables& tables) {
1.11 paf 284: const XMLByte* srcPtr=srcData;
285: const XMLByte* srcEnd=srcData+srcLen;
286: XMLByte* outPtr=toFill;
287: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 288:
1.35 paf 289: while(srcPtr<srcEnd) {
290: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 291: if(!curVal) {
1.35 paf 292: // use the replacement character
293: *outPtr++= '?';
294: srcPtr++;
295: continue;
296: }
1.1 paf 297:
1.35 paf 298: // Figure out how many bytes we need
299: unsigned int encodedBytes;
300: if(curVal<0x80)
301: encodedBytes = 1;
302: else if(curVal<0x800)
303: encodedBytes = 2;
304: else if(curVal<0x10000)
305: encodedBytes = 3;
306: else if(curVal<0x200000)
307: encodedBytes = 4;
308: else if(curVal<0x4000000)
309: encodedBytes = 5;
310: else if(curVal<= 0x7FFFFFFF)
311: encodedBytes = 6;
312: else {
313: // use the replacement character
314: *outPtr++= '?';
315: srcPtr++;
316: continue;
317: }
1.11 paf 318:
1.35 paf 319: // If we cannot fully get this char into the output buffer
320: if (outPtr + encodedBytes > outEnd)
321: break;
322:
323: // We can do it, so update the source index
324: srcPtr++;
325:
326: // And spit out the bytes. We spit them out in reverse order
327: // here, so bump up the output pointer and work down as we go.
328: outPtr+= encodedBytes;
329: switch(encodedBytes) {
1.60 misha 330: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
331: curVal>>= 6;
332: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
333: curVal>>= 6;
334: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
335: curVal>>= 6;
336: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
337: curVal>>= 6;
338: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
339: curVal>>= 6;
340: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 341: }
342:
343: // Add the encoded bytes back in again to indicate we've eaten them
344: outPtr+= encodedBytes;
345: }
346:
347: // Update the bytes eaten
348: srcLen = srcPtr - srcData;
349:
350: // Return the characters read
351: toFillLen = outPtr - toFill;
352:
1.29 paf 353: //return srcPtr==srcEnd?(int)toFillLen:-1;
354: /*
355: xmlCharEncodingInputFunc
356: Returns :
357: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
358: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
359: of ocetes consumed.
360: */
361: return 0;
1.1 paf 362: }
1.26 paf 363: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 364: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
365: XMLByte* toFill, int& toFillLen,
366: const Charset::Tables& tables) {
1.11 paf 367: const XMLByte* srcPtr=srcData;
368: const XMLByte* srcEnd=srcData+srcLen;
369: XMLByte* outPtr=toFill;
370: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 371:
1.35 paf 372: // We now loop until we either run out of input data, or room to store
373: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
374: // Get the next leading byte out
375: const XMLByte firstByte =* srcPtr;
376:
377: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 378: if(firstByte<=127) {
1.35 paf 379: *outPtr++= firstByte;
380: srcPtr++;
381: continue;
382: }
383:
384: // See how many trailing src bytes this sequence is going to require
385: const unsigned int trailingBytes = gUTFBytes[firstByte];
386:
387: // If there are not enough source bytes to do this one, then we
388: // are done. Note that we done>= here because we are implicitly
389: // counting the 1 byte we get no matter what.
390: if(srcPtr+trailingBytes>= srcEnd)
391: break;
392:
393: // Looks ok, so lets build up the value
394: uint tmpVal=0;
395: switch(trailingBytes) {
396: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
397: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
398: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
399: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
400: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
401: case 0: tmpVal+=*srcPtr++;
402: break;
403:
404: default:
405: throw Exception(0,
406: 0,
1.49 paf 407: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 408: }
409: tmpVal-=gUTFOffsets[trailingBytes];
410:
411: // If it will fit into a single char, then put it in. Otherwise
412: // fail [*encode it as a surrogate pair. If its not valid, use the
413: // replacement char.*]
414: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 415: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
416: *outPtr++=xlat;
1.49 paf 417: else {
1.50 paf 418: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 419: }
420: } else {
421: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
422: for(uint i=0; i<=trailingBytes; i++)
423: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
424: }
1.1 paf 425: }
1.35 paf 426:
427: // Update the bytes eaten
428: srcLen = srcPtr - srcData;
429:
430: // Return the characters read
431: toFillLen = outPtr - toFill;
1.11 paf 432:
1.29 paf 433: //return srcPtr==srcEnd?(int)toFillLen:-1;
434: /*
435: xmlCharEncodingOutputFunc
436: Returns :
437: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
438: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
439: of ocetes consumed.
440: */
441: return 0;
1.10 paf 442: }
443:
1.85 misha 444: static bool need_escape(XMLByte c){
1.60 misha 445: return
1.66 misha 446: !(
447: (c<=127)
448: && (
1.89 misha 449: pa_isalnum((unsigned char)c)
1.66 misha 450: || strchr("*@-_+./", c)!=0
451: )
452: );
1.60 misha 453: }
454:
1.70 misha 455: // read one UTF8 char and return length of this char (in bytes)
456: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 457: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
458: return 0;
459:
460: firstByte=*srcPtr;
461:
462: if(firstByte<=127){
463: UTF8Char=firstByte;
464: srcPtr++;
465: return 1;
466: }
467:
468: unsigned int trailingBytes=gUTFBytes[firstByte];
469:
470: if(srcPtr+trailingBytes>=srcEnd){
471: return 0; // not enough bytes in source string for reading
472: }
473:
474: uint tmpVal=0;
475: switch(trailingBytes){
476: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
477: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
478: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
479: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
480: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
481: case 0: tmpVal+=*srcPtr++;
482: }
483:
484: tmpVal-=gUTFOffsets[trailingBytes];
485: UTF8Char=tmpVal;
486:
487: return trailingBytes+1;
488: }
489:
1.70 misha 490: // skip UTF8 char and return length of this char (in bytes)
491: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 492: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
493: return 0;
494:
1.63 misha 495: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
496: srcPtr+=trailingBytes;
1.62 misha 497:
498: return trailingBytes;
1.61 misha 499: }
500:
1.85 misha 501: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 502: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 503: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
504: return 0;
505:
506: firstByte=*srcPtr++;
507: UTF8Char=tables.fromTable[firstByte];
508:
509: if(UTF8Char<0x80)
510: return 1;
511: else if(UTF8Char<0x800)
512: return 2;
513: else if(UTF8Char<0x10000)
514: return 3;
515: else if(UTF8Char<0x200000)
516: return 4;
517: else if(UTF8Char<0x4000000)
518: return 5;
519: else if(UTF8Char<= 0x7FFFFFFF)
520: return 6;
521:
522: // will use the replacement character '?'
523: firstByte=0;
524: return 1;
525: }
526:
1.85 misha 527: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
528: size_t dest_length=0;
529:
530: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
531: if(i.getCharSize()==1)
532: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
533: else
534: dest_length+=6; // %uXXXX
1.60 misha 535: }
536:
1.85 misha 537: return dest_length;
1.60 misha 538: }
539:
1.86 moko 540: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
541: const XMLByte* src_end=src+src_length;
542: XMLByte first_byte;
543: XMLCh UTF8_char;
1.85 misha 544: size_t dest_length=0;
545:
1.86 moko 546: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 547: if(char_size==1)
548: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
549: else
550: dest_length+=6; // %uXXXX
1.60 misha 551: }
552:
1.85 misha 553: return dest_length;
554: }
555:
556: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 557: if(!src.length)
1.85 misha 558: return 0;
559:
560: #ifdef PRECALCULATE_DEST_LENGTH
561: if(source_charset.isUTF8())
1.86 moko 562: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 563: else
1.86 moko 564: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 565: #else
566: return src_length*6; // enough for %uXXXX but too memory-hungry
567: #endif
568: }
569:
570: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
571: if(char_size==1) \
572: if(first_byte){ \
573: if(need_escape(first_byte)) \
574: dest_ptr+=sprintf((char*)dest_ptr, "%%%02X", first_byte); /* %XX */ \
575: else \
576: *dest_ptr++=first_byte; /*as is*/ \
577: } else \
578: *dest_ptr++='?'; /* replacement char '?' */ \
579: else \
580: dest_ptr+=sprintf((char*)dest_ptr, "%%u%04X", UTF8_char); // %uXXXX
581:
582:
583: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
584: XMLByte* dest_ptr=dest;
585:
586: // loop until we either run out of input data
587: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
588: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 589:
1.85 misha 590: return dest_ptr - dest;
1.60 misha 591: }
592:
1.85 misha 593: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
594: const XMLByte* src_end=src+src_length;
595: XMLByte* dest_ptr=dest;
596:
597: XMLByte first_byte;
598: XMLCh UTF8_char;
599: uint char_size;
600:
1.86 moko 601: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 602: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
603:
604: return dest_ptr - dest;
605: }
1.60 misha 606:
607: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 608: if(!src.length)
1.60 misha 609: return String::C("", 0);
610:
1.85 misha 611: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
612: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
613:
614: size_t dest_length;
615: if(source_charset.isUTF8())
1.86 moko 616: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 617: else
1.86 moko 618: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 619:
620: if(dest_length>dest_calculated_length)
621: throw Exception(0, 0, "Charset::escape buffer overflow");
622:
623: dest_body[dest_length]=0; // terminator
624: return String::C((char*)dest_body, dest_length);
625: }
626:
627: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 628: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 629: return String::Body(dest.length ? dest.str:0);
630: }
631:
632: String& Charset::escape(const String& src, const Charset& source_charset) {
633: if(src.is_empty())
634: return *new String();
635:
636: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
637: }
638:
639: inline bool need_json_escape(unsigned char c){
640: return strchr("\n\"\\/\t\r\b\f", c)!=0;
641: }
642:
643: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
644: size_t dest_length=0;
645:
646: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
647: if(i.getCharSize()==1)
648: dest_length+=need_json_escape(i.getFirstByte()) ? 2 : 1;
649: else
650: dest_length+=6; // \uXXXX
651: }
652:
653: return dest_length;
654: }
655:
1.86 moko 656: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
657: const XMLByte* src_end=src+src_length;
1.85 misha 658: XMLByte first_byte;
659: XMLCh UTF8_char;
1.60 misha 660: size_t dest_length=0;
661:
1.86 moko 662: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 663: if(char_size==1)
664: dest_length+=(!first_byte/*replacement char '?'*/ || !need_json_escape(first_byte))? 1 : 2;
665: else
666: dest_length+=6; // \uXXXX
1.60 misha 667: }
1.85 misha 668:
669: return dest_length;
670: }
671:
672: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 673: if(!src.length)
1.85 misha 674: return 0;
675:
676: #ifdef PRECALCULATE_DEST_LENGTH
677: if(source_charset.isUTF8())
1.86 moko 678: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 679: else
1.86 moko 680: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 681: #else
1.85 misha 682: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 683: #endif
1.85 misha 684: }
685:
686: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
687: if(char_size==1) \
688: switch(first_byte){ \
689: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
690: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
691: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
692: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
693: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
694: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
695: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
696: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
697: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
698: default : *dest_ptr++=first_byte; \
699: } \
700: else \
701: dest_ptr+=sprintf((char*)dest_ptr, "\\u%04X", UTF8_char); // \uXXXX
702:
703:
704: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
705: XMLByte* dest_ptr=dest;
706:
707: // loop until we either run out of input data
708: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
709: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
710:
711: return dest_ptr - dest;
712: }
713:
714: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
715: const XMLByte* src_end=src+src_length;
716: XMLByte* dest_ptr=dest;
717:
718: XMLByte first_byte;
719: XMLCh UTF8_char;
720: uint char_size;
721:
1.86 moko 722: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 723: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
724:
725: return dest_ptr - dest;
726: }
1.60 misha 727:
1.85 misha 728: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 729: if(!src.length)
1.85 misha 730: return String::C("", 0);
1.60 misha 731:
1.85 misha 732: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
733: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
734:
735: size_t dest_length;
736: if(source_charset.isUTF8())
1.86 moko 737: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 738: else
1.86 moko 739: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 740:
1.85 misha 741: if(dest_length>dest_calculated_length)
742: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 743:
744: dest_body[dest_length]=0; // terminator
745: return String::C((char*)dest_body, dest_length);
746: }
1.85 misha 747:
748: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 749: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 750: return String::Body(dest.length ? dest.str:0);
1.64 misha 751: }
752:
1.85 misha 753: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 754: if(src.is_empty())
1.73 misha 755: return *new String();
1.64 misha 756:
1.85 misha 757: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 758: }
1.60 misha 759:
1.35 paf 760: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 761: int src_length=src.length;
1.60 misha 762:
763: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 764: int dest_length=0;
1.60 misha 765: const XMLByte* srcPtr=(XMLByte*)src.str;
766: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 767: XMLByte firstByte;
768: XMLCh UTF8Char;
769: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 770: dest_length+=charSize;
771: #else
1.85 misha 772: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 773: #endif
774:
1.35 paf 775: #ifndef NDEBUG
1.71 misha 776: int saved_dest_length=dest_length;
1.35 paf 777: #endif
778: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 779:
780: if(::transcodeToUTF8(
1.35 paf 781: (XMLByte *)src.str, src_length,
782: dest_body, dest_length,
1.11 paf 783: tables)<0)
1.43 paf 784: throw Exception(0,
1.10 paf 785: 0,
1.11 paf 786: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 787:
1.60 misha 788: assert(dest_length<=saved_dest_length);
789: dest_body[dest_length]=0; // terminator
1.35 paf 790: return String::C((char*)dest_body, dest_length);
1.10 paf 791: }
1.38 paf 792:
793: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 794: int lo = 0;
795: int hi = table.size - 1;
1.39 paf 796: while(lo<=hi) {
1.38 paf 797: // Calc the mid point of the low and high offset.
1.39 paf 798: const unsigned int i = (lo + hi) / 2;
799:
800: XMLCh cur=table.records[i].from;
801: if(src==cur)
802: return table.records[i].to;
803: if(src>cur)
804: lo = i+1;
1.38 paf 805: else
1.39 paf 806: hi = i-1;
807: }
808:
809: // not found
1.38 paf 810: return src;
811: }
812:
1.58 misha 813: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 814: if(!src) {
815: // use the replacement character
816: *outPtr++= '?';
817: return;
818: }
819:
820: // Figure out how many bytes we need
821: unsigned int encodedBytes;
822: if(src<0x80)
823: encodedBytes = 1;
824: else if(src<0x800)
825: encodedBytes = 2;
826: else if(src<0x10000)
827: encodedBytes = 3;
828: else if(src<0x200000)
829: encodedBytes = 4;
830: else if(src<0x4000000)
831: encodedBytes = 5;
832: else if(src<= 0x7FFFFFFF)
833: encodedBytes = 6;
834: else {
835: // use the replacement character
836: *outPtr++= '?';
837: return;
838: }
839:
840: // And spit out the bytes. We spit them out in reverse order
841: // here, so bump up the output pointer and work down as we go.
842: outPtr+= encodedBytes;
843: switch(encodedBytes) {
844: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
845: src>>= 6;
846: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
847: src>>= 6;
848: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
849: src>>= 6;
850: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
851: src>>= 6;
852: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
853: src>>= 6;
854: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
855: }
856:
857: // Add the encoded bytes back in again to indicate we've eaten them
858: outPtr+= encodedBytes;
859: }
860:
861: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
862: const Charset::UTF8CaseTable& table) {
863: store_UTF8(change_case_UTF8(src, table), outPtr);
864: };
1.44 paf 865: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
866: XMLByte* toFill, size_t toFillLen,
867: const Charset::UTF8CaseTable& table) {
1.38 paf 868: const XMLByte* srcPtr=srcData;
1.44 paf 869: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 870: XMLByte* outPtr=toFill;
1.44 paf 871: XMLByte* outEnd=toFill+toFillLen;
872:
873: // We now loop until we either run out of input data, or room to store
874: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
875: // Get the next leading byte out
876: const XMLByte firstByte =* srcPtr;
1.38 paf 877:
1.60 misha 878: if(firstByte<=127) {
1.38 paf 879: change_case_UTF8(firstByte, outPtr, table);
880: srcPtr++;
881: continue;
882: }
883:
884: // See how many trailing src bytes this sequence is going to require
885: const unsigned int trailingBytes = gUTFBytes[firstByte];
886:
887: // Looks ok, so lets build up the value
888: uint tmpVal=0;
889: switch(trailingBytes) {
890: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
891: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
892: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
893: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
894: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
895: case 0: tmpVal+=*srcPtr++;
896: break;
897:
898: default:
899: throw Exception(0,
900: 0,
901: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
902: }
903: tmpVal-=gUTFOffsets[trailingBytes];
904:
905: // If it will fit into a single char, then put it in. Otherwise
906: // fail [*encode it as a surrogate pair. If its not valid, use the
907: // replacement char.*]
908: if(!(tmpVal & 0xFFFF0000))
909: change_case_UTF8(tmpVal, outPtr, table);
910: else
911: throw Exception(0,
912: 0,
913: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
914: }
915:
916: if(srcPtr!=outPtr)
917: throw Exception(0,
918: 0,
919: "change_case_UTF8 error: end pointers do not match");
920: }
921:
1.60 misha 922: static size_t getDecNumLength(XMLCh UTF8Char){
923: return
924: (UTF8Char < 100)
925: ?2
926: :(UTF8Char < 1000)
927: ?3
928: :(UTF8Char < 10000)
929: ?4
930: :5;
931: }
1.38 paf 932:
1.35 paf 933: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 934: int src_length=src.length;
1.60 misha 935: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 936: int dest_length=0;
1.82 misha 937: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 938: dest_length += ( i.next() & 0xFFFF0000 )
939: ? 3*i.getCharSize() // %XX for each byte
940: : ( xlatOneTo(i.next(), tables, 0) != 0 )
941: ? 1 // can convert it to a single char
942: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 943: }
944: #else
945: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 946: int dest_length=src_length*6;
1.60 misha 947: #endif
948:
1.35 paf 949: #ifndef NDEBUG
1.71 misha 950: int saved_dest_length=dest_length;
1.35 paf 951: #endif
952: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 953:
954: if(::transcodeFromUTF8(
1.82 misha 955: (XMLByte *)src.str, src_length,
1.35 paf 956: dest_body, dest_length,
1.11 paf 957: tables)<0)
1.43 paf 958: throw Exception(0,
1.10 paf 959: 0,
1.35 paf 960: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 961:
1.60 misha 962: assert(dest_length<=saved_dest_length);
963: dest_body[dest_length]=0; // terminator
1.35 paf 964: return String::C((char*)dest_body, dest_length);
1.1 paf 965: }
966:
967: /// transcode using both charsets
1.35 paf 968: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 969: const Charset& dest_charset) const {
1.35 paf 970: if(&dest_charset==this)
971: return src;
972: else {
973: size_t dest_length=src.length;
974: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
975:
976: XMLByte* output=dest_body;
977: const XMLByte* input=(XMLByte *)src.str;
978: while(XMLCh c=*input++) {
979: XMLCh curVal = tables.fromTable[c];
980: *output++=curVal?
981: xlatOneTo(curVal, dest_charset.tables, '?') // OK
982: :'?'; // use the replacement character
1.6 paf 983: }
1.1 paf 984:
1.35 paf 985: dest_body[dest_length]=0; // terminator
986: return String::C((char*)dest_body, dest_length);
1.6 paf 987: }
1.1 paf 988: }
989:
1.58 misha 990: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 991: if(isUTF8())
1.58 misha 992: store_UTF8(src, outPtr);
1.59 misha 993: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 994: *outPtr++=ch;
1.57 misha 995: }
996:
1.1 paf 997: #ifdef XML
1.10 paf 998:
1.35 paf 999: static const Charset::Tables* tables[MAX_CHARSETS];
1000:
1.46 paf 1001: #ifdef PA_PATCHED_LIBXML_BACKWARD
1002:
1003: #define declareXml256ioFuncs(i) \
1004: static int xml256CharEncodingInputFunc##i( \
1005: unsigned char *out, int *outlen, \
1006: const unsigned char *in, int *inlen, void*) { \
1007: return transcodeToUTF8( \
1.71 misha 1008: in, *inlen, \
1009: out, *outlen, \
1.46 paf 1010: *tables[i]); \
1011: } \
1012: static int xml256CharEncodingOutputFunc##i( \
1013: unsigned char *out, int *outlen, \
1014: const unsigned char *in, int *inlen, void*) { \
1015: return transcodeFromUTF8( \
1.71 misha 1016: in, *inlen, \
1017: out, *outlen, \
1.46 paf 1018: *tables[i]); \
1019: }
1020:
1021: #else
1022:
1.35 paf 1023: #define declareXml256ioFuncs(i) \
1024: static int xml256CharEncodingInputFunc##i( \
1025: unsigned char *out, int *outlen, \
1026: const unsigned char *in, int *inlen) { \
1027: return transcodeToUTF8( \
1.71 misha 1028: in, *inlen, \
1029: out, *outlen, \
1.35 paf 1030: *tables[i]); \
1031: } \
1032: static int xml256CharEncodingOutputFunc##i( \
1033: unsigned char *out, int *outlen, \
1034: const unsigned char *in, int *inlen) { \
1035: return transcodeFromUTF8( \
1.71 misha 1036: in, *inlen, \
1037: out, *outlen, \
1.35 paf 1038: *tables[i]); \
1039: }
1040:
1.46 paf 1041: #endif
1042:
1043:
1.35 paf 1044: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1045: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1046: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1047: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1048: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1049:
1050: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1051: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1052: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1053: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1054: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1055: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1056: };
1057: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1058: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1059: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1060: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1061: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1062: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1063: };
1064: static size_t handlers_count=0;
1.10 paf 1065:
1066: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1067: if(handlers_count==MAX_CHARSETS)
1068: throw Exception(0,
1069: 0,
1070: "already allocated %d handlers, no space for new encoding '%s'",
1071: MAX_CHARSETS, name_cstr);
1072:
1.45 paf 1073: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 1074: {
1075: handler->name=name_cstr;
1076: handler->input=inputFuncs[handlers_count];
1077: handler->output=outputFuncs[handlers_count];
1078: ::tables[handlers_count]=&tables;
1079: handlers_count++;
1080: }
1.10 paf 1081:
1082: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1083:
1.10 paf 1084: }
1085:
1.37 paf 1086: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1087: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1088: transcoder(NAME); // check right way
1.15 paf 1089: }
1090:
1.37 paf 1091: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1092: if(!ftranscoder)
1.56 misha 1093: throw Exception(PARSER_RUNTIME,
1.35 paf 1094: new String(NAME, String::L_TAINTED),
1.10 paf 1095: "unsupported encoding");
1.35 paf 1096: return *ftranscoder;
1.10 paf 1097: }
1098:
1.54 paf 1099: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1100: if(!s)
1.35 paf 1101: return String::C("", 0);
1.8 paf 1102:
1.35 paf 1103: int inlen=strlen((const char*)s);
1.51 paf 1104: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1105: #ifndef NDEBUG
1106: int saved_outlen=outlen;
1107: #endif
1108: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1109:
1.30 paf 1110: int error;
1.35 paf 1111: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1112: error=output(
1.17 paf 1113: (unsigned char*)out, &outlen,
1.46 paf 1114: (const unsigned char*)s, &inlen
1115: #ifdef PA_PATCHED_LIBXML_BACKWARD
1116: ,0
1117: #endif
1118: );
1.30 paf 1119: } else {
1120: memcpy(out, s, outlen=inlen);
1121: error=0;
1122: }
1123: if(error<0)
1.23 paf 1124: throw Exception(0,
1.8 paf 1125: 0,
1.30 paf 1126: "transcode_cstr failed (%d)", error);
1.8 paf 1127:
1.35 paf 1128: assert(outlen<=saved_outlen); out[outlen]=0;
1129: return String::C(out, outlen);
1.14 paf 1130: }
1.54 paf 1131: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1132: String::C cstr=transcode_cstr(s);
1.75 misha 1133: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1134: }
1135:
1.8 paf 1136: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1137: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1138: xmlChar* out;
1.30 paf 1139: int outlen;
1140: int error;
1.35 paf 1141: #ifndef NDEBUG
1142: int saved_outlen;
1143: #endif
1144: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1145: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1146: #ifndef NDEBUG
1147: saved_outlen=outlen;
1148: #endif
1.47 paf 1149: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1150: error=input(
1.17 paf 1151: out, &outlen,
1.46 paf 1152: (const unsigned char*)buf, (int*)&buf_size
1153: #ifdef PA_PATCHED_LIBXML_BACKWARD
1154: ,0
1155: #endif
1156: );
1.30 paf 1157: } else {
1158: outlen=buf_size;
1.35 paf 1159: #ifndef NDEBUG
1160: saved_outlen=outlen;
1161: #endif
1162: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1163: memcpy(out, buf, outlen);
1164: error=0;
1165: }
1.17 paf 1166:
1.30 paf 1167: if(error<0)
1.23 paf 1168: throw Exception(0,
1.8 paf 1169: 0,
1.30 paf 1170: "transcode_buf failed (%d)", error);
1.8 paf 1171:
1.35 paf 1172: assert(outlen<=saved_outlen); out[outlen]=0;
1173: return out;
1.24 paf 1174: }
1.1 paf 1175:
1.79 misha 1176: xmlChar* Charset::transcode(const String& s) {
1177: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1178: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1179: }
1.35 paf 1180:
1.79 misha 1181: xmlChar* Charset::transcode(const String::Body s) {
1182: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1183: }
1.36 paf 1184: #endif
1.34 paf 1185:
1.37 paf 1186: String::Body Charset::transcode(const String::Body src,
1.34 paf 1187: const Charset& source_transcoder,
1.35 paf 1188: const Charset& dest_transcoder) {
1.34 paf 1189:
1.35 paf 1190: const char *src_ptr=src.cstr();
1.83 misha 1191: size_t src_size=src.length();
1.34 paf 1192:
1.77 misha 1193: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1194:
1.77 misha 1195: return String::Body(dest.length ? dest.str:0);
1.35 paf 1196: }
1197:
1198: String& Charset::transcode(const String& src,
1199: const Charset& source_transcoder,
1200: const Charset& dest_transcoder) {
1.72 misha 1201: if(src.is_empty())
1.73 misha 1202: return *new String();
1.34 paf 1203:
1.37 paf 1204: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1205: }
1206:
1.35 paf 1207: void Charset::transcode(ArrayString& src,
1.34 paf 1208: const Charset& source_transcoder,
1.35 paf 1209: const Charset& dest_transcoder) {
1210: for(size_t i=0; i<src.count(); i++)
1211: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1212: }
1213:
1214: #ifndef DOXYGEN
1215: struct Transcode_pair_info {
1216: const Charset* source_transcoder;
1217: const Charset* dest_transcoder;
1218: };
1219: #endif
1.76 misha 1220: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1221: String::Body& avalue,
1.35 paf 1222: Transcode_pair_info* info) {
1223: avalue=Charset::transcode(avalue,
1224: *info->source_transcoder,
1225: *info->dest_transcoder);
1.34 paf 1226: }
1.61 misha 1227:
1.35 paf 1228: void Charset::transcode(HashStringString& src,
1.34 paf 1229: const Charset& source_transcoder,
1.35 paf 1230: const Charset& dest_transcoder) {
1231: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1232: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1233: }
1.61 misha 1234:
1235: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1236: const XMLByte* ptr=srcBegin;
1.70 misha 1237: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1238:
1239: return ptr-srcBegin;
1240: }
1241:
1242: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1243: size_t charPos=0;
1244: const XMLByte* ptr=srcBegin;
1245: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1246: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1247: if(ptr>ptrEnd)
1248: return charPos;
1249: charPos++;
1250: }
1251:
1252: // scan till end but position in bytes still too low
1253: throw Exception(0,
1254: 0,
1255: "Error convertion byte pos to char pos");
1256: }
1257:
1258: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1259: size_t size=0;
1.70 misha 1260: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1261: size++;
1262:
1263: return size;
1264: }
1.80 misha 1265:
1.84 misha 1266: unsigned int lengthUTF8Char(const XMLByte c){
1267: return gUTFBytes[c]+1;
1268: }
1269:
1.80 misha 1270: bool UTF8_string_iterator::has_next(){
1271: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1272: return fcharSize!=0;
1273: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main