Annotation of parser3/src/main/pa_charset.C, revision 1.87
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.68 misha 4: Copyright(c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.87 ! moko 8: static const char * const IDENT_CHARSET_C="$Date: 2010-08-29 21:23:40 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.35 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 18:
19: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 20: #define PRECALCULATE_DEST_LENGTH
1.46 paf 21:
1.38 paf 22: // globals
23:
24: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
25: #include "utf8-to-upper.inc"
26: };
27: Charset::UTF8CaseTable UTF8CaseToUpper={
28: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
29: UTF8CaseToUpperRecords};
30:
31: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
32: #include "utf8-to-lower.inc"
33: };
34: Charset::UTF8CaseTable UTF8CaseToLower={
35: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
36: UTF8CaseToLowerRecords};
37:
1.1 paf 38: // helpers
39:
40: inline void prepare_case_tables(unsigned char *tables) {
41: unsigned char *lcc_table=tables+lcc_offset;
42: unsigned char *fcc_table=tables+fcc_offset;
43: for(int i=0; i<0x100; i++)
1.53 paf 44: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 45: }
46: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
47: unsigned char bit) {
48: unsigned char *ctypes_table=tables+ctypes_offset;
49: ctypes_table[0]=bit;
50: for(; *cstr; cstr++) {
51: unsigned char c=*cstr;
52: ctypes_table[c]|=bit;
53: }
54: }
1.35 paf 55: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 56: if(!cstr || !*cstr)
57: return 0;
58: if(cstr[1]==0)
1.4 paf 59: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 60:
61: char *error_pos;
1.4 paf 62: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 63: }
1.35 paf 64: inline bool to_bool(const char* cstr) {
1.1 paf 65: return cstr && *cstr!=0;
66: }
67: static void element2ctypes(unsigned char c, bool belongs,
68: unsigned char *tables, unsigned char bit, int group_offset=-1) {
69: if(!belongs)
70: return;
71:
72: unsigned char *ctypes_table=tables+ctypes_offset;
73:
74: ctypes_table[c]|=bit;
75: if(group_offset>=0)
1.4 paf 76: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 77: }
78: static void element2case(unsigned char from, unsigned char to,
79: unsigned char *tables) {
80: if(!to)
81: return;
82:
83: unsigned char *lcc_table=tables+lcc_offset;
84: unsigned char *fcc_table=tables+fcc_offset;
85: lcc_table[from]=to;
86: fcc_table[from]=to; fcc_table[to]=from;
87: }
88:
89: // methods
90:
1.37 paf 91: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 92: FNAME(ANAME),
93: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 94:
1.35 paf 95: if(afile_spec) {
1.1 paf 96: fisUTF8=false;
1.35 paf 97: load_definition(*charsets, *afile_spec);
1.1 paf 98: #ifdef XML
1.35 paf 99: addEncoding(FNAME_CSTR);
1.1 paf 100: #endif
101: } else {
102: fisUTF8=true;
1.4 paf 103: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 104: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 105: }
106:
107: #ifdef XML
1.35 paf 108: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 109: #endif
110: }
111:
1.35 paf 112: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 113: // pcre_tables
114: // lowcase, flipcase, bits digit+word+whitespace, masks
115:
116: // must not move this inside of prepare_case_tables
117: // don't know the size there
118: memset(pcre_tables, 0, sizeof(pcre_tables));
119: prepare_case_tables(pcre_tables);
1.4 paf 120: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 121:
122: // charset
1.35 paf 123: memset(&tables, 0, sizeof(tables));
1.1 paf 124:
125: // loading text
1.35 paf 126: char *data=file_read_text(charsets, afile_spec);
1.1 paf 127:
128: // ignore header
129: getrow(&data);
130:
131: // parse cells
132: char *row;
1.42 paf 133: while((row=getrow(&data))) {
1.1 paf 134: // remove empty&comment lines
135: if(!*row || *row=='#')
136: continue;
137:
138: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 139: unsigned char c=0;
1.1 paf 140: char *cell;
1.42 paf 141: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 142: switch(column) {
1.53 paf 143: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 144: // pcre_tables
145: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
146: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
147: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
148: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
149: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 150: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 151: case 7:
152: case 8:
153: // charset
1.10 paf 154: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 155: throw Exception(PARSER_RUNTIME,
1.35 paf 156: &afile_spec,
1.1 paf 157: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
158:
159: XMLCh unicode=(XMLCh)to_wchar_code(cell);
160: if(!unicode && column==7/*unicode1 column*/)
161: unicode=(XMLCh)c;
162: if(unicode) {
1.10 paf 163: if(!tables.fromTable[c])
164: tables.fromTable[c]=unicode;
165: tables.toTable[tables.toTableSize].intCh=unicode;
166: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
167: tables.toTableSize++;
1.1 paf 168: }
169: break;
170: }
171: }
172: };
173:
1.87 ! moko 174: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
! 175: for(uint i=0; i<0x20; i++)
! 176: if(!tables.fromTable[i]){
! 177: tables.fromTable[i]=i;
! 178: tables.toTable[tables.toTableSize].intCh=i;
! 179: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
! 180: tables.toTableSize++;
! 181: }
! 182:
1.1 paf 183: // sort by the Unicode code point
184: sort_ToTable();
185: }
186:
187: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
188: return
1.38 paf 189: static_cast<const Charset::Tables::Rec *>(a)->intCh-
190: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 191: }
192:
193: void Charset::sort_ToTable() {
1.10 paf 194: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 195: sort_cmp_Trans_rec_intCh);
196: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 197: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 198: //fclose(f);
199: }
200:
1.60 misha 201: // @todo: precache for spedup searching
1.10 paf 202: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 203: const Charset::Tables& tables,
204: XMLByte not_found) {
1.80 misha 205: int lo = 0;
206: int hi = tables.toTableSize - 1;
1.39 paf 207: while(lo<=hi) {
1.35 paf 208: // Calc the mid point of the low and high offset.
1.39 paf 209: const unsigned int i = (lo + hi) / 2;
210:
211: XMLCh cur=tables.toTable[i].intCh;
212: if(toXlat==cur)
213: return tables.toTable[i].extCh;
214: if(toXlat>cur)
215: lo = i+1;
1.1 paf 216: else
1.39 paf 217: hi = i-1;
218: }
1.35 paf 219:
220: return not_found;
1.1 paf 221: }
222:
1.35 paf 223: String::C Charset::transcode(const String::C src,
224: const Charset& source_charset,
225: const Charset& dest_charset) {
226: if(!src.length)
227: return String::C("", 0);
1.4 paf 228:
1.1 paf 229: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
230: default: // 0x00
1.35 paf 231: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 232: case 0x01:
1.35 paf 233: return source_charset.transcodeToUTF8(src);
1.1 paf 234: case 0x10:
1.35 paf 235: return dest_charset.transcodeFromUTF8(src);
1.1 paf 236: case 0x11:
1.35 paf 237: return src;
1.1 paf 238: }
239: }
240:
241: // ---------------------------------------------------------------------------
242: // Local static data
243: //
244: // gUTFBytes
245: // A list of counts of trailing bytes for each initial byte in the input.
246: //
247: // gUTFOffsets
248: // A list of values to offset each result char type, according to how
249: // many source bytes when into making it.
250: //
251: // gFirstByteMark
252: // A list of values to mask onto the first byte of an encoded sequence,
253: // indexed by the number of bytes used to create the sequence.
254: // ---------------------------------------------------------------------------
255: static const XMLByte gUTFBytes[0x100] = {
256: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
257: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
258: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
259: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
260: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
261: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
262: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
263: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
264: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
265: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
266: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
267: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
268: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
269: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
270: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
271: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
272: };
273:
274: static const uint gUTFOffsets[6] = {
1.80 misha 275: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 276: };
277:
278: static const XMLByte gFirstByteMark[7] = {
1.80 misha 279: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 280: };
281:
1.71 misha 282: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
283: XMLByte *toFill, int& toFillLen,
284: const Charset::Tables& tables) {
1.11 paf 285: const XMLByte* srcPtr=srcData;
286: const XMLByte* srcEnd=srcData+srcLen;
287: XMLByte* outPtr=toFill;
288: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 289:
1.35 paf 290: while(srcPtr<srcEnd) {
291: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 292: if(!curVal) {
1.35 paf 293: // use the replacement character
294: *outPtr++= '?';
295: srcPtr++;
296: continue;
297: }
1.1 paf 298:
1.35 paf 299: // Figure out how many bytes we need
300: unsigned int encodedBytes;
301: if(curVal<0x80)
302: encodedBytes = 1;
303: else if(curVal<0x800)
304: encodedBytes = 2;
305: else if(curVal<0x10000)
306: encodedBytes = 3;
307: else if(curVal<0x200000)
308: encodedBytes = 4;
309: else if(curVal<0x4000000)
310: encodedBytes = 5;
311: else if(curVal<= 0x7FFFFFFF)
312: encodedBytes = 6;
313: else {
314: // use the replacement character
315: *outPtr++= '?';
316: srcPtr++;
317: continue;
318: }
1.11 paf 319:
1.35 paf 320: // If we cannot fully get this char into the output buffer
321: if (outPtr + encodedBytes > outEnd)
322: break;
323:
324: // We can do it, so update the source index
325: srcPtr++;
326:
327: // And spit out the bytes. We spit them out in reverse order
328: // here, so bump up the output pointer and work down as we go.
329: outPtr+= encodedBytes;
330: switch(encodedBytes) {
1.60 misha 331: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
332: curVal>>= 6;
333: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
334: curVal>>= 6;
335: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
336: curVal>>= 6;
337: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
338: curVal>>= 6;
339: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
340: curVal>>= 6;
341: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 342: }
343:
344: // Add the encoded bytes back in again to indicate we've eaten them
345: outPtr+= encodedBytes;
346: }
347:
348: // Update the bytes eaten
349: srcLen = srcPtr - srcData;
350:
351: // Return the characters read
352: toFillLen = outPtr - toFill;
353:
1.29 paf 354: //return srcPtr==srcEnd?(int)toFillLen:-1;
355: /*
356: xmlCharEncodingInputFunc
357: Returns :
358: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
359: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
360: of ocetes consumed.
361: */
362: return 0;
1.1 paf 363: }
1.26 paf 364: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 365: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
366: XMLByte* toFill, int& toFillLen,
367: const Charset::Tables& tables) {
1.11 paf 368: const XMLByte* srcPtr=srcData;
369: const XMLByte* srcEnd=srcData+srcLen;
370: XMLByte* outPtr=toFill;
371: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 372:
1.35 paf 373: // We now loop until we either run out of input data, or room to store
374: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
375: // Get the next leading byte out
376: const XMLByte firstByte =* srcPtr;
377:
378: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 379: if(firstByte<=127) {
1.35 paf 380: *outPtr++= firstByte;
381: srcPtr++;
382: continue;
383: }
384:
385: // See how many trailing src bytes this sequence is going to require
386: const unsigned int trailingBytes = gUTFBytes[firstByte];
387:
388: // If there are not enough source bytes to do this one, then we
389: // are done. Note that we done>= here because we are implicitly
390: // counting the 1 byte we get no matter what.
391: if(srcPtr+trailingBytes>= srcEnd)
392: break;
393:
394: // Looks ok, so lets build up the value
395: uint tmpVal=0;
396: switch(trailingBytes) {
397: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
398: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
399: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
400: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
401: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
402: case 0: tmpVal+=*srcPtr++;
403: break;
404:
405: default:
406: throw Exception(0,
407: 0,
1.49 paf 408: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 409: }
410: tmpVal-=gUTFOffsets[trailingBytes];
411:
412: // If it will fit into a single char, then put it in. Otherwise
413: // fail [*encode it as a surrogate pair. If its not valid, use the
414: // replacement char.*]
415: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 416: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
417: *outPtr++=xlat;
1.49 paf 418: else {
1.50 paf 419: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 420: }
421: } else {
422: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
423: for(uint i=0; i<=trailingBytes; i++)
424: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
425: }
1.1 paf 426: }
1.35 paf 427:
428: // Update the bytes eaten
429: srcLen = srcPtr - srcData;
430:
431: // Return the characters read
432: toFillLen = outPtr - toFill;
1.11 paf 433:
1.29 paf 434: //return srcPtr==srcEnd?(int)toFillLen:-1;
435: /*
436: xmlCharEncodingOutputFunc
437: Returns :
438: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
439: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
440: of ocetes consumed.
441: */
442: return 0;
1.10 paf 443: }
444:
1.85 misha 445: static bool need_escape(XMLByte c){
1.60 misha 446: return
1.66 misha 447: !(
448: (c<=127)
449: && (
450: ((c>='0') && (c<='9'))
451: || ((c>='A') && (c<='Z'))
452: || ((c>='a') && (c<='z'))
453: || strchr("*@-_+./", c)!=0
454: )
455: );
1.60 misha 456: }
457:
1.70 misha 458: // read one UTF8 char and return length of this char (in bytes)
459: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 460: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
461: return 0;
462:
463: firstByte=*srcPtr;
464:
465: if(firstByte<=127){
466: UTF8Char=firstByte;
467: srcPtr++;
468: return 1;
469: }
470:
471: unsigned int trailingBytes=gUTFBytes[firstByte];
472:
473: if(srcPtr+trailingBytes>=srcEnd){
474: return 0; // not enough bytes in source string for reading
475: }
476:
477: uint tmpVal=0;
478: switch(trailingBytes){
479: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
480: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
481: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
482: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
483: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
484: case 0: tmpVal+=*srcPtr++;
485: }
486:
487: tmpVal-=gUTFOffsets[trailingBytes];
488: UTF8Char=tmpVal;
489:
490: return trailingBytes+1;
491: }
492:
1.70 misha 493: // skip UTF8 char and return length of this char (in bytes)
494: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 495: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
496: return 0;
497:
1.63 misha 498: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
499: srcPtr+=trailingBytes;
1.62 misha 500:
501: return trailingBytes;
1.61 misha 502: }
503:
1.85 misha 504: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 505: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 506: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
507: return 0;
508:
509: firstByte=*srcPtr++;
510: UTF8Char=tables.fromTable[firstByte];
511:
512: if(UTF8Char<0x80)
513: return 1;
514: else if(UTF8Char<0x800)
515: return 2;
516: else if(UTF8Char<0x10000)
517: return 3;
518: else if(UTF8Char<0x200000)
519: return 4;
520: else if(UTF8Char<0x4000000)
521: return 5;
522: else if(UTF8Char<= 0x7FFFFFFF)
523: return 6;
524:
525: // will use the replacement character '?'
526: firstByte=0;
527: return 1;
528: }
529:
1.85 misha 530: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
531: size_t dest_length=0;
532:
533: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
534: if(i.getCharSize()==1)
535: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
536: else
537: dest_length+=6; // %uXXXX
1.60 misha 538: }
539:
1.85 misha 540: return dest_length;
1.60 misha 541: }
542:
1.86 moko 543: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
544: const XMLByte* src_end=src+src_length;
545: XMLByte first_byte;
546: XMLCh UTF8_char;
1.85 misha 547: size_t dest_length=0;
548:
1.86 moko 549: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 550: if(char_size==1)
551: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
552: else
553: dest_length+=6; // %uXXXX
1.60 misha 554: }
555:
1.85 misha 556: return dest_length;
557: }
558:
559: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 560: if(!src.length)
1.85 misha 561: return 0;
562:
563: #ifdef PRECALCULATE_DEST_LENGTH
564: if(source_charset.isUTF8())
1.86 moko 565: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 566: else
1.86 moko 567: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 568: #else
569: return src_length*6; // enough for %uXXXX but too memory-hungry
570: #endif
571: }
572:
573: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
574: if(char_size==1) \
575: if(first_byte){ \
576: if(need_escape(first_byte)) \
577: dest_ptr+=sprintf((char*)dest_ptr, "%%%02X", first_byte); /* %XX */ \
578: else \
579: *dest_ptr++=first_byte; /*as is*/ \
580: } else \
581: *dest_ptr++='?'; /* replacement char '?' */ \
582: else \
583: dest_ptr+=sprintf((char*)dest_ptr, "%%u%04X", UTF8_char); // %uXXXX
584:
585:
586: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
587: XMLByte* dest_ptr=dest;
588:
589: // loop until we either run out of input data
590: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
591: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 592:
1.85 misha 593: return dest_ptr - dest;
1.60 misha 594: }
595:
1.85 misha 596: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
597: const XMLByte* src_end=src+src_length;
598: XMLByte* dest_ptr=dest;
599:
600: XMLByte first_byte;
601: XMLCh UTF8_char;
602: uint char_size;
603:
1.86 moko 604: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 605: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
606:
607: return dest_ptr - dest;
608: }
1.60 misha 609:
610: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 611: if(!src.length)
1.60 misha 612: return String::C("", 0);
613:
1.85 misha 614: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
615: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
616:
617: size_t dest_length;
618: if(source_charset.isUTF8())
1.86 moko 619: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 620: else
1.86 moko 621: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 622:
623: if(dest_length>dest_calculated_length)
624: throw Exception(0, 0, "Charset::escape buffer overflow");
625:
626: dest_body[dest_length]=0; // terminator
627: return String::C((char*)dest_body, dest_length);
628: }
629:
630: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 631: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 632: return String::Body(dest.length ? dest.str:0);
633: }
634:
635: String& Charset::escape(const String& src, const Charset& source_charset) {
636: if(src.is_empty())
637: return *new String();
638:
639: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
640: }
641:
642: inline bool need_json_escape(unsigned char c){
643: return strchr("\n\"\\/\t\r\b\f", c)!=0;
644: }
645:
646: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
647: size_t dest_length=0;
648:
649: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
650: if(i.getCharSize()==1)
651: dest_length+=need_json_escape(i.getFirstByte()) ? 2 : 1;
652: else
653: dest_length+=6; // \uXXXX
654: }
655:
656: return dest_length;
657: }
658:
1.86 moko 659: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
660: const XMLByte* src_end=src+src_length;
1.85 misha 661: XMLByte first_byte;
662: XMLCh UTF8_char;
1.60 misha 663: size_t dest_length=0;
664:
1.86 moko 665: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 666: if(char_size==1)
667: dest_length+=(!first_byte/*replacement char '?'*/ || !need_json_escape(first_byte))? 1 : 2;
668: else
669: dest_length+=6; // \uXXXX
1.60 misha 670: }
1.85 misha 671:
672: return dest_length;
673: }
674:
675: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 676: if(!src.length)
1.85 misha 677: return 0;
678:
679: #ifdef PRECALCULATE_DEST_LENGTH
680: if(source_charset.isUTF8())
1.86 moko 681: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 682: else
1.86 moko 683: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 684: #else
1.85 misha 685: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 686: #endif
1.85 misha 687: }
688:
689: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
690: if(char_size==1) \
691: switch(first_byte){ \
692: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
693: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
694: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
695: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
696: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
697: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
698: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
699: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
700: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
701: default : *dest_ptr++=first_byte; \
702: } \
703: else \
704: dest_ptr+=sprintf((char*)dest_ptr, "\\u%04X", UTF8_char); // \uXXXX
705:
706:
707: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
708: XMLByte* dest_ptr=dest;
709:
710: // loop until we either run out of input data
711: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
712: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
713:
714: return dest_ptr - dest;
715: }
716:
717: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
718: const XMLByte* src_end=src+src_length;
719: XMLByte* dest_ptr=dest;
720:
721: XMLByte first_byte;
722: XMLCh UTF8_char;
723: uint char_size;
724:
1.86 moko 725: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 726: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
727:
728: return dest_ptr - dest;
729: }
1.60 misha 730:
1.85 misha 731: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 732: if(!src.length)
1.85 misha 733: return String::C("", 0);
1.60 misha 734:
1.85 misha 735: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
736: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
737:
738: size_t dest_length;
739: if(source_charset.isUTF8())
1.86 moko 740: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 741: else
1.86 moko 742: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 743:
1.85 misha 744: if(dest_length>dest_calculated_length)
745: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 746:
747: dest_body[dest_length]=0; // terminator
748: return String::C((char*)dest_body, dest_length);
749: }
1.85 misha 750:
751: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 752: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 753: return String::Body(dest.length ? dest.str:0);
1.64 misha 754: }
755:
1.85 misha 756: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 757: if(src.is_empty())
1.73 misha 758: return *new String();
1.64 misha 759:
1.85 misha 760: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 761: }
1.60 misha 762:
1.35 paf 763: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 764: int src_length=src.length;
1.60 misha 765:
766: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 767: int dest_length=0;
1.60 misha 768: const XMLByte* srcPtr=(XMLByte*)src.str;
769: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 770: XMLByte firstByte;
771: XMLCh UTF8Char;
772: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 773: dest_length+=charSize;
774: #else
1.85 misha 775: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 776: #endif
777:
1.35 paf 778: #ifndef NDEBUG
1.71 misha 779: int saved_dest_length=dest_length;
1.35 paf 780: #endif
781: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 782:
783: if(::transcodeToUTF8(
1.35 paf 784: (XMLByte *)src.str, src_length,
785: dest_body, dest_length,
1.11 paf 786: tables)<0)
1.43 paf 787: throw Exception(0,
1.10 paf 788: 0,
1.11 paf 789: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 790:
1.60 misha 791: assert(dest_length<=saved_dest_length);
792: dest_body[dest_length]=0; // terminator
1.35 paf 793: return String::C((char*)dest_body, dest_length);
1.10 paf 794: }
1.38 paf 795:
796: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 797: int lo = 0;
798: int hi = table.size - 1;
1.39 paf 799: while(lo<=hi) {
1.38 paf 800: // Calc the mid point of the low and high offset.
1.39 paf 801: const unsigned int i = (lo + hi) / 2;
802:
803: XMLCh cur=table.records[i].from;
804: if(src==cur)
805: return table.records[i].to;
806: if(src>cur)
807: lo = i+1;
1.38 paf 808: else
1.39 paf 809: hi = i-1;
810: }
811:
812: // not found
1.38 paf 813: return src;
814: }
815:
1.58 misha 816: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 817: if(!src) {
818: // use the replacement character
819: *outPtr++= '?';
820: return;
821: }
822:
823: // Figure out how many bytes we need
824: unsigned int encodedBytes;
825: if(src<0x80)
826: encodedBytes = 1;
827: else if(src<0x800)
828: encodedBytes = 2;
829: else if(src<0x10000)
830: encodedBytes = 3;
831: else if(src<0x200000)
832: encodedBytes = 4;
833: else if(src<0x4000000)
834: encodedBytes = 5;
835: else if(src<= 0x7FFFFFFF)
836: encodedBytes = 6;
837: else {
838: // use the replacement character
839: *outPtr++= '?';
840: return;
841: }
842:
843: // And spit out the bytes. We spit them out in reverse order
844: // here, so bump up the output pointer and work down as we go.
845: outPtr+= encodedBytes;
846: switch(encodedBytes) {
847: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
848: src>>= 6;
849: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
850: src>>= 6;
851: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
852: src>>= 6;
853: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
854: src>>= 6;
855: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
856: src>>= 6;
857: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
858: }
859:
860: // Add the encoded bytes back in again to indicate we've eaten them
861: outPtr+= encodedBytes;
862: }
863:
864: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
865: const Charset::UTF8CaseTable& table) {
866: store_UTF8(change_case_UTF8(src, table), outPtr);
867: };
1.44 paf 868: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
869: XMLByte* toFill, size_t toFillLen,
870: const Charset::UTF8CaseTable& table) {
1.38 paf 871: const XMLByte* srcPtr=srcData;
1.44 paf 872: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 873: XMLByte* outPtr=toFill;
1.44 paf 874: XMLByte* outEnd=toFill+toFillLen;
875:
876: // We now loop until we either run out of input data, or room to store
877: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
878: // Get the next leading byte out
879: const XMLByte firstByte =* srcPtr;
1.38 paf 880:
1.60 misha 881: if(firstByte<=127) {
1.38 paf 882: change_case_UTF8(firstByte, outPtr, table);
883: srcPtr++;
884: continue;
885: }
886:
887: // See how many trailing src bytes this sequence is going to require
888: const unsigned int trailingBytes = gUTFBytes[firstByte];
889:
890: // Looks ok, so lets build up the value
891: uint tmpVal=0;
892: switch(trailingBytes) {
893: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
894: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
895: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
896: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
897: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
898: case 0: tmpVal+=*srcPtr++;
899: break;
900:
901: default:
902: throw Exception(0,
903: 0,
904: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
905: }
906: tmpVal-=gUTFOffsets[trailingBytes];
907:
908: // If it will fit into a single char, then put it in. Otherwise
909: // fail [*encode it as a surrogate pair. If its not valid, use the
910: // replacement char.*]
911: if(!(tmpVal & 0xFFFF0000))
912: change_case_UTF8(tmpVal, outPtr, table);
913: else
914: throw Exception(0,
915: 0,
916: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
917: }
918:
919: if(srcPtr!=outPtr)
920: throw Exception(0,
921: 0,
922: "change_case_UTF8 error: end pointers do not match");
923: }
924:
1.60 misha 925: static size_t getDecNumLength(XMLCh UTF8Char){
926: return
927: (UTF8Char < 100)
928: ?2
929: :(UTF8Char < 1000)
930: ?3
931: :(UTF8Char < 10000)
932: ?4
933: :5;
934: }
1.38 paf 935:
1.35 paf 936: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 937: int src_length=src.length;
1.60 misha 938: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 939: int dest_length=0;
1.82 misha 940: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.81 misha 941: if(i.getCharSize()==1)
1.60 misha 942: dest_length++;
943: else
1.81 misha 944: dest_length+=(i.next() & 0xFFFF0000)
945: ?i.getCharSize()*3 // '%XX' for each byte
946: :(xlatOneTo(i.next(), tables, 0)!=0)
1.60 misha 947: ?1 // can convert it to single char
1.81 misha 948: :getDecNumLength(i.next())+3; // &#XX; - &#XXXXX;
1.60 misha 949: }
950: #else
951: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 952: int dest_length=src_length*6;
1.60 misha 953: #endif
954:
1.35 paf 955: #ifndef NDEBUG
1.71 misha 956: int saved_dest_length=dest_length;
1.35 paf 957: #endif
958: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 959:
960: if(::transcodeFromUTF8(
1.82 misha 961: (XMLByte *)src.str, src_length,
1.35 paf 962: dest_body, dest_length,
1.11 paf 963: tables)<0)
1.43 paf 964: throw Exception(0,
1.10 paf 965: 0,
1.35 paf 966: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 967:
1.60 misha 968: assert(dest_length<=saved_dest_length);
969: dest_body[dest_length]=0; // terminator
1.35 paf 970: return String::C((char*)dest_body, dest_length);
1.1 paf 971: }
972:
973: /// transcode using both charsets
1.35 paf 974: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 975: const Charset& dest_charset) const {
1.35 paf 976: if(&dest_charset==this)
977: return src;
978: else {
979: size_t dest_length=src.length;
980: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
981:
982: XMLByte* output=dest_body;
983: const XMLByte* input=(XMLByte *)src.str;
984: while(XMLCh c=*input++) {
985: XMLCh curVal = tables.fromTable[c];
986: *output++=curVal?
987: xlatOneTo(curVal, dest_charset.tables, '?') // OK
988: :'?'; // use the replacement character
1.6 paf 989: }
1.1 paf 990:
1.35 paf 991: dest_body[dest_length]=0; // terminator
992: return String::C((char*)dest_body, dest_length);
1.6 paf 993: }
1.1 paf 994: }
995:
1.58 misha 996: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 997: if(isUTF8())
1.58 misha 998: store_UTF8(src, outPtr);
1.59 misha 999: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1000: *outPtr++=ch;
1.57 misha 1001: }
1002:
1.1 paf 1003: #ifdef XML
1.10 paf 1004:
1.35 paf 1005: static const Charset::Tables* tables[MAX_CHARSETS];
1006:
1.46 paf 1007: #ifdef PA_PATCHED_LIBXML_BACKWARD
1008:
1009: #define declareXml256ioFuncs(i) \
1010: static int xml256CharEncodingInputFunc##i( \
1011: unsigned char *out, int *outlen, \
1012: const unsigned char *in, int *inlen, void*) { \
1013: return transcodeToUTF8( \
1.71 misha 1014: in, *inlen, \
1015: out, *outlen, \
1.46 paf 1016: *tables[i]); \
1017: } \
1018: static int xml256CharEncodingOutputFunc##i( \
1019: unsigned char *out, int *outlen, \
1020: const unsigned char *in, int *inlen, void*) { \
1021: return transcodeFromUTF8( \
1.71 misha 1022: in, *inlen, \
1023: out, *outlen, \
1.46 paf 1024: *tables[i]); \
1025: }
1026:
1027: #else
1028:
1.35 paf 1029: #define declareXml256ioFuncs(i) \
1030: static int xml256CharEncodingInputFunc##i( \
1031: unsigned char *out, int *outlen, \
1032: const unsigned char *in, int *inlen) { \
1033: return transcodeToUTF8( \
1.71 misha 1034: in, *inlen, \
1035: out, *outlen, \
1.35 paf 1036: *tables[i]); \
1037: } \
1038: static int xml256CharEncodingOutputFunc##i( \
1039: unsigned char *out, int *outlen, \
1040: const unsigned char *in, int *inlen) { \
1041: return transcodeFromUTF8( \
1.71 misha 1042: in, *inlen, \
1043: out, *outlen, \
1.35 paf 1044: *tables[i]); \
1045: }
1046:
1.46 paf 1047: #endif
1048:
1049:
1.35 paf 1050: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1051: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1052: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1053: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1054: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1055:
1056: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1057: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1058: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1059: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1060: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1061: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1062: };
1063: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1064: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1065: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1066: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1067: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1068: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1069: };
1070: static size_t handlers_count=0;
1.10 paf 1071:
1072: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1073: if(handlers_count==MAX_CHARSETS)
1074: throw Exception(0,
1075: 0,
1076: "already allocated %d handlers, no space for new encoding '%s'",
1077: MAX_CHARSETS, name_cstr);
1078:
1.45 paf 1079: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 1080: {
1081: handler->name=name_cstr;
1082: handler->input=inputFuncs[handlers_count];
1083: handler->output=outputFuncs[handlers_count];
1084: ::tables[handlers_count]=&tables;
1085: handlers_count++;
1086: }
1.10 paf 1087:
1088: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1089:
1.10 paf 1090: }
1091:
1.37 paf 1092: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1093: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1094: transcoder(NAME); // check right way
1.15 paf 1095: }
1096:
1.37 paf 1097: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1098: if(!ftranscoder)
1.56 misha 1099: throw Exception(PARSER_RUNTIME,
1.35 paf 1100: new String(NAME, String::L_TAINTED),
1.10 paf 1101: "unsupported encoding");
1.35 paf 1102: return *ftranscoder;
1.10 paf 1103: }
1104:
1.54 paf 1105: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1106: if(!s)
1.35 paf 1107: return String::C("", 0);
1.8 paf 1108:
1.35 paf 1109: int inlen=strlen((const char*)s);
1.51 paf 1110: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1111: #ifndef NDEBUG
1112: int saved_outlen=outlen;
1113: #endif
1114: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1115:
1.30 paf 1116: int error;
1.35 paf 1117: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1118: error=output(
1.17 paf 1119: (unsigned char*)out, &outlen,
1.46 paf 1120: (const unsigned char*)s, &inlen
1121: #ifdef PA_PATCHED_LIBXML_BACKWARD
1122: ,0
1123: #endif
1124: );
1.30 paf 1125: } else {
1126: memcpy(out, s, outlen=inlen);
1127: error=0;
1128: }
1129: if(error<0)
1.23 paf 1130: throw Exception(0,
1.8 paf 1131: 0,
1.30 paf 1132: "transcode_cstr failed (%d)", error);
1.8 paf 1133:
1.35 paf 1134: assert(outlen<=saved_outlen); out[outlen]=0;
1135: return String::C(out, outlen);
1.14 paf 1136: }
1.54 paf 1137: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1138: String::C cstr=transcode_cstr(s);
1.75 misha 1139: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1140: }
1141:
1.8 paf 1142: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1143: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1144: xmlChar* out;
1.30 paf 1145: int outlen;
1146: int error;
1.35 paf 1147: #ifndef NDEBUG
1148: int saved_outlen;
1149: #endif
1150: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1151: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1152: #ifndef NDEBUG
1153: saved_outlen=outlen;
1154: #endif
1.47 paf 1155: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1156: error=input(
1.17 paf 1157: out, &outlen,
1.46 paf 1158: (const unsigned char*)buf, (int*)&buf_size
1159: #ifdef PA_PATCHED_LIBXML_BACKWARD
1160: ,0
1161: #endif
1162: );
1.30 paf 1163: } else {
1164: outlen=buf_size;
1.35 paf 1165: #ifndef NDEBUG
1166: saved_outlen=outlen;
1167: #endif
1168: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1169: memcpy(out, buf, outlen);
1170: error=0;
1171: }
1.17 paf 1172:
1.30 paf 1173: if(error<0)
1.23 paf 1174: throw Exception(0,
1.8 paf 1175: 0,
1.30 paf 1176: "transcode_buf failed (%d)", error);
1.8 paf 1177:
1.35 paf 1178: assert(outlen<=saved_outlen); out[outlen]=0;
1179: return out;
1.24 paf 1180: }
1.1 paf 1181:
1.79 misha 1182: xmlChar* Charset::transcode(const String& s) {
1183: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1184: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1185: }
1.35 paf 1186:
1.79 misha 1187: xmlChar* Charset::transcode(const String::Body s) {
1188: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1189: }
1.36 paf 1190: #endif
1.34 paf 1191:
1.37 paf 1192: String::Body Charset::transcode(const String::Body src,
1.34 paf 1193: const Charset& source_transcoder,
1.35 paf 1194: const Charset& dest_transcoder) {
1.34 paf 1195:
1.35 paf 1196: const char *src_ptr=src.cstr();
1.83 misha 1197: size_t src_size=src.length();
1.34 paf 1198:
1.77 misha 1199: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1200:
1.77 misha 1201: return String::Body(dest.length ? dest.str:0);
1.35 paf 1202: }
1203:
1204: String& Charset::transcode(const String& src,
1205: const Charset& source_transcoder,
1206: const Charset& dest_transcoder) {
1.72 misha 1207: if(src.is_empty())
1.73 misha 1208: return *new String();
1.34 paf 1209:
1.37 paf 1210: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1211: }
1212:
1.35 paf 1213: void Charset::transcode(ArrayString& src,
1.34 paf 1214: const Charset& source_transcoder,
1.35 paf 1215: const Charset& dest_transcoder) {
1216: for(size_t i=0; i<src.count(); i++)
1217: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1218: }
1219:
1220: #ifndef DOXYGEN
1221: struct Transcode_pair_info {
1222: const Charset* source_transcoder;
1223: const Charset* dest_transcoder;
1224: };
1225: #endif
1.76 misha 1226: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1227: String::Body& avalue,
1.35 paf 1228: Transcode_pair_info* info) {
1229: avalue=Charset::transcode(avalue,
1230: *info->source_transcoder,
1231: *info->dest_transcoder);
1.34 paf 1232: }
1.61 misha 1233:
1.35 paf 1234: void Charset::transcode(HashStringString& src,
1.34 paf 1235: const Charset& source_transcoder,
1.35 paf 1236: const Charset& dest_transcoder) {
1237: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1238: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1239: }
1.61 misha 1240:
1241: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1242: const XMLByte* ptr=srcBegin;
1.70 misha 1243: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1244:
1245: return ptr-srcBegin;
1246: }
1247:
1248: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1249: size_t charPos=0;
1250: const XMLByte* ptr=srcBegin;
1251: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1252: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1253: if(ptr>ptrEnd)
1254: return charPos;
1255: charPos++;
1256: }
1257:
1258: // scan till end but position in bytes still too low
1259: throw Exception(0,
1260: 0,
1261: "Error convertion byte pos to char pos");
1262: }
1263:
1264: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1265: size_t size=0;
1.70 misha 1266: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1267: size++;
1268:
1269: return size;
1270: }
1.80 misha 1271:
1.84 misha 1272: unsigned int lengthUTF8Char(const XMLByte c){
1273: return gUTFBytes[c]+1;
1274: }
1275:
1.80 misha 1276: bool UTF8_string_iterator::has_next(){
1277: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1278: return fcharSize!=0;
1279: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main