Annotation of parser3/src/main/pa_charset.C, revision 1.85
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.68 misha 4: Copyright(c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.85 ! misha 8: static const char * const IDENT_CHARSET_C="$Date: 2009-11-06 05:01:18 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.35 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 18:
19: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 20: #define PRECALCULATE_DEST_LENGTH
1.46 paf 21:
1.38 paf 22: // globals
23:
24: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
25: #include "utf8-to-upper.inc"
26: };
27: Charset::UTF8CaseTable UTF8CaseToUpper={
28: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
29: UTF8CaseToUpperRecords};
30:
31: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
32: #include "utf8-to-lower.inc"
33: };
34: Charset::UTF8CaseTable UTF8CaseToLower={
35: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
36: UTF8CaseToLowerRecords};
37:
1.1 paf 38: // helpers
39:
40: inline void prepare_case_tables(unsigned char *tables) {
41: unsigned char *lcc_table=tables+lcc_offset;
42: unsigned char *fcc_table=tables+fcc_offset;
43: for(int i=0; i<0x100; i++)
1.53 paf 44: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 45: }
46: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
47: unsigned char bit) {
48: unsigned char *ctypes_table=tables+ctypes_offset;
49: ctypes_table[0]=bit;
50: for(; *cstr; cstr++) {
51: unsigned char c=*cstr;
52: ctypes_table[c]|=bit;
53: }
54: }
1.35 paf 55: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 56: if(!cstr || !*cstr)
57: return 0;
58: if(cstr[1]==0)
1.4 paf 59: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 60:
61: char *error_pos;
1.4 paf 62: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 63: }
1.35 paf 64: inline bool to_bool(const char* cstr) {
1.1 paf 65: return cstr && *cstr!=0;
66: }
67: static void element2ctypes(unsigned char c, bool belongs,
68: unsigned char *tables, unsigned char bit, int group_offset=-1) {
69: if(!belongs)
70: return;
71:
72: unsigned char *ctypes_table=tables+ctypes_offset;
73:
74: ctypes_table[c]|=bit;
75: if(group_offset>=0)
1.4 paf 76: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 77: }
78: static void element2case(unsigned char from, unsigned char to,
79: unsigned char *tables) {
80: if(!to)
81: return;
82:
83: unsigned char *lcc_table=tables+lcc_offset;
84: unsigned char *fcc_table=tables+fcc_offset;
85: lcc_table[from]=to;
86: fcc_table[from]=to; fcc_table[to]=from;
87: }
88:
89: // methods
90:
1.37 paf 91: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 92: FNAME(ANAME),
93: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 94:
1.35 paf 95: if(afile_spec) {
1.1 paf 96: fisUTF8=false;
1.35 paf 97: load_definition(*charsets, *afile_spec);
1.1 paf 98: #ifdef XML
1.35 paf 99: addEncoding(FNAME_CSTR);
1.1 paf 100: #endif
101: } else {
102: fisUTF8=true;
1.4 paf 103: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 104: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 105: }
106:
107: #ifdef XML
1.35 paf 108: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 109: #endif
110: }
111:
1.35 paf 112: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 113: // pcre_tables
114: // lowcase, flipcase, bits digit+word+whitespace, masks
115:
116: // must not move this inside of prepare_case_tables
117: // don't know the size there
118: memset(pcre_tables, 0, sizeof(pcre_tables));
119: prepare_case_tables(pcre_tables);
1.4 paf 120: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 121:
122: // charset
1.35 paf 123: memset(&tables, 0, sizeof(tables));
1.1 paf 124:
125: // loading text
1.35 paf 126: char *data=file_read_text(charsets, afile_spec);
1.1 paf 127:
128: // ignore header
129: getrow(&data);
130:
131: // parse cells
132: char *row;
1.42 paf 133: while((row=getrow(&data))) {
1.1 paf 134: // remove empty&comment lines
135: if(!*row || *row=='#')
136: continue;
137:
138: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 139: unsigned char c=0;
1.1 paf 140: char *cell;
1.42 paf 141: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 142: switch(column) {
1.53 paf 143: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 144: // pcre_tables
145: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
146: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
147: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
148: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
149: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 150: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 151: case 7:
152: case 8:
153: // charset
1.10 paf 154: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 155: throw Exception(PARSER_RUNTIME,
1.35 paf 156: &afile_spec,
1.1 paf 157: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
158:
159: XMLCh unicode=(XMLCh)to_wchar_code(cell);
160: if(!unicode && column==7/*unicode1 column*/)
161: unicode=(XMLCh)c;
162: if(unicode) {
1.10 paf 163: if(!tables.fromTable[c])
164: tables.fromTable[c]=unicode;
165: tables.toTable[tables.toTableSize].intCh=unicode;
166: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
167: tables.toTableSize++;
1.1 paf 168: }
169: break;
170: }
171: }
172: };
173:
174: // sort by the Unicode code point
175: sort_ToTable();
176: }
177:
178: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
179: return
1.38 paf 180: static_cast<const Charset::Tables::Rec *>(a)->intCh-
181: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 182: }
183:
184: void Charset::sort_ToTable() {
1.10 paf 185: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 186: sort_cmp_Trans_rec_intCh);
187: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 188: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 189: //fclose(f);
190: }
191:
1.60 misha 192: // @todo: precache for spedup searching
1.10 paf 193: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 194: const Charset::Tables& tables,
195: XMLByte not_found) {
1.80 misha 196: int lo = 0;
197: int hi = tables.toTableSize - 1;
1.39 paf 198: while(lo<=hi) {
1.35 paf 199: // Calc the mid point of the low and high offset.
1.39 paf 200: const unsigned int i = (lo + hi) / 2;
201:
202: XMLCh cur=tables.toTable[i].intCh;
203: if(toXlat==cur)
204: return tables.toTable[i].extCh;
205: if(toXlat>cur)
206: lo = i+1;
1.1 paf 207: else
1.39 paf 208: hi = i-1;
209: }
1.35 paf 210:
211: return not_found;
1.1 paf 212: }
213:
1.35 paf 214: String::C Charset::transcode(const String::C src,
215: const Charset& source_charset,
216: const Charset& dest_charset) {
217: if(!src.length)
218: return String::C("", 0);
1.4 paf 219:
1.1 paf 220: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
221: default: // 0x00
1.35 paf 222: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 223: case 0x01:
1.35 paf 224: return source_charset.transcodeToUTF8(src);
1.1 paf 225: case 0x10:
1.35 paf 226: return dest_charset.transcodeFromUTF8(src);
1.1 paf 227: case 0x11:
1.35 paf 228: return src;
1.1 paf 229: }
230: }
231:
232: // ---------------------------------------------------------------------------
233: // Local static data
234: //
235: // gUTFBytes
236: // A list of counts of trailing bytes for each initial byte in the input.
237: //
238: // gUTFOffsets
239: // A list of values to offset each result char type, according to how
240: // many source bytes when into making it.
241: //
242: // gFirstByteMark
243: // A list of values to mask onto the first byte of an encoded sequence,
244: // indexed by the number of bytes used to create the sequence.
245: // ---------------------------------------------------------------------------
246: static const XMLByte gUTFBytes[0x100] = {
247: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
248: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
249: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
250: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
251: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
252: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
253: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
254: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
255: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
256: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
257: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
258: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
259: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
260: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
261: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
262: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
263: };
264:
265: static const uint gUTFOffsets[6] = {
1.80 misha 266: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 267: };
268:
269: static const XMLByte gFirstByteMark[7] = {
1.80 misha 270: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 271: };
272:
1.71 misha 273: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
274: XMLByte *toFill, int& toFillLen,
275: const Charset::Tables& tables) {
1.11 paf 276: const XMLByte* srcPtr=srcData;
277: const XMLByte* srcEnd=srcData+srcLen;
278: XMLByte* outPtr=toFill;
279: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 280:
1.35 paf 281: while(srcPtr<srcEnd) {
282: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 283: if(!curVal) {
1.35 paf 284: // use the replacement character
285: *outPtr++= '?';
286: srcPtr++;
287: continue;
288: }
1.1 paf 289:
1.35 paf 290: // Figure out how many bytes we need
291: unsigned int encodedBytes;
292: if(curVal<0x80)
293: encodedBytes = 1;
294: else if(curVal<0x800)
295: encodedBytes = 2;
296: else if(curVal<0x10000)
297: encodedBytes = 3;
298: else if(curVal<0x200000)
299: encodedBytes = 4;
300: else if(curVal<0x4000000)
301: encodedBytes = 5;
302: else if(curVal<= 0x7FFFFFFF)
303: encodedBytes = 6;
304: else {
305: // use the replacement character
306: *outPtr++= '?';
307: srcPtr++;
308: continue;
309: }
1.11 paf 310:
1.35 paf 311: // If we cannot fully get this char into the output buffer
312: if (outPtr + encodedBytes > outEnd)
313: break;
314:
315: // We can do it, so update the source index
316: srcPtr++;
317:
318: // And spit out the bytes. We spit them out in reverse order
319: // here, so bump up the output pointer and work down as we go.
320: outPtr+= encodedBytes;
321: switch(encodedBytes) {
1.60 misha 322: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
323: curVal>>= 6;
324: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
325: curVal>>= 6;
326: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
327: curVal>>= 6;
328: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
329: curVal>>= 6;
330: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
331: curVal>>= 6;
332: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 333: }
334:
335: // Add the encoded bytes back in again to indicate we've eaten them
336: outPtr+= encodedBytes;
337: }
338:
339: // Update the bytes eaten
340: srcLen = srcPtr - srcData;
341:
342: // Return the characters read
343: toFillLen = outPtr - toFill;
344:
1.29 paf 345: //return srcPtr==srcEnd?(int)toFillLen:-1;
346: /*
347: xmlCharEncodingInputFunc
348: Returns :
349: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
350: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
351: of ocetes consumed.
352: */
353: return 0;
1.1 paf 354: }
1.26 paf 355: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 356: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
357: XMLByte* toFill, int& toFillLen,
358: const Charset::Tables& tables) {
1.11 paf 359: const XMLByte* srcPtr=srcData;
360: const XMLByte* srcEnd=srcData+srcLen;
361: XMLByte* outPtr=toFill;
362: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 363:
1.35 paf 364: // We now loop until we either run out of input data, or room to store
365: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
366: // Get the next leading byte out
367: const XMLByte firstByte =* srcPtr;
368:
369: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 370: if(firstByte<=127) {
1.35 paf 371: *outPtr++= firstByte;
372: srcPtr++;
373: continue;
374: }
375:
376: // See how many trailing src bytes this sequence is going to require
377: const unsigned int trailingBytes = gUTFBytes[firstByte];
378:
379: // If there are not enough source bytes to do this one, then we
380: // are done. Note that we done>= here because we are implicitly
381: // counting the 1 byte we get no matter what.
382: if(srcPtr+trailingBytes>= srcEnd)
383: break;
384:
385: // Looks ok, so lets build up the value
386: uint tmpVal=0;
387: switch(trailingBytes) {
388: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
389: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
390: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
391: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
392: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
393: case 0: tmpVal+=*srcPtr++;
394: break;
395:
396: default:
397: throw Exception(0,
398: 0,
1.49 paf 399: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 400: }
401: tmpVal-=gUTFOffsets[trailingBytes];
402:
403: // If it will fit into a single char, then put it in. Otherwise
404: // fail [*encode it as a surrogate pair. If its not valid, use the
405: // replacement char.*]
406: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 407: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
408: *outPtr++=xlat;
1.49 paf 409: else {
1.50 paf 410: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 411: }
412: } else {
413: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
414: for(uint i=0; i<=trailingBytes; i++)
415: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
416: }
1.1 paf 417: }
1.35 paf 418:
419: // Update the bytes eaten
420: srcLen = srcPtr - srcData;
421:
422: // Return the characters read
423: toFillLen = outPtr - toFill;
1.11 paf 424:
1.29 paf 425: //return srcPtr==srcEnd?(int)toFillLen:-1;
426: /*
427: xmlCharEncodingOutputFunc
428: Returns :
429: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
430: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
431: of ocetes consumed.
432: */
433: return 0;
1.10 paf 434: }
435:
1.85 ! misha 436: static bool need_escape(XMLByte c){
1.60 misha 437: return
1.66 misha 438: !(
439: (c<=127)
440: && (
441: ((c>='0') && (c<='9'))
442: || ((c>='A') && (c<='Z'))
443: || ((c>='a') && (c<='z'))
444: || strchr("*@-_+./", c)!=0
445: )
446: );
1.60 misha 447: }
448:
1.70 misha 449: // read one UTF8 char and return length of this char (in bytes)
450: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 451: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
452: return 0;
453:
454: firstByte=*srcPtr;
455:
456: if(firstByte<=127){
457: UTF8Char=firstByte;
458: srcPtr++;
459: return 1;
460: }
461:
462: unsigned int trailingBytes=gUTFBytes[firstByte];
463:
464: if(srcPtr+trailingBytes>=srcEnd){
465: return 0; // not enough bytes in source string for reading
466: }
467:
468: uint tmpVal=0;
469: switch(trailingBytes){
470: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
471: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
472: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
473: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
474: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
475: case 0: tmpVal+=*srcPtr++;
476: }
477:
478: tmpVal-=gUTFOffsets[trailingBytes];
479: UTF8Char=tmpVal;
480:
481: return trailingBytes+1;
482: }
483:
1.70 misha 484: // skip UTF8 char and return length of this char (in bytes)
485: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 486: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
487: return 0;
488:
1.63 misha 489: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
490: srcPtr+=trailingBytes;
1.62 misha 491:
492: return trailingBytes;
1.61 misha 493: }
494:
1.85 ! misha 495: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 496: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 497: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
498: return 0;
499:
500: firstByte=*srcPtr++;
501: UTF8Char=tables.fromTable[firstByte];
502:
503: if(UTF8Char<0x80)
504: return 1;
505: else if(UTF8Char<0x800)
506: return 2;
507: else if(UTF8Char<0x10000)
508: return 3;
509: else if(UTF8Char<0x200000)
510: return 4;
511: else if(UTF8Char<0x4000000)
512: return 5;
513: else if(UTF8Char<= 0x7FFFFFFF)
514: return 6;
515:
516: // will use the replacement character '?'
517: firstByte=0;
518: return 1;
519: }
520:
1.85 ! misha 521: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
! 522: size_t dest_length=0;
! 523:
! 524: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
! 525: if(i.getCharSize()==1)
! 526: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
! 527: else
! 528: dest_length+=6; // %uXXXX
1.60 misha 529: }
530:
1.85 ! misha 531: return dest_length;
1.60 misha 532: }
533:
1.85 ! misha 534: size_t Charset::calc_escaped_length(XMLByte* src, size_t src_length, const Charset::Tables& tables){
! 535: size_t dest_length=0;
! 536:
! 537: const XMLByte* src_ptr=src;
! 538: const XMLByte* src_end=src_ptr+src_length;
! 539: XMLByte first_byte;
! 540: XMLCh UTF8_char;
! 541: while(uint char_size=readChar(src_ptr, src_end, first_byte, UTF8_char, tables)){
! 542: if(char_size==1)
! 543: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
! 544: else
! 545: dest_length+=6; // %uXXXX
1.60 misha 546: }
547:
1.85 ! misha 548: return dest_length;
! 549: }
! 550:
! 551: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
! 552: size_t src_length=src.length;
! 553: if(!src_length)
! 554: return 0;
! 555:
! 556: #ifdef PRECALCULATE_DEST_LENGTH
! 557: if(source_charset.isUTF8())
! 558: return calc_escaped_length_UTF8((XMLByte *)src.str, src_length);
! 559: else
! 560: return calc_escaped_length((XMLByte *)src.str, src_length, source_charset.tables);
! 561: #else
! 562: return src_length*6; // enough for %uXXXX but too memory-hungry
! 563: #endif
! 564: }
! 565:
! 566: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
! 567: if(char_size==1) \
! 568: if(first_byte){ \
! 569: if(need_escape(first_byte)) \
! 570: dest_ptr+=sprintf((char*)dest_ptr, "%%%02X", first_byte); /* %XX */ \
! 571: else \
! 572: *dest_ptr++=first_byte; /*as is*/ \
! 573: } else \
! 574: *dest_ptr++='?'; /* replacement char '?' */ \
! 575: else \
! 576: dest_ptr+=sprintf((char*)dest_ptr, "%%u%04X", UTF8_char); // %uXXXX
! 577:
! 578:
! 579: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
! 580: XMLByte* dest_ptr=dest;
! 581:
! 582: // loop until we either run out of input data
! 583: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
! 584: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 585:
1.85 ! misha 586: return dest_ptr - dest;
1.60 misha 587: }
588:
1.85 ! misha 589: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
! 590: const XMLByte* src_ptr=src;
! 591: const XMLByte* src_end=src+src_length;
! 592: XMLByte* dest_ptr=dest;
! 593:
! 594: XMLByte first_byte;
! 595: XMLCh UTF8_char;
! 596: uint char_size;
! 597:
! 598: while(char_size=readChar(src_ptr, src_end, first_byte, UTF8_char, tables))
! 599: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
! 600:
! 601: return dest_ptr - dest;
! 602: }
1.60 misha 603:
604: String::C Charset::escape(const String::C src, const Charset& source_charset){
605: size_t src_length=src.length;
606: if(!src_length)
607: return String::C("", 0);
608:
1.85 ! misha 609: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
! 610: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
! 611:
! 612: size_t dest_length;
! 613: if(source_charset.isUTF8())
! 614: dest_length=escape_UTF8((XMLByte *)src.str, src_length, dest_body);
! 615: else
! 616: dest_length=escape((XMLByte *)src.str, src_length, dest_body, source_charset.tables);
! 617:
! 618: if(dest_length>dest_calculated_length)
! 619: throw Exception(0, 0, "Charset::escape buffer overflow");
! 620:
! 621: dest_body[dest_length]=0; // terminator
! 622: return String::C((char*)dest_body, dest_length);
! 623: }
! 624:
! 625: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
! 626: const char *src_ptr=src.cstr();
! 627: size_t src_size=src.length();
! 628:
! 629: String::C dest=Charset::escape(String::C(src_ptr, src_size), source_charset);
! 630:
! 631: return String::Body(dest.length ? dest.str:0);
! 632: }
! 633:
! 634: String& Charset::escape(const String& src, const Charset& source_charset) {
! 635: if(src.is_empty())
! 636: return *new String();
! 637:
! 638: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
! 639: }
! 640:
! 641: inline bool need_json_escape(unsigned char c){
! 642: return strchr("\n\"\\/\t\r\b\f", c)!=0;
! 643: }
! 644:
! 645: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
! 646: size_t dest_length=0;
! 647:
! 648: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
! 649: if(i.getCharSize()==1)
! 650: dest_length+=need_json_escape(i.getFirstByte()) ? 2 : 1;
! 651: else
! 652: dest_length+=6; // \uXXXX
! 653: }
! 654:
! 655: return dest_length;
! 656: }
! 657:
! 658: size_t Charset::calc_JSON_escaped_length(XMLByte* src, size_t src_length, const Charset::Tables& tables){
! 659: const XMLByte* src_ptr=src;
! 660: const XMLByte* src_end=src_ptr+src_length;
! 661: XMLByte first_byte;
! 662: XMLCh UTF8_char;
1.60 misha 663: size_t dest_length=0;
664:
1.85 ! misha 665: while(uint char_size=readChar(src_ptr, src_end, first_byte, UTF8_char, tables)){
! 666: if(char_size==1)
! 667: dest_length+=(!first_byte/*replacement char '?'*/ || !need_json_escape(first_byte))? 1 : 2;
! 668: else
! 669: dest_length+=6; // \uXXXX
1.60 misha 670: }
1.85 ! misha 671:
! 672: return dest_length;
! 673: }
! 674:
! 675: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
! 676: size_t src_length=src.length;
! 677: if(!src_length)
! 678: return 0;
! 679:
! 680: #ifdef PRECALCULATE_DEST_LENGTH
! 681: if(source_charset.isUTF8())
! 682: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src_length);
! 683: else
! 684: return calc_JSON_escaped_length((XMLByte *)src.str, src_length, source_charset.tables);
1.60 misha 685: #else
1.85 ! misha 686: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 687: #endif
1.85 ! misha 688: }
! 689:
! 690: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
! 691: if(char_size==1) \
! 692: switch(first_byte){ \
! 693: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
! 694: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
! 695: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
! 696: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
! 697: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
! 698: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
! 699: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
! 700: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
! 701: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
! 702: default : *dest_ptr++=first_byte; \
! 703: } \
! 704: else \
! 705: dest_ptr+=sprintf((char*)dest_ptr, "\\u%04X", UTF8_char); // \uXXXX
! 706:
! 707:
! 708: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
! 709: XMLByte* dest_ptr=dest;
! 710:
! 711: // loop until we either run out of input data
! 712: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
! 713: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
! 714:
! 715: return dest_ptr - dest;
! 716: }
! 717:
! 718: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
! 719: const XMLByte* src_ptr=src;
! 720: const XMLByte* src_end=src+src_length;
! 721: XMLByte* dest_ptr=dest;
! 722:
! 723: XMLByte first_byte;
! 724: XMLCh UTF8_char;
! 725: uint char_size;
! 726:
! 727: while(char_size=readChar(src_ptr, src_end, first_byte, UTF8_char, tables))
! 728: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
! 729:
! 730: return dest_ptr - dest;
! 731: }
1.60 misha 732:
1.85 ! misha 733: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
! 734: size_t src_length=src.length;
! 735: if(!src_length)
! 736: return String::C("", 0);
1.60 misha 737:
738:
1.85 ! misha 739: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
! 740: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
! 741:
! 742: size_t dest_length;
! 743: if(source_charset.isUTF8())
! 744: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src_length, dest_body);
! 745: else
! 746: dest_length=escape_JSON((XMLByte *)src.str, src_length, dest_body, source_charset.tables);
1.60 misha 747:
1.85 ! misha 748: if(dest_length>dest_calculated_length)
! 749: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 750:
751: dest_body[dest_length]=0; // terminator
752: return String::C((char*)dest_body, dest_length);
753: }
1.85 ! misha 754:
! 755: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.64 misha 756: const char *src_ptr=src.cstr();
1.83 misha 757: size_t src_size=src.length();
1.64 misha 758:
1.85 ! misha 759: String::C dest=Charset::escape_JSON(String::C(src_ptr, src_size), source_charset);
1.64 misha 760:
1.77 misha 761: return String::Body(dest.length ? dest.str:0);
1.64 misha 762: }
763:
1.85 ! misha 764: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 765: if(src.is_empty())
1.73 misha 766: return *new String();
1.64 misha 767:
1.85 ! misha 768: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 769: }
1.60 misha 770:
1.35 paf 771: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 772: int src_length=src.length;
1.60 misha 773:
774: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 775: int dest_length=0;
1.60 misha 776: const XMLByte* srcPtr=(XMLByte*)src.str;
777: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 778: XMLByte firstByte;
779: XMLCh UTF8Char;
780: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 781: dest_length+=charSize;
782: #else
1.85 ! misha 783: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 784: #endif
785:
1.35 paf 786: #ifndef NDEBUG
1.71 misha 787: int saved_dest_length=dest_length;
1.35 paf 788: #endif
789: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 790:
791: if(::transcodeToUTF8(
1.35 paf 792: (XMLByte *)src.str, src_length,
793: dest_body, dest_length,
1.11 paf 794: tables)<0)
1.43 paf 795: throw Exception(0,
1.10 paf 796: 0,
1.11 paf 797: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 798:
1.60 misha 799: assert(dest_length<=saved_dest_length);
800: dest_body[dest_length]=0; // terminator
1.35 paf 801: return String::C((char*)dest_body, dest_length);
1.10 paf 802: }
1.38 paf 803:
804: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 805: int lo = 0;
806: int hi = table.size - 1;
1.39 paf 807: while(lo<=hi) {
1.38 paf 808: // Calc the mid point of the low and high offset.
1.39 paf 809: const unsigned int i = (lo + hi) / 2;
810:
811: XMLCh cur=table.records[i].from;
812: if(src==cur)
813: return table.records[i].to;
814: if(src>cur)
815: lo = i+1;
1.38 paf 816: else
1.39 paf 817: hi = i-1;
818: }
819:
820: // not found
1.38 paf 821: return src;
822: }
823:
1.58 misha 824: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 825: if(!src) {
826: // use the replacement character
827: *outPtr++= '?';
828: return;
829: }
830:
831: // Figure out how many bytes we need
832: unsigned int encodedBytes;
833: if(src<0x80)
834: encodedBytes = 1;
835: else if(src<0x800)
836: encodedBytes = 2;
837: else if(src<0x10000)
838: encodedBytes = 3;
839: else if(src<0x200000)
840: encodedBytes = 4;
841: else if(src<0x4000000)
842: encodedBytes = 5;
843: else if(src<= 0x7FFFFFFF)
844: encodedBytes = 6;
845: else {
846: // use the replacement character
847: *outPtr++= '?';
848: return;
849: }
850:
851: // And spit out the bytes. We spit them out in reverse order
852: // here, so bump up the output pointer and work down as we go.
853: outPtr+= encodedBytes;
854: switch(encodedBytes) {
855: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
856: src>>= 6;
857: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
858: src>>= 6;
859: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
860: src>>= 6;
861: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
862: src>>= 6;
863: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
864: src>>= 6;
865: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
866: }
867:
868: // Add the encoded bytes back in again to indicate we've eaten them
869: outPtr+= encodedBytes;
870: }
871:
872: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
873: const Charset::UTF8CaseTable& table) {
874: store_UTF8(change_case_UTF8(src, table), outPtr);
875: };
1.44 paf 876: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
877: XMLByte* toFill, size_t toFillLen,
878: const Charset::UTF8CaseTable& table) {
1.38 paf 879: const XMLByte* srcPtr=srcData;
1.44 paf 880: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 881: XMLByte* outPtr=toFill;
1.44 paf 882: XMLByte* outEnd=toFill+toFillLen;
883:
884: // We now loop until we either run out of input data, or room to store
885: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
886: // Get the next leading byte out
887: const XMLByte firstByte =* srcPtr;
1.38 paf 888:
1.60 misha 889: if(firstByte<=127) {
1.38 paf 890: change_case_UTF8(firstByte, outPtr, table);
891: srcPtr++;
892: continue;
893: }
894:
895: // See how many trailing src bytes this sequence is going to require
896: const unsigned int trailingBytes = gUTFBytes[firstByte];
897:
898: // Looks ok, so lets build up the value
899: uint tmpVal=0;
900: switch(trailingBytes) {
901: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
902: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
903: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
904: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
905: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
906: case 0: tmpVal+=*srcPtr++;
907: break;
908:
909: default:
910: throw Exception(0,
911: 0,
912: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
913: }
914: tmpVal-=gUTFOffsets[trailingBytes];
915:
916: // If it will fit into a single char, then put it in. Otherwise
917: // fail [*encode it as a surrogate pair. If its not valid, use the
918: // replacement char.*]
919: if(!(tmpVal & 0xFFFF0000))
920: change_case_UTF8(tmpVal, outPtr, table);
921: else
922: throw Exception(0,
923: 0,
924: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
925: }
926:
927: if(srcPtr!=outPtr)
928: throw Exception(0,
929: 0,
930: "change_case_UTF8 error: end pointers do not match");
931: }
932:
1.60 misha 933: static size_t getDecNumLength(XMLCh UTF8Char){
934: return
935: (UTF8Char < 100)
936: ?2
937: :(UTF8Char < 1000)
938: ?3
939: :(UTF8Char < 10000)
940: ?4
941: :5;
942: }
1.38 paf 943:
1.35 paf 944: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 945: int src_length=src.length;
1.60 misha 946: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 947: int dest_length=0;
1.82 misha 948: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.81 misha 949: if(i.getCharSize()==1)
1.60 misha 950: dest_length++;
951: else
1.81 misha 952: dest_length+=(i.next() & 0xFFFF0000)
953: ?i.getCharSize()*3 // '%XX' for each byte
954: :(xlatOneTo(i.next(), tables, 0)!=0)
1.60 misha 955: ?1 // can convert it to single char
1.81 misha 956: :getDecNumLength(i.next())+3; // &#XX; - &#XXXXX;
1.60 misha 957: }
958: #else
959: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 960: int dest_length=src_length*6;
1.60 misha 961: #endif
962:
1.35 paf 963: #ifndef NDEBUG
1.71 misha 964: int saved_dest_length=dest_length;
1.35 paf 965: #endif
966: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 967:
968: if(::transcodeFromUTF8(
1.82 misha 969: (XMLByte *)src.str, src_length,
1.35 paf 970: dest_body, dest_length,
1.11 paf 971: tables)<0)
1.43 paf 972: throw Exception(0,
1.10 paf 973: 0,
1.35 paf 974: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 975:
1.60 misha 976: assert(dest_length<=saved_dest_length);
977: dest_body[dest_length]=0; // terminator
1.35 paf 978: return String::C((char*)dest_body, dest_length);
1.1 paf 979: }
980:
981: /// transcode using both charsets
1.35 paf 982: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 983: const Charset& dest_charset) const {
1.35 paf 984: if(&dest_charset==this)
985: return src;
986: else {
987: size_t dest_length=src.length;
988: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
989:
990: XMLByte* output=dest_body;
991: const XMLByte* input=(XMLByte *)src.str;
992: while(XMLCh c=*input++) {
993: XMLCh curVal = tables.fromTable[c];
994: *output++=curVal?
995: xlatOneTo(curVal, dest_charset.tables, '?') // OK
996: :'?'; // use the replacement character
1.6 paf 997: }
1.1 paf 998:
1.35 paf 999: dest_body[dest_length]=0; // terminator
1000: return String::C((char*)dest_body, dest_length);
1.6 paf 1001: }
1.1 paf 1002: }
1003:
1.58 misha 1004: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 1005: if(isUTF8())
1.58 misha 1006: store_UTF8(src, outPtr);
1.59 misha 1007: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1008: *outPtr++=ch;
1.57 misha 1009: }
1010:
1.1 paf 1011: #ifdef XML
1.10 paf 1012:
1.35 paf 1013: static const Charset::Tables* tables[MAX_CHARSETS];
1014:
1.46 paf 1015: #ifdef PA_PATCHED_LIBXML_BACKWARD
1016:
1017: #define declareXml256ioFuncs(i) \
1018: static int xml256CharEncodingInputFunc##i( \
1019: unsigned char *out, int *outlen, \
1020: const unsigned char *in, int *inlen, void*) { \
1021: return transcodeToUTF8( \
1.71 misha 1022: in, *inlen, \
1023: out, *outlen, \
1.46 paf 1024: *tables[i]); \
1025: } \
1026: static int xml256CharEncodingOutputFunc##i( \
1027: unsigned char *out, int *outlen, \
1028: const unsigned char *in, int *inlen, void*) { \
1029: return transcodeFromUTF8( \
1.71 misha 1030: in, *inlen, \
1031: out, *outlen, \
1.46 paf 1032: *tables[i]); \
1033: }
1034:
1035: #else
1036:
1.35 paf 1037: #define declareXml256ioFuncs(i) \
1038: static int xml256CharEncodingInputFunc##i( \
1039: unsigned char *out, int *outlen, \
1040: const unsigned char *in, int *inlen) { \
1041: return transcodeToUTF8( \
1.71 misha 1042: in, *inlen, \
1043: out, *outlen, \
1.35 paf 1044: *tables[i]); \
1045: } \
1046: static int xml256CharEncodingOutputFunc##i( \
1047: unsigned char *out, int *outlen, \
1048: const unsigned char *in, int *inlen) { \
1049: return transcodeFromUTF8( \
1.71 misha 1050: in, *inlen, \
1051: out, *outlen, \
1.35 paf 1052: *tables[i]); \
1053: }
1054:
1.46 paf 1055: #endif
1056:
1057:
1.35 paf 1058: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1059: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1060: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1061: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1062: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1063:
1064: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1065: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1066: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1067: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1068: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1069: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1070: };
1071: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1072: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1073: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1074: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1075: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1076: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1077: };
1078: static size_t handlers_count=0;
1.10 paf 1079:
1080: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1081: if(handlers_count==MAX_CHARSETS)
1082: throw Exception(0,
1083: 0,
1084: "already allocated %d handlers, no space for new encoding '%s'",
1085: MAX_CHARSETS, name_cstr);
1086:
1.45 paf 1087: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 1088: {
1089: handler->name=name_cstr;
1090: handler->input=inputFuncs[handlers_count];
1091: handler->output=outputFuncs[handlers_count];
1092: ::tables[handlers_count]=&tables;
1093: handlers_count++;
1094: }
1.10 paf 1095:
1096: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1097:
1.10 paf 1098: }
1099:
1.37 paf 1100: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1101: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1102: transcoder(NAME); // check right way
1.15 paf 1103: }
1104:
1.37 paf 1105: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1106: if(!ftranscoder)
1.56 misha 1107: throw Exception(PARSER_RUNTIME,
1.35 paf 1108: new String(NAME, String::L_TAINTED),
1.10 paf 1109: "unsupported encoding");
1.35 paf 1110: return *ftranscoder;
1.10 paf 1111: }
1112:
1.54 paf 1113: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1114: if(!s)
1.35 paf 1115: return String::C("", 0);
1.8 paf 1116:
1.35 paf 1117: int inlen=strlen((const char*)s);
1.51 paf 1118: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1119: #ifndef NDEBUG
1120: int saved_outlen=outlen;
1121: #endif
1122: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1123:
1.30 paf 1124: int error;
1.35 paf 1125: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1126: error=output(
1.17 paf 1127: (unsigned char*)out, &outlen,
1.46 paf 1128: (const unsigned char*)s, &inlen
1129: #ifdef PA_PATCHED_LIBXML_BACKWARD
1130: ,0
1131: #endif
1132: );
1.30 paf 1133: } else {
1134: memcpy(out, s, outlen=inlen);
1135: error=0;
1136: }
1137: if(error<0)
1.23 paf 1138: throw Exception(0,
1.8 paf 1139: 0,
1.30 paf 1140: "transcode_cstr failed (%d)", error);
1.8 paf 1141:
1.35 paf 1142: assert(outlen<=saved_outlen); out[outlen]=0;
1143: return String::C(out, outlen);
1.14 paf 1144: }
1.54 paf 1145: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1146: String::C cstr=transcode_cstr(s);
1.75 misha 1147: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1148: }
1149:
1.8 paf 1150: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1151: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1152: xmlChar* out;
1.30 paf 1153: int outlen;
1154: int error;
1.35 paf 1155: #ifndef NDEBUG
1156: int saved_outlen;
1157: #endif
1158: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1159: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1160: #ifndef NDEBUG
1161: saved_outlen=outlen;
1162: #endif
1.47 paf 1163: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1164: error=input(
1.17 paf 1165: out, &outlen,
1.46 paf 1166: (const unsigned char*)buf, (int*)&buf_size
1167: #ifdef PA_PATCHED_LIBXML_BACKWARD
1168: ,0
1169: #endif
1170: );
1.30 paf 1171: } else {
1172: outlen=buf_size;
1.35 paf 1173: #ifndef NDEBUG
1174: saved_outlen=outlen;
1175: #endif
1176: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1177: memcpy(out, buf, outlen);
1178: error=0;
1179: }
1.17 paf 1180:
1.30 paf 1181: if(error<0)
1.23 paf 1182: throw Exception(0,
1.8 paf 1183: 0,
1.30 paf 1184: "transcode_buf failed (%d)", error);
1.8 paf 1185:
1.35 paf 1186: assert(outlen<=saved_outlen); out[outlen]=0;
1187: return out;
1.24 paf 1188: }
1.1 paf 1189:
1.79 misha 1190: xmlChar* Charset::transcode(const String& s) {
1191: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1192: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1193: }
1.35 paf 1194:
1.79 misha 1195: xmlChar* Charset::transcode(const String::Body s) {
1196: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1197: }
1.36 paf 1198: #endif
1.34 paf 1199:
1.37 paf 1200: String::Body Charset::transcode(const String::Body src,
1.34 paf 1201: const Charset& source_transcoder,
1.35 paf 1202: const Charset& dest_transcoder) {
1.34 paf 1203:
1.35 paf 1204: const char *src_ptr=src.cstr();
1.83 misha 1205: size_t src_size=src.length();
1.34 paf 1206:
1.77 misha 1207: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1208:
1.77 misha 1209: return String::Body(dest.length ? dest.str:0);
1.35 paf 1210: }
1211:
1212: String& Charset::transcode(const String& src,
1213: const Charset& source_transcoder,
1214: const Charset& dest_transcoder) {
1.72 misha 1215: if(src.is_empty())
1.73 misha 1216: return *new String();
1.34 paf 1217:
1.37 paf 1218: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1219: }
1220:
1.35 paf 1221: void Charset::transcode(ArrayString& src,
1.34 paf 1222: const Charset& source_transcoder,
1.35 paf 1223: const Charset& dest_transcoder) {
1224: for(size_t i=0; i<src.count(); i++)
1225: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1226: }
1227:
1228: #ifndef DOXYGEN
1229: struct Transcode_pair_info {
1230: const Charset* source_transcoder;
1231: const Charset* dest_transcoder;
1232: };
1233: #endif
1.76 misha 1234: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1235: String::Body& avalue,
1.35 paf 1236: Transcode_pair_info* info) {
1237: avalue=Charset::transcode(avalue,
1238: *info->source_transcoder,
1239: *info->dest_transcoder);
1.34 paf 1240: }
1.61 misha 1241:
1.35 paf 1242: void Charset::transcode(HashStringString& src,
1.34 paf 1243: const Charset& source_transcoder,
1.35 paf 1244: const Charset& dest_transcoder) {
1245: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1246: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1247: }
1.61 misha 1248:
1249: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1250: const XMLByte* ptr=srcBegin;
1.70 misha 1251: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1252:
1253: return ptr-srcBegin;
1254: }
1255:
1256: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1257: size_t charPos=0;
1258: const XMLByte* ptr=srcBegin;
1259: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1260: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1261: if(ptr>ptrEnd)
1262: return charPos;
1263: charPos++;
1264: }
1265:
1266: // scan till end but position in bytes still too low
1267: throw Exception(0,
1268: 0,
1269: "Error convertion byte pos to char pos");
1270: }
1271:
1272: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1273: size_t size=0;
1.70 misha 1274: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1275: size++;
1276:
1277: return size;
1278: }
1.80 misha 1279:
1.84 misha 1280: unsigned int lengthUTF8Char(const XMLByte c){
1281: return gUTFBytes[c]+1;
1282: }
1283:
1.80 misha 1284: bool UTF8_string_iterator::has_next(){
1285: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1286: return fcharSize!=0;
1287: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main