Annotation of parser3/src/main/pa_charset.C, revision 1.93
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.90 moko 4: Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.93 ! moko 11: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.92 2013/07/30 12:36:22 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 12:
1.1 paf 13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 18:
19: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 20: #define PRECALCULATE_DEST_LENGTH
1.46 paf 21:
1.38 paf 22: // globals
23:
24: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
25: #include "utf8-to-upper.inc"
26: };
27: Charset::UTF8CaseTable UTF8CaseToUpper={
28: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
29: UTF8CaseToUpperRecords};
30:
31: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
32: #include "utf8-to-lower.inc"
33: };
34: Charset::UTF8CaseTable UTF8CaseToLower={
35: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
36: UTF8CaseToLowerRecords};
37:
1.1 paf 38: // helpers
39:
40: inline void prepare_case_tables(unsigned char *tables) {
41: unsigned char *lcc_table=tables+lcc_offset;
42: unsigned char *fcc_table=tables+fcc_offset;
43: for(int i=0; i<0x100; i++)
1.53 paf 44: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 45: }
46: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
47: unsigned char bit) {
48: unsigned char *ctypes_table=tables+ctypes_offset;
49: ctypes_table[0]=bit;
50: for(; *cstr; cstr++) {
51: unsigned char c=*cstr;
52: ctypes_table[c]|=bit;
53: }
54: }
1.35 paf 55: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 56: if(!cstr || !*cstr)
57: return 0;
58: if(cstr[1]==0)
1.4 paf 59: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 60:
1.91 moko 61: return pa_atoui(cstr,0);
1.1 paf 62: }
1.35 paf 63: inline bool to_bool(const char* cstr) {
1.1 paf 64: return cstr && *cstr!=0;
65: }
66: static void element2ctypes(unsigned char c, bool belongs,
67: unsigned char *tables, unsigned char bit, int group_offset=-1) {
68: if(!belongs)
69: return;
70:
71: unsigned char *ctypes_table=tables+ctypes_offset;
72:
73: ctypes_table[c]|=bit;
74: if(group_offset>=0)
1.4 paf 75: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 76: }
77: static void element2case(unsigned char from, unsigned char to,
78: unsigned char *tables) {
79: if(!to)
80: return;
81:
82: unsigned char *lcc_table=tables+lcc_offset;
83: unsigned char *fcc_table=tables+fcc_offset;
84: lcc_table[from]=to;
85: fcc_table[from]=to; fcc_table[to]=from;
86: }
87:
1.93 ! moko 88: inline void append_hex_8(char*& dest, unsigned char c, const char* prefix=0) {
! 89: if(prefix) {
! 90: strcpy(dest, prefix);
! 91: dest+=strlen(prefix);
! 92: }
! 93: *dest++=hex_digits[c >> 4];
! 94: *dest++=hex_digits[c & 0x0F];
! 95: }
! 96:
! 97: inline void append_hex_16(char*& dest, unsigned int c, const char* prefix=0) {
! 98: if(prefix) {
! 99: strcpy(dest, prefix);
! 100: dest+=strlen(prefix);
! 101: }
! 102: *dest++=hex_digits[(c >> 12) & 0x0F];
! 103: *dest++=hex_digits[(c >> 8) & 0x0F];
! 104: *dest++=hex_digits[(c >> 4) & 0x0F];
! 105: *dest++=hex_digits[(c) & 0x0F];
! 106: }
! 107:
1.1 paf 108: // methods
109:
1.37 paf 110: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 111: FNAME(ANAME),
112: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 113:
1.35 paf 114: if(afile_spec) {
1.1 paf 115: fisUTF8=false;
1.35 paf 116: load_definition(*charsets, *afile_spec);
1.1 paf 117: #ifdef XML
1.35 paf 118: addEncoding(FNAME_CSTR);
1.1 paf 119: #endif
120: } else {
121: fisUTF8=true;
1.4 paf 122: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 123: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 124: }
125:
126: #ifdef XML
1.35 paf 127: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 128: #endif
129: }
130:
1.35 paf 131: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 132: // pcre_tables
133: // lowcase, flipcase, bits digit+word+whitespace, masks
134:
135: // must not move this inside of prepare_case_tables
136: // don't know the size there
137: memset(pcre_tables, 0, sizeof(pcre_tables));
138: prepare_case_tables(pcre_tables);
1.4 paf 139: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 140:
141: // charset
1.35 paf 142: memset(&tables, 0, sizeof(tables));
1.1 paf 143:
144: // loading text
1.35 paf 145: char *data=file_read_text(charsets, afile_spec);
1.1 paf 146:
147: // ignore header
148: getrow(&data);
149:
150: // parse cells
151: char *row;
1.42 paf 152: while((row=getrow(&data))) {
1.1 paf 153: // remove empty&comment lines
154: if(!*row || *row=='#')
155: continue;
156:
157: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 158: unsigned char c=0;
1.1 paf 159: char *cell;
1.42 paf 160: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 161: switch(column) {
1.53 paf 162: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 163: // pcre_tables
164: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
165: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
166: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
167: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
168: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 169: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 170: case 7:
171: case 8:
172: // charset
1.10 paf 173: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 174: throw Exception(PARSER_RUNTIME,
1.35 paf 175: &afile_spec,
1.1 paf 176: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
177:
178: XMLCh unicode=(XMLCh)to_wchar_code(cell);
179: if(!unicode && column==7/*unicode1 column*/)
180: unicode=(XMLCh)c;
181: if(unicode) {
1.10 paf 182: if(!tables.fromTable[c])
183: tables.fromTable[c]=unicode;
184: tables.toTable[tables.toTableSize].intCh=unicode;
185: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
186: tables.toTableSize++;
1.1 paf 187: }
188: break;
189: }
190: }
191: };
192:
1.87 moko 193: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
194: for(uint i=0; i<0x20; i++)
195: if(!tables.fromTable[i]){
196: tables.fromTable[i]=i;
197: tables.toTable[tables.toTableSize].intCh=i;
198: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
199: tables.toTableSize++;
200: }
201:
1.1 paf 202: // sort by the Unicode code point
203: sort_ToTable();
204: }
205:
206: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
207: return
1.38 paf 208: static_cast<const Charset::Tables::Rec *>(a)->intCh-
209: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 210: }
211:
212: void Charset::sort_ToTable() {
1.92 moko 213: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 214: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 215: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 216: //fclose(f);
217: }
218:
1.60 misha 219: // @todo: precache for spedup searching
1.10 paf 220: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 221: const Charset::Tables& tables,
222: XMLByte not_found) {
1.80 misha 223: int lo = 0;
224: int hi = tables.toTableSize - 1;
1.39 paf 225: while(lo<=hi) {
1.35 paf 226: // Calc the mid point of the low and high offset.
1.39 paf 227: const unsigned int i = (lo + hi) / 2;
228:
229: XMLCh cur=tables.toTable[i].intCh;
230: if(toXlat==cur)
231: return tables.toTable[i].extCh;
232: if(toXlat>cur)
233: lo = i+1;
1.1 paf 234: else
1.39 paf 235: hi = i-1;
236: }
1.35 paf 237:
238: return not_found;
1.1 paf 239: }
240:
1.35 paf 241: String::C Charset::transcode(const String::C src,
242: const Charset& source_charset,
243: const Charset& dest_charset) {
244: if(!src.length)
245: return String::C("", 0);
1.4 paf 246:
1.1 paf 247: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
248: default: // 0x00
1.35 paf 249: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 250: case 0x01:
1.35 paf 251: return source_charset.transcodeToUTF8(src);
1.1 paf 252: case 0x10:
1.35 paf 253: return dest_charset.transcodeFromUTF8(src);
1.1 paf 254: case 0x11:
1.35 paf 255: return src;
1.1 paf 256: }
257: }
258:
259: // ---------------------------------------------------------------------------
260: // Local static data
261: //
262: // gUTFBytes
263: // A list of counts of trailing bytes for each initial byte in the input.
264: //
265: // gUTFOffsets
266: // A list of values to offset each result char type, according to how
267: // many source bytes when into making it.
268: //
269: // gFirstByteMark
270: // A list of values to mask onto the first byte of an encoded sequence,
271: // indexed by the number of bytes used to create the sequence.
272: // ---------------------------------------------------------------------------
273: static const XMLByte gUTFBytes[0x100] = {
274: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
275: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
276: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
277: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
278: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
279: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
280: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
281: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
282: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
283: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
284: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
285: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
287: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
288: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
289: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
290: };
291:
292: static const uint gUTFOffsets[6] = {
1.80 misha 293: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 294: };
295:
296: static const XMLByte gFirstByteMark[7] = {
1.80 misha 297: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 298: };
299:
1.71 misha 300: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,
301: XMLByte *toFill, int& toFillLen,
302: const Charset::Tables& tables) {
1.11 paf 303: const XMLByte* srcPtr=srcData;
304: const XMLByte* srcEnd=srcData+srcLen;
305: XMLByte* outPtr=toFill;
306: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 307:
1.35 paf 308: while(srcPtr<srcEnd) {
309: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 310: if(!curVal) {
1.35 paf 311: // use the replacement character
312: *outPtr++= '?';
313: srcPtr++;
314: continue;
315: }
1.1 paf 316:
1.35 paf 317: // Figure out how many bytes we need
318: unsigned int encodedBytes;
319: if(curVal<0x80)
320: encodedBytes = 1;
321: else if(curVal<0x800)
322: encodedBytes = 2;
323: else if(curVal<0x10000)
324: encodedBytes = 3;
325: else if(curVal<0x200000)
326: encodedBytes = 4;
327: else if(curVal<0x4000000)
328: encodedBytes = 5;
329: else if(curVal<= 0x7FFFFFFF)
330: encodedBytes = 6;
331: else {
332: // use the replacement character
333: *outPtr++= '?';
334: srcPtr++;
335: continue;
336: }
1.11 paf 337:
1.35 paf 338: // If we cannot fully get this char into the output buffer
339: if (outPtr + encodedBytes > outEnd)
340: break;
341:
342: // We can do it, so update the source index
343: srcPtr++;
344:
345: // And spit out the bytes. We spit them out in reverse order
346: // here, so bump up the output pointer and work down as we go.
347: outPtr+= encodedBytes;
348: switch(encodedBytes) {
1.60 misha 349: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
350: curVal>>= 6;
351: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
352: curVal>>= 6;
353: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
354: curVal>>= 6;
355: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
356: curVal>>= 6;
357: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
358: curVal>>= 6;
359: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 360: }
361:
362: // Add the encoded bytes back in again to indicate we've eaten them
363: outPtr+= encodedBytes;
364: }
365:
366: // Update the bytes eaten
367: srcLen = srcPtr - srcData;
368:
369: // Return the characters read
370: toFillLen = outPtr - toFill;
371:
1.29 paf 372: //return srcPtr==srcEnd?(int)toFillLen:-1;
373: /*
374: xmlCharEncodingInputFunc
375: Returns :
376: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
377: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
378: of ocetes consumed.
379: */
380: return 0;
1.1 paf 381: }
1.26 paf 382: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.71 misha 383: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,
384: XMLByte* toFill, int& toFillLen,
385: const Charset::Tables& tables) {
1.11 paf 386: const XMLByte* srcPtr=srcData;
387: const XMLByte* srcEnd=srcData+srcLen;
388: XMLByte* outPtr=toFill;
389: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 390:
1.35 paf 391: // We now loop until we either run out of input data, or room to store
392: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
393: // Get the next leading byte out
394: const XMLByte firstByte =* srcPtr;
395:
396: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 397: if(firstByte<=127) {
1.35 paf 398: *outPtr++= firstByte;
399: srcPtr++;
400: continue;
401: }
402:
403: // See how many trailing src bytes this sequence is going to require
404: const unsigned int trailingBytes = gUTFBytes[firstByte];
405:
406: // If there are not enough source bytes to do this one, then we
407: // are done. Note that we done>= here because we are implicitly
408: // counting the 1 byte we get no matter what.
409: if(srcPtr+trailingBytes>= srcEnd)
410: break;
411:
412: // Looks ok, so lets build up the value
413: uint tmpVal=0;
414: switch(trailingBytes) {
415: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
416: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
417: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
418: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
419: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
420: case 0: tmpVal+=*srcPtr++;
421: break;
422:
423: default:
424: throw Exception(0,
425: 0,
1.49 paf 426: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 427: }
428: tmpVal-=gUTFOffsets[trailingBytes];
429:
430: // If it will fit into a single char, then put it in. Otherwise
431: // fail [*encode it as a surrogate pair. If its not valid, use the
432: // replacement char.*]
433: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 434: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
435: *outPtr++=xlat;
1.49 paf 436: else {
1.50 paf 437: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 438: }
439: } else {
440: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
441: for(uint i=0; i<=trailingBytes; i++)
442: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
443: }
1.1 paf 444: }
1.35 paf 445:
446: // Update the bytes eaten
447: srcLen = srcPtr - srcData;
448:
449: // Return the characters read
450: toFillLen = outPtr - toFill;
1.11 paf 451:
1.29 paf 452: //return srcPtr==srcEnd?(int)toFillLen:-1;
453: /*
454: xmlCharEncodingOutputFunc
455: Returns :
456: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
457: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
458: of ocetes consumed.
459: */
460: return 0;
1.10 paf 461: }
462:
1.85 misha 463: static bool need_escape(XMLByte c){
1.60 misha 464: return
1.66 misha 465: !(
466: (c<=127)
467: && (
1.89 misha 468: pa_isalnum((unsigned char)c)
1.66 misha 469: || strchr("*@-_+./", c)!=0
470: )
471: );
1.60 misha 472: }
473:
1.70 misha 474: // read one UTF8 char and return length of this char (in bytes)
475: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 476: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
477: return 0;
478:
479: firstByte=*srcPtr;
480:
481: if(firstByte<=127){
482: UTF8Char=firstByte;
483: srcPtr++;
484: return 1;
485: }
486:
487: unsigned int trailingBytes=gUTFBytes[firstByte];
488:
489: if(srcPtr+trailingBytes>=srcEnd){
490: return 0; // not enough bytes in source string for reading
491: }
492:
493: uint tmpVal=0;
494: switch(trailingBytes){
495: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
496: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
497: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
498: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
499: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
500: case 0: tmpVal+=*srcPtr++;
501: }
502:
503: tmpVal-=gUTFOffsets[trailingBytes];
504: UTF8Char=tmpVal;
505:
506: return trailingBytes+1;
507: }
508:
1.70 misha 509: // skip UTF8 char and return length of this char (in bytes)
510: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 511: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
512: return 0;
513:
1.63 misha 514: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
515: srcPtr+=trailingBytes;
1.62 misha 516:
517: return trailingBytes;
1.61 misha 518: }
519:
1.85 misha 520: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 521: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 522: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
523: return 0;
524:
525: firstByte=*srcPtr++;
526: UTF8Char=tables.fromTable[firstByte];
527:
528: if(UTF8Char<0x80)
529: return 1;
530: else if(UTF8Char<0x800)
531: return 2;
532: else if(UTF8Char<0x10000)
533: return 3;
534: else if(UTF8Char<0x200000)
535: return 4;
536: else if(UTF8Char<0x4000000)
537: return 5;
538: else if(UTF8Char<= 0x7FFFFFFF)
539: return 6;
540:
541: // will use the replacement character '?'
542: firstByte=0;
543: return 1;
544: }
545:
1.85 misha 546: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
547: size_t dest_length=0;
548:
549: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
550: if(i.getCharSize()==1)
551: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
552: else
553: dest_length+=6; // %uXXXX
1.60 misha 554: }
555:
1.85 misha 556: return dest_length;
1.60 misha 557: }
558:
1.86 moko 559: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
560: const XMLByte* src_end=src+src_length;
561: XMLByte first_byte;
562: XMLCh UTF8_char;
1.85 misha 563: size_t dest_length=0;
564:
1.86 moko 565: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 566: if(char_size==1)
567: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
568: else
569: dest_length+=6; // %uXXXX
1.60 misha 570: }
571:
1.85 misha 572: return dest_length;
573: }
574:
575: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 576: if(!src.length)
1.85 misha 577: return 0;
578:
579: #ifdef PRECALCULATE_DEST_LENGTH
580: if(source_charset.isUTF8())
1.86 moko 581: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 582: else
1.86 moko 583: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 584: #else
585: return src_length*6; // enough for %uXXXX but too memory-hungry
586: #endif
587: }
588:
589: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
590: if(char_size==1) \
591: if(first_byte){ \
592: if(need_escape(first_byte)) \
1.93 ! moko 593: append_hex_8((char*&)dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 594: else \
595: *dest_ptr++=first_byte; /*as is*/ \
596: } else \
597: *dest_ptr++='?'; /* replacement char '?' */ \
598: else \
1.93 ! moko 599: append_hex_16((char*&)dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 600:
601:
602: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
603: XMLByte* dest_ptr=dest;
604:
605: // loop until we either run out of input data
606: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
607: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 608:
1.85 misha 609: return dest_ptr - dest;
1.60 misha 610: }
611:
1.85 misha 612: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
613: const XMLByte* src_end=src+src_length;
614: XMLByte* dest_ptr=dest;
615:
616: XMLByte first_byte;
617: XMLCh UTF8_char;
618: uint char_size;
619:
1.86 moko 620: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 621: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
622:
623: return dest_ptr - dest;
624: }
1.60 misha 625:
626: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 627: if(!src.length)
1.60 misha 628: return String::C("", 0);
629:
1.85 misha 630: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
631: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
632:
633: size_t dest_length;
634: if(source_charset.isUTF8())
1.86 moko 635: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 636: else
1.86 moko 637: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 638:
639: if(dest_length>dest_calculated_length)
640: throw Exception(0, 0, "Charset::escape buffer overflow");
641:
642: dest_body[dest_length]=0; // terminator
643: return String::C((char*)dest_body, dest_length);
644: }
645:
646: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 647: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 648: return String::Body(dest.length ? dest.str:0);
649: }
650:
651: String& Charset::escape(const String& src, const Charset& source_charset) {
652: if(src.is_empty())
653: return *new String();
654:
655: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
656: }
657:
658: inline bool need_json_escape(unsigned char c){
659: return strchr("\n\"\\/\t\r\b\f", c)!=0;
660: }
661:
662: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
663: size_t dest_length=0;
664:
665: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 ! moko 666: if(i.getCharSize()==1){
! 667: XMLByte first_byte=i.getFirstByte();
! 668: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
! 669: } else
1.85 misha 670: dest_length+=6; // \uXXXX
671: }
672:
673: return dest_length;
674: }
675:
1.86 moko 676: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
677: const XMLByte* src_end=src+src_length;
1.85 misha 678: XMLByte first_byte;
679: XMLCh UTF8_char;
1.60 misha 680: size_t dest_length=0;
681:
1.86 moko 682: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 683: if(char_size==1)
1.93 ! moko 684: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 685: else
686: dest_length+=6; // \uXXXX
1.60 misha 687: }
1.85 misha 688:
689: return dest_length;
690: }
691:
692: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 693: if(!src.length)
1.85 misha 694: return 0;
695:
696: #ifdef PRECALCULATE_DEST_LENGTH
697: if(source_charset.isUTF8())
1.86 moko 698: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 699: else
1.86 moko 700: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 701: #else
1.85 misha 702: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 703: #endif
1.85 misha 704: }
705:
706: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
707: if(char_size==1) \
708: switch(first_byte){ \
709: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
710: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
711: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
712: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
713: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
714: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
715: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
716: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
717: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.93 ! moko 718: default : if(first_byte < 0x20) append_hex_16((char*&)dest_ptr, UTF8_char, "\\u"); \
! 719: else *dest_ptr++=first_byte; \
1.85 misha 720: } \
721: else \
1.93 ! moko 722: append_hex_16((char*&)dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 723:
724:
725: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
726: XMLByte* dest_ptr=dest;
727:
728: // loop until we either run out of input data
729: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
730: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
731:
732: return dest_ptr - dest;
733: }
734:
735: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
736: const XMLByte* src_end=src+src_length;
737: XMLByte* dest_ptr=dest;
738:
739: XMLByte first_byte;
740: XMLCh UTF8_char;
741: uint char_size;
742:
1.86 moko 743: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 744: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
745:
746: return dest_ptr - dest;
747: }
1.60 misha 748:
1.85 misha 749: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 750: if(!src.length)
1.85 misha 751: return String::C("", 0);
1.60 misha 752:
1.85 misha 753: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
754: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
755:
756: size_t dest_length;
757: if(source_charset.isUTF8())
1.86 moko 758: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 759: else
1.86 moko 760: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 761:
1.85 misha 762: if(dest_length>dest_calculated_length)
763: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 764:
765: dest_body[dest_length]=0; // terminator
766: return String::C((char*)dest_body, dest_length);
767: }
1.85 misha 768:
769: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 770: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 771: return String::Body(dest.length ? dest.str:0);
1.64 misha 772: }
773:
1.85 misha 774: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 775: if(src.is_empty())
1.73 misha 776: return *new String();
1.64 misha 777:
1.85 misha 778: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 779: }
1.60 misha 780:
1.35 paf 781: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 782: int src_length=src.length;
1.60 misha 783:
784: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 785: int dest_length=0;
1.60 misha 786: const XMLByte* srcPtr=(XMLByte*)src.str;
787: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 788: XMLByte firstByte;
789: XMLCh UTF8Char;
790: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 791: dest_length+=charSize;
792: #else
1.85 misha 793: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 794: #endif
795:
1.35 paf 796: #ifndef NDEBUG
1.71 misha 797: int saved_dest_length=dest_length;
1.35 paf 798: #endif
799: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 800:
801: if(::transcodeToUTF8(
1.35 paf 802: (XMLByte *)src.str, src_length,
803: dest_body, dest_length,
1.11 paf 804: tables)<0)
1.43 paf 805: throw Exception(0,
1.10 paf 806: 0,
1.11 paf 807: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 808:
1.60 misha 809: assert(dest_length<=saved_dest_length);
810: dest_body[dest_length]=0; // terminator
1.35 paf 811: return String::C((char*)dest_body, dest_length);
1.10 paf 812: }
1.38 paf 813:
814: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 815: int lo = 0;
816: int hi = table.size - 1;
1.39 paf 817: while(lo<=hi) {
1.38 paf 818: // Calc the mid point of the low and high offset.
1.39 paf 819: const unsigned int i = (lo + hi) / 2;
820:
821: XMLCh cur=table.records[i].from;
822: if(src==cur)
823: return table.records[i].to;
824: if(src>cur)
825: lo = i+1;
1.38 paf 826: else
1.39 paf 827: hi = i-1;
828: }
829:
830: // not found
1.38 paf 831: return src;
832: }
833:
1.58 misha 834: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 835: if(!src) {
836: // use the replacement character
837: *outPtr++= '?';
838: return;
839: }
840:
841: // Figure out how many bytes we need
842: unsigned int encodedBytes;
843: if(src<0x80)
844: encodedBytes = 1;
845: else if(src<0x800)
846: encodedBytes = 2;
847: else if(src<0x10000)
848: encodedBytes = 3;
849: else if(src<0x200000)
850: encodedBytes = 4;
851: else if(src<0x4000000)
852: encodedBytes = 5;
853: else if(src<= 0x7FFFFFFF)
854: encodedBytes = 6;
855: else {
856: // use the replacement character
857: *outPtr++= '?';
858: return;
859: }
860:
861: // And spit out the bytes. We spit them out in reverse order
862: // here, so bump up the output pointer and work down as we go.
863: outPtr+= encodedBytes;
864: switch(encodedBytes) {
865: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
866: src>>= 6;
867: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
868: src>>= 6;
869: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
870: src>>= 6;
871: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
872: src>>= 6;
873: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
874: src>>= 6;
875: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
876: }
877:
878: // Add the encoded bytes back in again to indicate we've eaten them
879: outPtr+= encodedBytes;
880: }
881:
882: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
883: const Charset::UTF8CaseTable& table) {
884: store_UTF8(change_case_UTF8(src, table), outPtr);
885: };
1.44 paf 886: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
887: XMLByte* toFill, size_t toFillLen,
888: const Charset::UTF8CaseTable& table) {
1.38 paf 889: const XMLByte* srcPtr=srcData;
1.44 paf 890: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 891: XMLByte* outPtr=toFill;
1.44 paf 892: XMLByte* outEnd=toFill+toFillLen;
893:
894: // We now loop until we either run out of input data, or room to store
895: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
896: // Get the next leading byte out
897: const XMLByte firstByte =* srcPtr;
1.38 paf 898:
1.60 misha 899: if(firstByte<=127) {
1.38 paf 900: change_case_UTF8(firstByte, outPtr, table);
901: srcPtr++;
902: continue;
903: }
904:
905: // See how many trailing src bytes this sequence is going to require
906: const unsigned int trailingBytes = gUTFBytes[firstByte];
907:
908: // Looks ok, so lets build up the value
909: uint tmpVal=0;
910: switch(trailingBytes) {
911: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
912: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
913: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
914: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
915: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
916: case 0: tmpVal+=*srcPtr++;
917: break;
918:
919: default:
920: throw Exception(0,
921: 0,
922: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
923: }
924: tmpVal-=gUTFOffsets[trailingBytes];
925:
926: // If it will fit into a single char, then put it in. Otherwise
927: // fail [*encode it as a surrogate pair. If its not valid, use the
928: // replacement char.*]
929: if(!(tmpVal & 0xFFFF0000))
930: change_case_UTF8(tmpVal, outPtr, table);
931: else
932: throw Exception(0,
933: 0,
934: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
935: }
936:
937: if(srcPtr!=outPtr)
938: throw Exception(0,
939: 0,
940: "change_case_UTF8 error: end pointers do not match");
941: }
942:
1.60 misha 943: static size_t getDecNumLength(XMLCh UTF8Char){
944: return
945: (UTF8Char < 100)
946: ?2
947: :(UTF8Char < 1000)
948: ?3
949: :(UTF8Char < 10000)
950: ?4
951: :5;
952: }
1.38 paf 953:
1.35 paf 954: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 955: int src_length=src.length;
1.60 misha 956: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 957: int dest_length=0;
1.82 misha 958: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 959: dest_length += ( i.next() & 0xFFFF0000 )
960: ? 3*i.getCharSize() // %XX for each byte
961: : ( xlatOneTo(i.next(), tables, 0) != 0 )
962: ? 1 // can convert it to a single char
963: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 964: }
965: #else
966: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 967: int dest_length=src_length*6;
1.60 misha 968: #endif
969:
1.35 paf 970: #ifndef NDEBUG
1.71 misha 971: int saved_dest_length=dest_length;
1.35 paf 972: #endif
973: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 974:
975: if(::transcodeFromUTF8(
1.82 misha 976: (XMLByte *)src.str, src_length,
1.35 paf 977: dest_body, dest_length,
1.11 paf 978: tables)<0)
1.43 paf 979: throw Exception(0,
1.10 paf 980: 0,
1.35 paf 981: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 982:
1.60 misha 983: assert(dest_length<=saved_dest_length);
984: dest_body[dest_length]=0; // terminator
1.35 paf 985: return String::C((char*)dest_body, dest_length);
1.1 paf 986: }
987:
988: /// transcode using both charsets
1.35 paf 989: const String::C Charset::transcodeToCharset(const String::C src,
1.80 misha 990: const Charset& dest_charset) const {
1.35 paf 991: if(&dest_charset==this)
992: return src;
993: else {
994: size_t dest_length=src.length;
995: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
996:
997: XMLByte* output=dest_body;
998: const XMLByte* input=(XMLByte *)src.str;
999: while(XMLCh c=*input++) {
1000: XMLCh curVal = tables.fromTable[c];
1001: *output++=curVal?
1002: xlatOneTo(curVal, dest_charset.tables, '?') // OK
1003: :'?'; // use the replacement character
1.6 paf 1004: }
1.1 paf 1005:
1.35 paf 1006: dest_body[dest_length]=0; // terminator
1007: return String::C((char*)dest_body, dest_length);
1.6 paf 1008: }
1.1 paf 1009: }
1010:
1.58 misha 1011: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 1012: if(isUTF8())
1.58 misha 1013: store_UTF8(src, outPtr);
1.59 misha 1014: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1015: *outPtr++=ch;
1.57 misha 1016: }
1017:
1.1 paf 1018: #ifdef XML
1.10 paf 1019:
1.35 paf 1020: static const Charset::Tables* tables[MAX_CHARSETS];
1021:
1.46 paf 1022: #ifdef PA_PATCHED_LIBXML_BACKWARD
1023:
1024: #define declareXml256ioFuncs(i) \
1025: static int xml256CharEncodingInputFunc##i( \
1026: unsigned char *out, int *outlen, \
1027: const unsigned char *in, int *inlen, void*) { \
1028: return transcodeToUTF8( \
1.71 misha 1029: in, *inlen, \
1030: out, *outlen, \
1.46 paf 1031: *tables[i]); \
1032: } \
1033: static int xml256CharEncodingOutputFunc##i( \
1034: unsigned char *out, int *outlen, \
1035: const unsigned char *in, int *inlen, void*) { \
1036: return transcodeFromUTF8( \
1.71 misha 1037: in, *inlen, \
1038: out, *outlen, \
1.46 paf 1039: *tables[i]); \
1040: }
1041:
1042: #else
1043:
1.35 paf 1044: #define declareXml256ioFuncs(i) \
1045: static int xml256CharEncodingInputFunc##i( \
1046: unsigned char *out, int *outlen, \
1047: const unsigned char *in, int *inlen) { \
1048: return transcodeToUTF8( \
1.71 misha 1049: in, *inlen, \
1050: out, *outlen, \
1.35 paf 1051: *tables[i]); \
1052: } \
1053: static int xml256CharEncodingOutputFunc##i( \
1054: unsigned char *out, int *outlen, \
1055: const unsigned char *in, int *inlen) { \
1056: return transcodeFromUTF8( \
1.71 misha 1057: in, *inlen, \
1058: out, *outlen, \
1.35 paf 1059: *tables[i]); \
1060: }
1061:
1.46 paf 1062: #endif
1063:
1064:
1.35 paf 1065: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1066: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1067: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1068: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1069: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1070:
1071: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1072: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1073: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1074: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1075: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1076: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1077: };
1078: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1079: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1080: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1081: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1082: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1083: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1084: };
1085: static size_t handlers_count=0;
1.10 paf 1086:
1087: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1088: if(handlers_count==MAX_CHARSETS)
1089: throw Exception(0,
1090: 0,
1091: "already allocated %d handlers, no space for new encoding '%s'",
1092: MAX_CHARSETS, name_cstr);
1093:
1.45 paf 1094: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 1095: {
1096: handler->name=name_cstr;
1097: handler->input=inputFuncs[handlers_count];
1098: handler->output=outputFuncs[handlers_count];
1099: ::tables[handlers_count]=&tables;
1100: handlers_count++;
1101: }
1.10 paf 1102:
1103: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1104:
1.10 paf 1105: }
1106:
1.37 paf 1107: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1108: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1109: transcoder(NAME); // check right way
1.15 paf 1110: }
1111:
1.37 paf 1112: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1113: if(!ftranscoder)
1.56 misha 1114: throw Exception(PARSER_RUNTIME,
1.35 paf 1115: new String(NAME, String::L_TAINTED),
1.10 paf 1116: "unsupported encoding");
1.35 paf 1117: return *ftranscoder;
1.10 paf 1118: }
1119:
1.54 paf 1120: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1121: if(!s)
1.35 paf 1122: return String::C("", 0);
1.8 paf 1123:
1.35 paf 1124: int inlen=strlen((const char*)s);
1.51 paf 1125: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1126: #ifndef NDEBUG
1127: int saved_outlen=outlen;
1128: #endif
1129: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1130:
1.30 paf 1131: int error;
1.35 paf 1132: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1133: error=output(
1.17 paf 1134: (unsigned char*)out, &outlen,
1.46 paf 1135: (const unsigned char*)s, &inlen
1136: #ifdef PA_PATCHED_LIBXML_BACKWARD
1137: ,0
1138: #endif
1139: );
1.30 paf 1140: } else {
1141: memcpy(out, s, outlen=inlen);
1142: error=0;
1143: }
1144: if(error<0)
1.23 paf 1145: throw Exception(0,
1.8 paf 1146: 0,
1.30 paf 1147: "transcode_cstr failed (%d)", error);
1.8 paf 1148:
1.35 paf 1149: assert(outlen<=saved_outlen); out[outlen]=0;
1150: return String::C(out, outlen);
1.14 paf 1151: }
1.54 paf 1152: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1153: String::C cstr=transcode_cstr(s);
1.75 misha 1154: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1155: }
1156:
1.8 paf 1157: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1158: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1159: xmlChar* out;
1.30 paf 1160: int outlen;
1161: int error;
1.35 paf 1162: #ifndef NDEBUG
1163: int saved_outlen;
1164: #endif
1165: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1166: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1167: #ifndef NDEBUG
1168: saved_outlen=outlen;
1169: #endif
1.47 paf 1170: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1171: error=input(
1.17 paf 1172: out, &outlen,
1.46 paf 1173: (const unsigned char*)buf, (int*)&buf_size
1174: #ifdef PA_PATCHED_LIBXML_BACKWARD
1175: ,0
1176: #endif
1177: );
1.30 paf 1178: } else {
1179: outlen=buf_size;
1.35 paf 1180: #ifndef NDEBUG
1181: saved_outlen=outlen;
1182: #endif
1183: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1184: memcpy(out, buf, outlen);
1185: error=0;
1186: }
1.17 paf 1187:
1.30 paf 1188: if(error<0)
1.23 paf 1189: throw Exception(0,
1.8 paf 1190: 0,
1.30 paf 1191: "transcode_buf failed (%d)", error);
1.8 paf 1192:
1.35 paf 1193: assert(outlen<=saved_outlen); out[outlen]=0;
1194: return out;
1.24 paf 1195: }
1.1 paf 1196:
1.79 misha 1197: xmlChar* Charset::transcode(const String& s) {
1198: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1199: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1200: }
1.35 paf 1201:
1.79 misha 1202: xmlChar* Charset::transcode(const String::Body s) {
1203: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1204: }
1.36 paf 1205: #endif
1.34 paf 1206:
1.37 paf 1207: String::Body Charset::transcode(const String::Body src,
1.34 paf 1208: const Charset& source_transcoder,
1.35 paf 1209: const Charset& dest_transcoder) {
1.34 paf 1210:
1.35 paf 1211: const char *src_ptr=src.cstr();
1.83 misha 1212: size_t src_size=src.length();
1.34 paf 1213:
1.77 misha 1214: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1215:
1.77 misha 1216: return String::Body(dest.length ? dest.str:0);
1.35 paf 1217: }
1218:
1219: String& Charset::transcode(const String& src,
1220: const Charset& source_transcoder,
1221: const Charset& dest_transcoder) {
1.72 misha 1222: if(src.is_empty())
1.73 misha 1223: return *new String();
1.34 paf 1224:
1.37 paf 1225: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1226: }
1227:
1.35 paf 1228: void Charset::transcode(ArrayString& src,
1.34 paf 1229: const Charset& source_transcoder,
1.35 paf 1230: const Charset& dest_transcoder) {
1231: for(size_t i=0; i<src.count(); i++)
1232: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1233: }
1234:
1235: #ifndef DOXYGEN
1236: struct Transcode_pair_info {
1237: const Charset* source_transcoder;
1238: const Charset* dest_transcoder;
1239: };
1240: #endif
1.76 misha 1241: static void transcode_pair(HashStringValue::key_type /*akey*/,
1.37 paf 1242: String::Body& avalue,
1.35 paf 1243: Transcode_pair_info* info) {
1244: avalue=Charset::transcode(avalue,
1245: *info->source_transcoder,
1246: *info->dest_transcoder);
1.34 paf 1247: }
1.61 misha 1248:
1.35 paf 1249: void Charset::transcode(HashStringString& src,
1.34 paf 1250: const Charset& source_transcoder,
1.35 paf 1251: const Charset& dest_transcoder) {
1252: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1253: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1254: }
1.61 misha 1255:
1256: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1257: const XMLByte* ptr=srcBegin;
1.70 misha 1258: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1259:
1260: return ptr-srcBegin;
1261: }
1262:
1263: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1264: size_t charPos=0;
1265: const XMLByte* ptr=srcBegin;
1266: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1267: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1268: if(ptr>ptrEnd)
1269: return charPos;
1270: charPos++;
1271: }
1272:
1273: // scan till end but position in bytes still too low
1274: throw Exception(0,
1275: 0,
1276: "Error convertion byte pos to char pos");
1277: }
1278:
1279: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1280: size_t size=0;
1.70 misha 1281: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1282: size++;
1283:
1284: return size;
1285: }
1.80 misha 1286:
1.84 misha 1287: unsigned int lengthUTF8Char(const XMLByte c){
1288: return gUTFBytes[c]+1;
1289: }
1290:
1.80 misha 1291: bool UTF8_string_iterator::has_next(){
1292: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1293: return fcharSize!=0;
1294: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main