Annotation of parser3/src/main/pa_charset.C, revision 1.49
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.48 paf 4: Copyright(c) 2001-2004 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.49 ! paf 8: static const char * const IDENT_CHARSET_C="$Date: 2004/02/11 15:33:15 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.35 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
18:
1.38 paf 19: // globals
20:
21: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
22: #include "utf8-to-upper.inc"
23: };
24: Charset::UTF8CaseTable UTF8CaseToUpper={
25: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
26: UTF8CaseToUpperRecords};
27:
28: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
29: #include "utf8-to-lower.inc"
30: };
31: Charset::UTF8CaseTable UTF8CaseToLower={
32: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
33: UTF8CaseToLowerRecords};
34:
1.1 paf 35: // helpers
36:
37: inline void prepare_case_tables(unsigned char *tables) {
38: unsigned char *lcc_table=tables+lcc_offset;
39: unsigned char *fcc_table=tables+fcc_offset;
40: for(int i=0; i<0x100; i++)
41: lcc_table[i]=fcc_table[i]=i;
42: }
43: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
44: unsigned char bit) {
45: unsigned char *ctypes_table=tables+ctypes_offset;
46: ctypes_table[0]=bit;
47: for(; *cstr; cstr++) {
48: unsigned char c=*cstr;
49: ctypes_table[c]|=bit;
50: }
51: }
1.35 paf 52: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 53: if(!cstr || !*cstr)
54: return 0;
55: if(cstr[1]==0)
1.4 paf 56: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 57:
58: char *error_pos;
1.4 paf 59: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 60: }
1.35 paf 61: inline bool to_bool(const char* cstr) {
1.1 paf 62: return cstr && *cstr!=0;
63: }
64: static void element2ctypes(unsigned char c, bool belongs,
65: unsigned char *tables, unsigned char bit, int group_offset=-1) {
66: if(!belongs)
67: return;
68:
69: unsigned char *ctypes_table=tables+ctypes_offset;
70:
71: ctypes_table[c]|=bit;
72: if(group_offset>=0)
1.4 paf 73: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 74: }
75: static void element2case(unsigned char from, unsigned char to,
76: unsigned char *tables) {
77: if(!to)
78: return;
79:
80: unsigned char *lcc_table=tables+lcc_offset;
81: unsigned char *fcc_table=tables+fcc_offset;
82: lcc_table[from]=to;
83: fcc_table[from]=to; fcc_table[to]=from;
84: }
85:
86: // methods
87:
88: extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c
1.37 paf 89: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 90: FNAME(ANAME),
91: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 92:
1.35 paf 93: if(afile_spec) {
1.1 paf 94: fisUTF8=false;
1.35 paf 95: load_definition(*charsets, *afile_spec);
1.1 paf 96: #ifdef XML
1.35 paf 97: addEncoding(FNAME_CSTR);
1.1 paf 98: #endif
99: } else {
100: fisUTF8=true;
1.4 paf 101: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.1 paf 102: memcpy(pcre_tables, pcre_default_tables, sizeof(pcre_tables));
103: }
104:
105: #ifdef XML
1.35 paf 106: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 107: #endif
108: }
109:
1.35 paf 110: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 111: // pcre_tables
112: // lowcase, flipcase, bits digit+word+whitespace, masks
113:
114: // must not move this inside of prepare_case_tables
115: // don't know the size there
116: memset(pcre_tables, 0, sizeof(pcre_tables));
117: prepare_case_tables(pcre_tables);
1.4 paf 118: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 119:
120: // charset
1.35 paf 121: memset(&tables, 0, sizeof(tables));
1.1 paf 122:
123: // loading text
1.35 paf 124: char *data=file_read_text(charsets, afile_spec);
1.1 paf 125:
126: // ignore header
127: getrow(&data);
128:
129: // parse cells
130: char *row;
1.42 paf 131: while((row=getrow(&data))) {
1.1 paf 132: // remove empty&comment lines
133: if(!*row || *row=='#')
134: continue;
135:
136: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
137: unsigned int c=0;
138: char *cell;
1.42 paf 139: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 140: switch(column) {
141: case 0: c=to_wchar_code(cell); break;
142: // pcre_tables
143: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
144: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
145: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
146: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
147: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
148: case 6: element2case(c, to_wchar_code(cell), pcre_tables); break;
149: case 7:
150: case 8:
151: // charset
1.10 paf 152: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.23 paf 153: throw Exception("parser.runtime",
1.35 paf 154: &afile_spec,
1.1 paf 155: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
156:
157: XMLCh unicode=(XMLCh)to_wchar_code(cell);
158: if(!unicode && column==7/*unicode1 column*/)
159: unicode=(XMLCh)c;
160: if(unicode) {
1.10 paf 161: if(!tables.fromTable[c])
162: tables.fromTable[c]=unicode;
163: tables.toTable[tables.toTableSize].intCh=unicode;
164: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
165: tables.toTableSize++;
1.1 paf 166: }
167: break;
168: }
169: }
170: };
171:
172: // sort by the Unicode code point
173: sort_ToTable();
174: }
175:
176: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
177: return
1.38 paf 178: static_cast<const Charset::Tables::Rec *>(a)->intCh-
179: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 180: }
181:
182: void Charset::sort_ToTable() {
1.10 paf 183: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 184: sort_cmp_Trans_rec_intCh);
185: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 186: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 187: //fclose(f);
188: }
189:
1.10 paf 190: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 191: const Charset::Tables& tables,
192: XMLByte not_found) {
1.39 paf 193: int lo = 0;
194: int hi = tables.toTableSize - 1;
195: while(lo<=hi) {
1.35 paf 196: // Calc the mid point of the low and high offset.
1.39 paf 197: const unsigned int i = (lo + hi) / 2;
198:
199: XMLCh cur=tables.toTable[i].intCh;
200: if(toXlat==cur)
201: return tables.toTable[i].extCh;
202: if(toXlat>cur)
203: lo = i+1;
1.1 paf 204: else
1.39 paf 205: hi = i-1;
206: }
1.35 paf 207:
208: return not_found;
1.1 paf 209: }
210:
1.35 paf 211: String::C Charset::transcode(const String::C src,
212: const Charset& source_charset,
213: const Charset& dest_charset) {
214: if(!src.length)
215: return String::C("", 0);
1.4 paf 216:
1.1 paf 217: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
218: default: // 0x00
1.35 paf 219: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 220: case 0x01:
1.35 paf 221: return source_charset.transcodeToUTF8(src);
1.1 paf 222: case 0x10:
1.35 paf 223: return dest_charset.transcodeFromUTF8(src);
1.1 paf 224: case 0x11:
1.35 paf 225: return src;
1.1 paf 226: }
227: }
228:
229: // ---------------------------------------------------------------------------
230: // Local static data
231: //
232: // gUTFBytes
233: // A list of counts of trailing bytes for each initial byte in the input.
234: //
235: // gUTFOffsets
236: // A list of values to offset each result char type, according to how
237: // many source bytes when into making it.
238: //
239: // gFirstByteMark
240: // A list of values to mask onto the first byte of an encoded sequence,
241: // indexed by the number of bytes used to create the sequence.
242: // ---------------------------------------------------------------------------
243: static const XMLByte gUTFBytes[0x100] = {
244: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
245: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
246: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
247: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
248: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
249: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
250: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
251: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
252: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
253: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
254: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
255: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
256: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
257: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
258: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
259: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
260: };
261:
262: static const uint gUTFOffsets[6] = {
263: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
264: };
265:
266: static const XMLByte gFirstByteMark[7] = {
267: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
268: };
269:
1.35 paf 270: static int transcodeToUTF8(const XMLByte* srcData, size_t& srcLen,
271: XMLByte *toFill, size_t& toFillLen,
272: const Charset::Tables& tables) {
1.11 paf 273: const XMLByte* srcPtr=srcData;
274: const XMLByte* srcEnd=srcData+srcLen;
275: XMLByte* outPtr=toFill;
276: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 277:
1.35 paf 278: while(srcPtr<srcEnd) {
279: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 280: if(!curVal) {
1.35 paf 281: // use the replacement character
282: *outPtr++= '?';
283: srcPtr++;
284: continue;
285: }
1.1 paf 286:
1.35 paf 287: // Figure out how many bytes we need
288: unsigned int encodedBytes;
289: if(curVal<0x80)
290: encodedBytes = 1;
291: else if(curVal<0x800)
292: encodedBytes = 2;
293: else if(curVal<0x10000)
294: encodedBytes = 3;
295: else if(curVal<0x200000)
296: encodedBytes = 4;
297: else if(curVal<0x4000000)
298: encodedBytes = 5;
299: else if(curVal<= 0x7FFFFFFF)
300: encodedBytes = 6;
301: else {
302: // use the replacement character
303: *outPtr++= '?';
304: srcPtr++;
305: continue;
306: }
1.11 paf 307:
1.35 paf 308: // If we cannot fully get this char into the output buffer
309: if (outPtr + encodedBytes > outEnd)
310: break;
311:
312: // We can do it, so update the source index
313: srcPtr++;
314:
315: // And spit out the bytes. We spit them out in reverse order
316: // here, so bump up the output pointer and work down as we go.
317: outPtr+= encodedBytes;
318: switch(encodedBytes) {
319: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
320: curVal>>= 6;
321: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
322: curVal>>= 6;
323: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
324: curVal>>= 6;
325: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
326: curVal>>= 6;
327: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
328: curVal>>= 6;
329: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
330: }
331:
332: // Add the encoded bytes back in again to indicate we've eaten them
333: outPtr+= encodedBytes;
334: }
335:
336: // Update the bytes eaten
337: srcLen = srcPtr - srcData;
338:
339: // Return the characters read
340: toFillLen = outPtr - toFill;
341:
1.29 paf 342: //return srcPtr==srcEnd?(int)toFillLen:-1;
343: /*
344: xmlCharEncodingInputFunc
345: Returns :
346: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
347: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
348: of ocetes consumed.
349: */
350: return 0;
1.1 paf 351: }
1.26 paf 352: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.35 paf 353: static int transcodeFromUTF8(const XMLByte* srcData, size_t& srcLen,
354: XMLByte* toFill, size_t& toFillLen,
355: const Charset::Tables& tables) {
1.11 paf 356: const XMLByte* srcPtr=srcData;
357: const XMLByte* srcEnd=srcData+srcLen;
358: XMLByte* outPtr=toFill;
359: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 360:
1.35 paf 361: // We now loop until we either run out of input data, or room to store
362: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
363: // Get the next leading byte out
364: const XMLByte firstByte =* srcPtr;
365:
366: // Special-case ASCII, which is a leading byte value of<= 127
367: if(firstByte<= 127) {
368: *outPtr++= firstByte;
369: srcPtr++;
370: continue;
371: }
372:
373: // See how many trailing src bytes this sequence is going to require
374: const unsigned int trailingBytes = gUTFBytes[firstByte];
375:
376: // If there are not enough source bytes to do this one, then we
377: // are done. Note that we done>= here because we are implicitly
378: // counting the 1 byte we get no matter what.
379: if(srcPtr+trailingBytes>= srcEnd)
380: break;
381:
382: // Looks ok, so lets build up the value
383: uint tmpVal=0;
384: switch(trailingBytes) {
385: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
386: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
387: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
388: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
389: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
390: case 0: tmpVal+=*srcPtr++;
391: break;
392:
393: default:
394: throw Exception(0,
395: 0,
1.49 ! paf 396: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 397: }
398: tmpVal-=gUTFOffsets[trailingBytes];
399:
400: // If it will fit into a single char, then put it in. Otherwise
401: // fail [*encode it as a surrogate pair. If its not valid, use the
402: // replacement char.*]
403: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 404: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
405: *outPtr++=xlat;
1.49 ! paf 406: else {
! 407: if(tmpVal & 0xFFFFFF00)
! 408: goto fail;
1.25 paf 409: outPtr+=sprintf((char *)outPtr, "&#%d;", tmpVal); // &#decimal;
1.49 ! paf 410: }
! 411: } else {
! 412: fail:
! 413: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
! 414: for(uint i=0; i<=trailingBytes; i++)
! 415: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
! 416: }
1.1 paf 417: }
1.35 paf 418:
419: // Update the bytes eaten
420: srcLen = srcPtr - srcData;
421:
422: // Return the characters read
423: toFillLen = outPtr - toFill;
1.11 paf 424:
1.29 paf 425: //return srcPtr==srcEnd?(int)toFillLen:-1;
426: /*
427: xmlCharEncodingOutputFunc
428: Returns :
429: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
430: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
431: of ocetes consumed.
432: */
433: return 0;
1.10 paf 434: }
435:
436: /// @todo not so memory-hungry with prescan
1.35 paf 437: const String::C Charset::transcodeToUTF8(const String::C src) const {
438: size_t src_length=src.length;
439: size_t dest_length=src.length*6/*so that surly enough, max utf8 seq len=6*/;
440: #ifndef NDEBUG
441: size_t saved_dest_length=dest_length;
442: #endif
443: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 444:
445: if(::transcodeToUTF8(
1.35 paf 446: (XMLByte *)src.str, src_length,
447: dest_body, dest_length,
1.11 paf 448: tables)<0)
1.43 paf 449: throw Exception(0,
1.10 paf 450: 0,
1.11 paf 451: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 452:
1.35 paf 453: assert(dest_length<=saved_dest_length); dest_body[dest_length]=0; // terminator
454: return String::C((char*)dest_body, dest_length);
1.10 paf 455: }
1.38 paf 456:
457: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.39 paf 458: int lo = 0;
459: int hi = table.size - 1;
460: while(lo<=hi) {
1.38 paf 461: // Calc the mid point of the low and high offset.
1.39 paf 462: const unsigned int i = (lo + hi) / 2;
463:
464: XMLCh cur=table.records[i].from;
465: if(src==cur)
466: return table.records[i].to;
467: if(src>cur)
468: lo = i+1;
1.38 paf 469: else
1.39 paf 470: hi = i-1;
471: }
472:
473: // not found
1.38 paf 474: return src;
475: }
476:
477: static void store_UTF8(XMLCh src, XMLByte*& outPtr ) {
478: if(!src) {
479: // use the replacement character
480: *outPtr++= '?';
481: return;
482: }
483:
484: // Figure out how many bytes we need
485: unsigned int encodedBytes;
486: if(src<0x80)
487: encodedBytes = 1;
488: else if(src<0x800)
489: encodedBytes = 2;
490: else if(src<0x10000)
491: encodedBytes = 3;
492: else if(src<0x200000)
493: encodedBytes = 4;
494: else if(src<0x4000000)
495: encodedBytes = 5;
496: else if(src<= 0x7FFFFFFF)
497: encodedBytes = 6;
498: else {
499: // use the replacement character
500: *outPtr++= '?';
501: return;
502: }
503:
504: // And spit out the bytes. We spit them out in reverse order
505: // here, so bump up the output pointer and work down as we go.
506: outPtr+= encodedBytes;
507: switch(encodedBytes) {
508: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
509: src>>= 6;
510: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
511: src>>= 6;
512: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
513: src>>= 6;
514: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
515: src>>= 6;
516: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
517: src>>= 6;
518: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
519: }
520:
521: // Add the encoded bytes back in again to indicate we've eaten them
522: outPtr+= encodedBytes;
523: }
524:
525: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
526: const Charset::UTF8CaseTable& table) {
527: store_UTF8(change_case_UTF8(src, table), outPtr);
528: };
1.44 paf 529: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
530: XMLByte* toFill, size_t toFillLen,
531: const Charset::UTF8CaseTable& table) {
1.38 paf 532: const XMLByte* srcPtr=srcData;
1.44 paf 533: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 534: XMLByte* outPtr=toFill;
1.44 paf 535: XMLByte* outEnd=toFill+toFillLen;
536:
537: // We now loop until we either run out of input data, or room to store
538: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
539: // Get the next leading byte out
540: const XMLByte firstByte =* srcPtr;
1.38 paf 541:
542: if(firstByte<= 127) {
543: change_case_UTF8(firstByte, outPtr, table);
544: srcPtr++;
545: continue;
546: }
547:
548: // See how many trailing src bytes this sequence is going to require
549: const unsigned int trailingBytes = gUTFBytes[firstByte];
550:
551: // Looks ok, so lets build up the value
552: uint tmpVal=0;
553: switch(trailingBytes) {
554: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
555: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
556: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
557: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
558: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
559: case 0: tmpVal+=*srcPtr++;
560: break;
561:
562: default:
563: throw Exception(0,
564: 0,
565: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
566: }
567: tmpVal-=gUTFOffsets[trailingBytes];
568:
569: // If it will fit into a single char, then put it in. Otherwise
570: // fail [*encode it as a surrogate pair. If its not valid, use the
571: // replacement char.*]
572: if(!(tmpVal & 0xFFFF0000))
573: change_case_UTF8(tmpVal, outPtr, table);
574: else
575: throw Exception(0,
576: 0,
577: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
578: }
579:
580: if(srcPtr!=outPtr)
581: throw Exception(0,
582: 0,
583: "change_case_UTF8 error: end pointers do not match");
584: }
585:
586:
1.35 paf 587: const String::C Charset::transcodeFromUTF8(const String::C src) const {
588: size_t src_length=src.length;
589: size_t dest_length=src.length*6/*so that surly enough, "ÿ" has max ratio */;
590: #ifndef NDEBUG
591: size_t saved_dest_length=dest_length;
592: #endif
593: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 594:
595: if(::transcodeFromUTF8(
1.35 paf 596: (XMLByte *)src.str, src_length,
597: dest_body, dest_length,
1.11 paf 598: tables)<0)
1.43 paf 599: throw Exception(0,
1.10 paf 600: 0,
1.35 paf 601: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 602:
1.35 paf 603: assert(dest_length<=saved_dest_length); dest_body[dest_length]=0; // terminator
604: return String::C((char*)dest_body, dest_length);
1.1 paf 605: }
606:
607: /// transcode using both charsets
1.35 paf 608: const String::C Charset::transcodeToCharset(const String::C src,
609: const Charset& dest_charset) const {
610: if(&dest_charset==this)
611: return src;
612: else {
613: size_t dest_length=src.length;
614: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
615:
616: XMLByte* output=dest_body;
617: const XMLByte* input=(XMLByte *)src.str;
618: while(XMLCh c=*input++) {
619: XMLCh curVal = tables.fromTable[c];
620: *output++=curVal?
621: xlatOneTo(curVal, dest_charset.tables, '?') // OK
622: :'?'; // use the replacement character
1.6 paf 623: }
1.1 paf 624:
1.35 paf 625: dest_body[dest_length]=0; // terminator
626: return String::C((char*)dest_body, dest_length);
1.6 paf 627: }
1.1 paf 628: }
629:
630: #ifdef XML
1.10 paf 631:
1.35 paf 632: static const Charset::Tables* tables[MAX_CHARSETS];
633:
1.46 paf 634: #ifdef PA_PATCHED_LIBXML_BACKWARD
635:
636: #define declareXml256ioFuncs(i) \
637: static int xml256CharEncodingInputFunc##i( \
638: unsigned char *out, int *outlen, \
639: const unsigned char *in, int *inlen, void*) { \
640: return transcodeToUTF8( \
641: in, *(size_t*)inlen, \
642: out, *(size_t*)outlen, \
643: *tables[i]); \
644: } \
645: static int xml256CharEncodingOutputFunc##i( \
646: unsigned char *out, int *outlen, \
647: const unsigned char *in, int *inlen, void*) { \
648: return transcodeFromUTF8( \
649: in, *(size_t*)inlen, \
650: out, *(size_t*)outlen, \
651: *tables[i]); \
652: }
653:
654: #else
655:
1.35 paf 656: #define declareXml256ioFuncs(i) \
657: static int xml256CharEncodingInputFunc##i( \
658: unsigned char *out, int *outlen, \
659: const unsigned char *in, int *inlen) { \
660: return transcodeToUTF8( \
661: in, *(size_t*)inlen, \
662: out, *(size_t*)outlen, \
663: *tables[i]); \
664: } \
665: static int xml256CharEncodingOutputFunc##i( \
666: unsigned char *out, int *outlen, \
667: const unsigned char *in, int *inlen) { \
668: return transcodeFromUTF8( \
669: in, *(size_t*)inlen, \
670: out, *(size_t*)outlen, \
671: *tables[i]); \
672: }
673:
1.46 paf 674: #endif
675:
676:
1.35 paf 677: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
678: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
679: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
680: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
681: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
682:
683: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
684: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
685: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
686: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
687: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
688: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
689: };
690: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
691: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
692: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
693: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
694: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
695: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
696: };
697: static size_t handlers_count=0;
1.10 paf 698:
699: void Charset::addEncoding(char *name_cstr) {
1.35 paf 700: if(handlers_count==MAX_CHARSETS)
701: throw Exception(0,
702: 0,
703: "already allocated %d handlers, no space for new encoding '%s'",
704: MAX_CHARSETS, name_cstr);
705:
1.45 paf 706: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 707: {
708: handler->name=name_cstr;
709: handler->input=inputFuncs[handlers_count];
710: handler->output=outputFuncs[handlers_count];
711: ::tables[handlers_count]=&tables;
712: handlers_count++;
713: }
1.10 paf 714:
715: xmlRegisterCharEncodingHandler(handler);
1.35 paf 716:
1.10 paf 717: }
718:
1.37 paf 719: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 720: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 721: transcoder(NAME); // check right way
1.15 paf 722: }
723:
1.37 paf 724: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 725: if(!ftranscoder)
1.23 paf 726: throw Exception("parser.runtime",
1.35 paf 727: new String(NAME, String::L_TAINTED),
1.10 paf 728: "unsupported encoding");
1.35 paf 729: return *ftranscoder;
1.10 paf 730: }
731:
1.35 paf 732: String::C Charset::transcode_cstr(xmlChar* s) {
1.13 paf 733: if(!s)
1.35 paf 734: return String::C("", 0);
1.8 paf 735:
1.35 paf 736: int inlen=strlen((const char*)s);
737: int outlen=inlen; // max
738: #ifndef NDEBUG
739: int saved_outlen=outlen;
740: #endif
741: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 742:
1.30 paf 743: int error;
1.35 paf 744: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 745: error=output(
1.17 paf 746: (unsigned char*)out, &outlen,
1.46 paf 747: (const unsigned char*)s, &inlen
748: #ifdef PA_PATCHED_LIBXML_BACKWARD
749: ,0
750: #endif
751: );
1.30 paf 752: } else {
753: memcpy(out, s, outlen=inlen);
754: error=0;
755: }
756: if(error<0)
1.23 paf 757: throw Exception(0,
1.8 paf 758: 0,
1.30 paf 759: "transcode_cstr failed (%d)", error);
1.8 paf 760:
1.35 paf 761: assert(outlen<=saved_outlen); out[outlen]=0;
762: return String::C(out, outlen);
1.14 paf 763: }
1.35 paf 764: const String& Charset::transcode(xmlChar* s) {
765: String::C cstr=transcode_cstr(s);
766: return *new String(cstr.str, cstr.length, true);
767: }
768: String::C Charset::transcode_cstr(GdomeDOMString* s) {
769: return s?transcode_cstr(BAD_CAST s->str)
770: :String::C("", 0);
771: }
772: const String& Charset::transcode(GdomeDOMString* s) {
773: String::C cstr=transcode_cstr(s);
774: return *new String(cstr.str, cstr.length, true);
1.1 paf 775: }
776:
1.8 paf 777: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 778: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
779: xmlChar* out;
1.30 paf 780: int outlen;
781: int error;
1.35 paf 782: #ifndef NDEBUG
783: int saved_outlen;
784: #endif
785: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.32 paf 786: outlen=buf_size*6/*max*/;
1.35 paf 787: #ifndef NDEBUG
788: saved_outlen=outlen;
789: #endif
1.47 paf 790: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 791: error=input(
1.17 paf 792: out, &outlen,
1.46 paf 793: (const unsigned char*)buf, (int*)&buf_size
794: #ifdef PA_PATCHED_LIBXML_BACKWARD
795: ,0
796: #endif
797: );
1.30 paf 798: } else {
799: outlen=buf_size;
1.35 paf 800: #ifndef NDEBUG
801: saved_outlen=outlen;
802: #endif
803: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 804: memcpy(out, buf, outlen);
805: error=0;
806: }
1.17 paf 807:
1.30 paf 808: if(error<0)
1.23 paf 809: throw Exception(0,
1.8 paf 810: 0,
1.30 paf 811: "transcode_buf failed (%d)", error);
1.8 paf 812:
1.35 paf 813: assert(outlen<=saved_outlen); out[outlen]=0;
814: return out;
1.24 paf 815: }
1.35 paf 816: GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char* buf, size_t buf_size) {
817: return GdomeDOMString_auto_ptr(transcode_buf2xchar(buf, buf_size));
1.1 paf 818: }
1.12 paf 819: GdomeDOMString_auto_ptr Charset::transcode(const String& s) {
1.35 paf 820: const char* cstr=s.cstr(String::L_UNSPECIFIED);
1.1 paf 821:
1.24 paf 822: return transcode_buf2dom(cstr, strlen(cstr));
1.1 paf 823: }
1.37 paf 824: GdomeDOMString_auto_ptr Charset::transcode(const String::Body s) {
1.35 paf 825: const char* cstr=s.cstr();
826:
827: return transcode_buf2dom(cstr, s.length());
828: }
1.36 paf 829: #endif
1.34 paf 830:
1.37 paf 831: String::Body Charset::transcode(const String::Body src,
1.34 paf 832: const Charset& source_transcoder,
1.35 paf 833: const Charset& dest_transcoder) {
1.34 paf 834:
1.35 paf 835: const char *src_ptr=src.cstr();
1.34 paf 836: size_t src_size=strlen(src_ptr);
837:
1.35 paf 838: String::C dest=Charset::transcode(String::C(src_ptr, src_size),
839: source_transcoder,
840: dest_transcoder);
1.34 paf 841:
1.37 paf 842: return String::Body(dest.str, dest.length);
1.35 paf 843: }
844:
845: String& Charset::transcode(const String& src,
846: const Charset& source_transcoder,
847: const Charset& dest_transcoder) {
848: if(!src.length())
849: return *new String("", 0, false);
1.34 paf 850:
1.37 paf 851: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 852: }
853:
1.35 paf 854: void Charset::transcode(ArrayString& src,
1.34 paf 855: const Charset& source_transcoder,
1.35 paf 856: const Charset& dest_transcoder) {
857: for(size_t i=0; i<src.count(); i++)
858: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 859: }
860:
861: #ifndef DOXYGEN
862: struct Transcode_pair_info {
863: const Charset* source_transcoder;
864: const Charset* dest_transcoder;
865: };
866: #endif
1.40 paf 867: static void transcode_pair(const String::Body /*akey*/,
1.37 paf 868: String::Body& avalue,
1.35 paf 869: Transcode_pair_info* info) {
870: avalue=Charset::transcode(avalue,
871: *info->source_transcoder,
872: *info->dest_transcoder);
1.34 paf 873: }
1.35 paf 874: void Charset::transcode(HashStringString& src,
1.34 paf 875: const Charset& source_transcoder,
1.35 paf 876: const Charset& dest_transcoder) {
877: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
878: src.for_each_ref(transcode_pair, &info);
1.34 paf 879: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main