Annotation of parser3/src/main/pa_charset.C, revision 1.45
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.35 paf 4: Copyright(c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.45 ! paf 8: static const char * const IDENT_CHARSET_C="$Date: 2003/12/10 14:17:45 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.35 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.38 paf 17: // globals
18:
19: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
20: #include "utf8-to-upper.inc"
21: };
22: Charset::UTF8CaseTable UTF8CaseToUpper={
23: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
24: UTF8CaseToUpperRecords};
25:
26: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
27: #include "utf8-to-lower.inc"
28: };
29: Charset::UTF8CaseTable UTF8CaseToLower={
30: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
31: UTF8CaseToLowerRecords};
32:
1.1 paf 33: // helpers
34:
35: inline void prepare_case_tables(unsigned char *tables) {
36: unsigned char *lcc_table=tables+lcc_offset;
37: unsigned char *fcc_table=tables+fcc_offset;
38: for(int i=0; i<0x100; i++)
39: lcc_table[i]=fcc_table[i]=i;
40: }
41: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
42: unsigned char bit) {
43: unsigned char *ctypes_table=tables+ctypes_offset;
44: ctypes_table[0]=bit;
45: for(; *cstr; cstr++) {
46: unsigned char c=*cstr;
47: ctypes_table[c]|=bit;
48: }
49: }
1.35 paf 50: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 51: if(!cstr || !*cstr)
52: return 0;
53: if(cstr[1]==0)
1.4 paf 54: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 55:
56: char *error_pos;
1.4 paf 57: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 58: }
1.35 paf 59: inline bool to_bool(const char* cstr) {
1.1 paf 60: return cstr && *cstr!=0;
61: }
62: static void element2ctypes(unsigned char c, bool belongs,
63: unsigned char *tables, unsigned char bit, int group_offset=-1) {
64: if(!belongs)
65: return;
66:
67: unsigned char *ctypes_table=tables+ctypes_offset;
68:
69: ctypes_table[c]|=bit;
70: if(group_offset>=0)
1.4 paf 71: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 72: }
73: static void element2case(unsigned char from, unsigned char to,
74: unsigned char *tables) {
75: if(!to)
76: return;
77:
78: unsigned char *lcc_table=tables+lcc_offset;
79: unsigned char *fcc_table=tables+fcc_offset;
80: lcc_table[from]=to;
81: fcc_table[from]=to; fcc_table[to]=from;
82: }
83:
84: // methods
85:
86: extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c
1.37 paf 87: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 88: FNAME(ANAME),
89: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 90:
1.35 paf 91: if(afile_spec) {
1.1 paf 92: fisUTF8=false;
1.35 paf 93: load_definition(*charsets, *afile_spec);
1.1 paf 94: #ifdef XML
1.35 paf 95: addEncoding(FNAME_CSTR);
1.1 paf 96: #endif
97: } else {
98: fisUTF8=true;
1.4 paf 99: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.1 paf 100: memcpy(pcre_tables, pcre_default_tables, sizeof(pcre_tables));
101: }
102:
103: #ifdef XML
1.35 paf 104: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 105: #endif
106: }
107:
1.35 paf 108: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 109: // pcre_tables
110: // lowcase, flipcase, bits digit+word+whitespace, masks
111:
112: // must not move this inside of prepare_case_tables
113: // don't know the size there
114: memset(pcre_tables, 0, sizeof(pcre_tables));
115: prepare_case_tables(pcre_tables);
1.4 paf 116: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 117:
118: // charset
1.35 paf 119: memset(&tables, 0, sizeof(tables));
1.1 paf 120:
121: // loading text
1.35 paf 122: char *data=file_read_text(charsets, afile_spec);
1.1 paf 123:
124: // ignore header
125: getrow(&data);
126:
127: // parse cells
128: char *row;
1.42 paf 129: while((row=getrow(&data))) {
1.1 paf 130: // remove empty&comment lines
131: if(!*row || *row=='#')
132: continue;
133:
134: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
135: unsigned int c=0;
136: char *cell;
1.42 paf 137: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 138: switch(column) {
139: case 0: c=to_wchar_code(cell); break;
140: // pcre_tables
141: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
142: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
143: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
144: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
145: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
146: case 6: element2case(c, to_wchar_code(cell), pcre_tables); break;
147: case 7:
148: case 8:
149: // charset
1.10 paf 150: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.23 paf 151: throw Exception("parser.runtime",
1.35 paf 152: &afile_spec,
1.1 paf 153: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
154:
155: XMLCh unicode=(XMLCh)to_wchar_code(cell);
156: if(!unicode && column==7/*unicode1 column*/)
157: unicode=(XMLCh)c;
158: if(unicode) {
1.10 paf 159: if(!tables.fromTable[c])
160: tables.fromTable[c]=unicode;
161: tables.toTable[tables.toTableSize].intCh=unicode;
162: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
163: tables.toTableSize++;
1.1 paf 164: }
165: break;
166: }
167: }
168: };
169:
170: // sort by the Unicode code point
171: sort_ToTable();
172: }
173:
174: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
175: return
1.38 paf 176: static_cast<const Charset::Tables::Rec *>(a)->intCh-
177: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 178: }
179:
180: void Charset::sort_ToTable() {
1.10 paf 181: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 182: sort_cmp_Trans_rec_intCh);
183: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 184: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 185: //fclose(f);
186: }
187:
1.10 paf 188: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 189: const Charset::Tables& tables,
190: XMLByte not_found) {
1.39 paf 191: int lo = 0;
192: int hi = tables.toTableSize - 1;
193: while(lo<=hi) {
1.35 paf 194: // Calc the mid point of the low and high offset.
1.39 paf 195: const unsigned int i = (lo + hi) / 2;
196:
197: XMLCh cur=tables.toTable[i].intCh;
198: if(toXlat==cur)
199: return tables.toTable[i].extCh;
200: if(toXlat>cur)
201: lo = i+1;
1.1 paf 202: else
1.39 paf 203: hi = i-1;
204: }
1.35 paf 205:
206: return not_found;
1.1 paf 207: }
208:
1.35 paf 209: String::C Charset::transcode(const String::C src,
210: const Charset& source_charset,
211: const Charset& dest_charset) {
212: if(!src.length)
213: return String::C("", 0);
1.4 paf 214:
1.1 paf 215: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
216: default: // 0x00
1.35 paf 217: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 218: case 0x01:
1.35 paf 219: return source_charset.transcodeToUTF8(src);
1.1 paf 220: case 0x10:
1.35 paf 221: return dest_charset.transcodeFromUTF8(src);
1.1 paf 222: case 0x11:
1.35 paf 223: return src;
1.1 paf 224: }
225: }
226:
227: // ---------------------------------------------------------------------------
228: // Local static data
229: //
230: // gUTFBytes
231: // A list of counts of trailing bytes for each initial byte in the input.
232: //
233: // gUTFOffsets
234: // A list of values to offset each result char type, according to how
235: // many source bytes when into making it.
236: //
237: // gFirstByteMark
238: // A list of values to mask onto the first byte of an encoded sequence,
239: // indexed by the number of bytes used to create the sequence.
240: // ---------------------------------------------------------------------------
241: static const XMLByte gUTFBytes[0x100] = {
242: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
243: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
244: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
245: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
246: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
247: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
248: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
249: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
250: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
251: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
252: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
253: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
254: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
255: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
256: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
257: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
258: };
259:
260: static const uint gUTFOffsets[6] = {
261: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
262: };
263:
264: static const XMLByte gFirstByteMark[7] = {
265: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
266: };
267:
1.35 paf 268: static int transcodeToUTF8(const XMLByte* srcData, size_t& srcLen,
269: XMLByte *toFill, size_t& toFillLen,
270: const Charset::Tables& tables) {
1.11 paf 271: const XMLByte* srcPtr=srcData;
272: const XMLByte* srcEnd=srcData+srcLen;
273: XMLByte* outPtr=toFill;
274: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 275:
1.35 paf 276: while(srcPtr<srcEnd) {
277: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 278: if(!curVal) {
1.35 paf 279: // use the replacement character
280: *outPtr++= '?';
281: srcPtr++;
282: continue;
283: }
1.1 paf 284:
1.35 paf 285: // Figure out how many bytes we need
286: unsigned int encodedBytes;
287: if(curVal<0x80)
288: encodedBytes = 1;
289: else if(curVal<0x800)
290: encodedBytes = 2;
291: else if(curVal<0x10000)
292: encodedBytes = 3;
293: else if(curVal<0x200000)
294: encodedBytes = 4;
295: else if(curVal<0x4000000)
296: encodedBytes = 5;
297: else if(curVal<= 0x7FFFFFFF)
298: encodedBytes = 6;
299: else {
300: // use the replacement character
301: *outPtr++= '?';
302: srcPtr++;
303: continue;
304: }
1.11 paf 305:
1.35 paf 306: // If we cannot fully get this char into the output buffer
307: if (outPtr + encodedBytes > outEnd)
308: break;
309:
310: // We can do it, so update the source index
311: srcPtr++;
312:
313: // And spit out the bytes. We spit them out in reverse order
314: // here, so bump up the output pointer and work down as we go.
315: outPtr+= encodedBytes;
316: switch(encodedBytes) {
317: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
318: curVal>>= 6;
319: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
320: curVal>>= 6;
321: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
322: curVal>>= 6;
323: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
324: curVal>>= 6;
325: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
326: curVal>>= 6;
327: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
328: }
329:
330: // Add the encoded bytes back in again to indicate we've eaten them
331: outPtr+= encodedBytes;
332: }
333:
334: // Update the bytes eaten
335: srcLen = srcPtr - srcData;
336:
337: // Return the characters read
338: toFillLen = outPtr - toFill;
339:
1.29 paf 340: //return srcPtr==srcEnd?(int)toFillLen:-1;
341: /*
342: xmlCharEncodingInputFunc
343: Returns :
344: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
345: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
346: of ocetes consumed.
347: */
348: return 0;
1.1 paf 349: }
1.26 paf 350: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.35 paf 351: static int transcodeFromUTF8(const XMLByte* srcData, size_t& srcLen,
352: XMLByte* toFill, size_t& toFillLen,
353: const Charset::Tables& tables) {
1.11 paf 354: const XMLByte* srcPtr=srcData;
355: const XMLByte* srcEnd=srcData+srcLen;
356: XMLByte* outPtr=toFill;
357: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 358:
1.35 paf 359: // We now loop until we either run out of input data, or room to store
360: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
361: // Get the next leading byte out
362: const XMLByte firstByte =* srcPtr;
363:
364: // Special-case ASCII, which is a leading byte value of<= 127
365: if(firstByte<= 127) {
366: *outPtr++= firstByte;
367: srcPtr++;
368: continue;
369: }
370:
371: // See how many trailing src bytes this sequence is going to require
372: const unsigned int trailingBytes = gUTFBytes[firstByte];
373:
374: // If there are not enough source bytes to do this one, then we
375: // are done. Note that we done>= here because we are implicitly
376: // counting the 1 byte we get no matter what.
377: if(srcPtr+trailingBytes>= srcEnd)
378: break;
379:
380: // Looks ok, so lets build up the value
381: uint tmpVal=0;
382: switch(trailingBytes) {
383: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
384: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
385: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
386: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
387: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
388: case 0: tmpVal+=*srcPtr++;
389: break;
390:
391: default:
392: throw Exception(0,
393: 0,
394: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes);
395: }
396: tmpVal-=gUTFOffsets[trailingBytes];
397:
398: // If it will fit into a single char, then put it in. Otherwise
399: // fail [*encode it as a surrogate pair. If its not valid, use the
400: // replacement char.*]
401: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 402: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
403: *outPtr++=xlat;
404: else
405: outPtr+=sprintf((char *)outPtr, "&#%d;", tmpVal); // &#decimal;
406: } else
1.23 paf 407: throw Exception(0,
1.35 paf 408: 0,
409: "transcodeFromUTF8 error: too big tmpVal(0x%08X)", tmpVal);
1.1 paf 410: }
1.35 paf 411:
412: // Update the bytes eaten
413: srcLen = srcPtr - srcData;
414:
415: // Return the characters read
416: toFillLen = outPtr - toFill;
1.11 paf 417:
1.29 paf 418: //return srcPtr==srcEnd?(int)toFillLen:-1;
419: /*
420: xmlCharEncodingOutputFunc
421: Returns :
422: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
423: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
424: of ocetes consumed.
425: */
426: return 0;
1.10 paf 427: }
428:
429: /// @todo not so memory-hungry with prescan
1.35 paf 430: const String::C Charset::transcodeToUTF8(const String::C src) const {
431: size_t src_length=src.length;
432: size_t dest_length=src.length*6/*so that surly enough, max utf8 seq len=6*/;
433: #ifndef NDEBUG
434: size_t saved_dest_length=dest_length;
435: #endif
436: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 437:
438: if(::transcodeToUTF8(
1.35 paf 439: (XMLByte *)src.str, src_length,
440: dest_body, dest_length,
1.11 paf 441: tables)<0)
1.43 paf 442: throw Exception(0,
1.10 paf 443: 0,
1.11 paf 444: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 445:
1.35 paf 446: assert(dest_length<=saved_dest_length); dest_body[dest_length]=0; // terminator
447: return String::C((char*)dest_body, dest_length);
1.10 paf 448: }
1.38 paf 449:
450: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.39 paf 451: int lo = 0;
452: int hi = table.size - 1;
453: while(lo<=hi) {
1.38 paf 454: // Calc the mid point of the low and high offset.
1.39 paf 455: const unsigned int i = (lo + hi) / 2;
456:
457: XMLCh cur=table.records[i].from;
458: if(src==cur)
459: return table.records[i].to;
460: if(src>cur)
461: lo = i+1;
1.38 paf 462: else
1.39 paf 463: hi = i-1;
464: }
465:
466: // not found
1.38 paf 467: return src;
468: }
469:
470: static void store_UTF8(XMLCh src, XMLByte*& outPtr ) {
471: if(!src) {
472: // use the replacement character
473: *outPtr++= '?';
474: return;
475: }
476:
477: // Figure out how many bytes we need
478: unsigned int encodedBytes;
479: if(src<0x80)
480: encodedBytes = 1;
481: else if(src<0x800)
482: encodedBytes = 2;
483: else if(src<0x10000)
484: encodedBytes = 3;
485: else if(src<0x200000)
486: encodedBytes = 4;
487: else if(src<0x4000000)
488: encodedBytes = 5;
489: else if(src<= 0x7FFFFFFF)
490: encodedBytes = 6;
491: else {
492: // use the replacement character
493: *outPtr++= '?';
494: return;
495: }
496:
497: // And spit out the bytes. We spit them out in reverse order
498: // here, so bump up the output pointer and work down as we go.
499: outPtr+= encodedBytes;
500: switch(encodedBytes) {
501: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
502: src>>= 6;
503: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
504: src>>= 6;
505: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
506: src>>= 6;
507: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
508: src>>= 6;
509: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
510: src>>= 6;
511: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
512: }
513:
514: // Add the encoded bytes back in again to indicate we've eaten them
515: outPtr+= encodedBytes;
516: }
517:
518: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
519: const Charset::UTF8CaseTable& table) {
520: store_UTF8(change_case_UTF8(src, table), outPtr);
521: };
1.44 paf 522: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
523: XMLByte* toFill, size_t toFillLen,
524: const Charset::UTF8CaseTable& table) {
1.38 paf 525: const XMLByte* srcPtr=srcData;
1.44 paf 526: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 527: XMLByte* outPtr=toFill;
1.44 paf 528: XMLByte* outEnd=toFill+toFillLen;
529:
530: // We now loop until we either run out of input data, or room to store
531: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
532: // Get the next leading byte out
533: const XMLByte firstByte =* srcPtr;
1.38 paf 534:
535: if(firstByte<= 127) {
536: change_case_UTF8(firstByte, outPtr, table);
537: srcPtr++;
538: continue;
539: }
540:
541: // See how many trailing src bytes this sequence is going to require
542: const unsigned int trailingBytes = gUTFBytes[firstByte];
543:
544: // Looks ok, so lets build up the value
545: uint tmpVal=0;
546: switch(trailingBytes) {
547: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
548: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
549: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
550: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
551: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
552: case 0: tmpVal+=*srcPtr++;
553: break;
554:
555: default:
556: throw Exception(0,
557: 0,
558: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
559: }
560: tmpVal-=gUTFOffsets[trailingBytes];
561:
562: // If it will fit into a single char, then put it in. Otherwise
563: // fail [*encode it as a surrogate pair. If its not valid, use the
564: // replacement char.*]
565: if(!(tmpVal & 0xFFFF0000))
566: change_case_UTF8(tmpVal, outPtr, table);
567: else
568: throw Exception(0,
569: 0,
570: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
571: }
572:
573: if(srcPtr!=outPtr)
574: throw Exception(0,
575: 0,
576: "change_case_UTF8 error: end pointers do not match");
577: }
578:
579:
1.35 paf 580: const String::C Charset::transcodeFromUTF8(const String::C src) const {
581: size_t src_length=src.length;
582: size_t dest_length=src.length*6/*so that surly enough, "ÿ" has max ratio */;
583: #ifndef NDEBUG
584: size_t saved_dest_length=dest_length;
585: #endif
586: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 587:
588: if(::transcodeFromUTF8(
1.35 paf 589: (XMLByte *)src.str, src_length,
590: dest_body, dest_length,
1.11 paf 591: tables)<0)
1.43 paf 592: throw Exception(0,
1.10 paf 593: 0,
1.35 paf 594: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 595:
1.35 paf 596: assert(dest_length<=saved_dest_length); dest_body[dest_length]=0; // terminator
597: return String::C((char*)dest_body, dest_length);
1.1 paf 598: }
599:
600: /// transcode using both charsets
1.35 paf 601: const String::C Charset::transcodeToCharset(const String::C src,
602: const Charset& dest_charset) const {
603: if(&dest_charset==this)
604: return src;
605: else {
606: size_t dest_length=src.length;
607: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
608:
609: XMLByte* output=dest_body;
610: const XMLByte* input=(XMLByte *)src.str;
611: while(XMLCh c=*input++) {
612: XMLCh curVal = tables.fromTable[c];
613: *output++=curVal?
614: xlatOneTo(curVal, dest_charset.tables, '?') // OK
615: :'?'; // use the replacement character
1.6 paf 616: }
1.1 paf 617:
1.35 paf 618: dest_body[dest_length]=0; // terminator
619: return String::C((char*)dest_body, dest_length);
1.6 paf 620: }
1.1 paf 621: }
622:
623: #ifdef XML
1.10 paf 624:
1.35 paf 625: static const Charset::Tables* tables[MAX_CHARSETS];
626:
627: #define declareXml256ioFuncs(i) \
628: static int xml256CharEncodingInputFunc##i( \
629: unsigned char *out, int *outlen, \
630: const unsigned char *in, int *inlen) { \
631: return transcodeToUTF8( \
632: in, *(size_t*)inlen, \
633: out, *(size_t*)outlen, \
634: *tables[i]); \
635: } \
636: static int xml256CharEncodingOutputFunc##i( \
637: unsigned char *out, int *outlen, \
638: const unsigned char *in, int *inlen) { \
639: return transcodeFromUTF8( \
640: in, *(size_t*)inlen, \
641: out, *(size_t*)outlen, \
642: *tables[i]); \
643: }
644:
645: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
646: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
647: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
648: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
649: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
650:
651: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
652: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
653: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
654: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
655: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
656: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
657: };
658: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
659: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
660: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
661: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
662: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
663: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
664: };
665: static size_t handlers_count=0;
1.10 paf 666:
667: void Charset::addEncoding(char *name_cstr) {
1.35 paf 668: if(handlers_count==MAX_CHARSETS)
669: throw Exception(0,
670: 0,
671: "already allocated %d handlers, no space for new encoding '%s'",
672: MAX_CHARSETS, name_cstr);
673:
1.45 ! paf 674: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 675: {
676: handler->name=name_cstr;
677: handler->input=inputFuncs[handlers_count];
678: handler->output=outputFuncs[handlers_count];
679: ::tables[handlers_count]=&tables;
680: handlers_count++;
681: }
1.10 paf 682:
683: xmlRegisterCharEncodingHandler(handler);
1.35 paf 684:
1.10 paf 685: }
686:
1.37 paf 687: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 688: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 689: transcoder(NAME); // check right way
1.15 paf 690: }
691:
1.37 paf 692: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 693: if(!ftranscoder)
1.23 paf 694: throw Exception("parser.runtime",
1.35 paf 695: new String(NAME, String::L_TAINTED),
1.10 paf 696: "unsupported encoding");
1.35 paf 697: return *ftranscoder;
1.10 paf 698: }
699:
1.35 paf 700: String::C Charset::transcode_cstr(xmlChar* s) {
1.13 paf 701: if(!s)
1.35 paf 702: return String::C("", 0);
1.8 paf 703:
1.35 paf 704: int inlen=strlen((const char*)s);
705: int outlen=inlen; // max
706: #ifndef NDEBUG
707: int saved_outlen=outlen;
708: #endif
709: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 710:
1.30 paf 711: int error;
1.35 paf 712: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 713: error=output(
1.17 paf 714: (unsigned char*)out, &outlen,
1.35 paf 715: (const unsigned char*)s, &inlen);
1.30 paf 716: } else {
717: memcpy(out, s, outlen=inlen);
718: error=0;
719: }
720: if(error<0)
1.23 paf 721: throw Exception(0,
1.8 paf 722: 0,
1.30 paf 723: "transcode_cstr failed (%d)", error);
1.8 paf 724:
1.35 paf 725: assert(outlen<=saved_outlen); out[outlen]=0;
726: return String::C(out, outlen);
1.14 paf 727: }
1.35 paf 728: const String& Charset::transcode(xmlChar* s) {
729: String::C cstr=transcode_cstr(s);
730: return *new String(cstr.str, cstr.length, true);
731: }
732: String::C Charset::transcode_cstr(GdomeDOMString* s) {
733: return s?transcode_cstr(BAD_CAST s->str)
734: :String::C("", 0);
735: }
736: const String& Charset::transcode(GdomeDOMString* s) {
737: String::C cstr=transcode_cstr(s);
738: return *new String(cstr.str, cstr.length, true);
1.1 paf 739: }
740:
1.8 paf 741: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 742: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
743: xmlChar* out;
1.30 paf 744: int outlen;
745: int error;
1.35 paf 746: #ifndef NDEBUG
747: int saved_outlen;
748: #endif
749: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.32 paf 750: outlen=buf_size*6/*max*/;
1.35 paf 751: #ifndef NDEBUG
752: saved_outlen=outlen;
753: #endif
754: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 755: error=input(
1.17 paf 756: out, &outlen,
1.35 paf 757: (const unsigned char*)buf, (int*)&buf_size);
1.30 paf 758: } else {
759: outlen=buf_size;
1.35 paf 760: #ifndef NDEBUG
761: saved_outlen=outlen;
762: #endif
763: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 764: memcpy(out, buf, outlen);
765: error=0;
766: }
1.17 paf 767:
1.30 paf 768: if(error<0)
1.23 paf 769: throw Exception(0,
1.8 paf 770: 0,
1.30 paf 771: "transcode_buf failed (%d)", error);
1.8 paf 772:
1.35 paf 773: assert(outlen<=saved_outlen); out[outlen]=0;
774: return out;
1.24 paf 775: }
1.35 paf 776: GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char* buf, size_t buf_size) {
777: return GdomeDOMString_auto_ptr(transcode_buf2xchar(buf, buf_size));
1.1 paf 778: }
1.12 paf 779: GdomeDOMString_auto_ptr Charset::transcode(const String& s) {
1.35 paf 780: const char* cstr=s.cstr(String::L_UNSPECIFIED);
1.1 paf 781:
1.24 paf 782: return transcode_buf2dom(cstr, strlen(cstr));
1.1 paf 783: }
1.37 paf 784: GdomeDOMString_auto_ptr Charset::transcode(const String::Body s) {
1.35 paf 785: const char* cstr=s.cstr();
786:
787: return transcode_buf2dom(cstr, s.length());
788: }
1.36 paf 789: #endif
1.34 paf 790:
1.37 paf 791: String::Body Charset::transcode(const String::Body src,
1.34 paf 792: const Charset& source_transcoder,
1.35 paf 793: const Charset& dest_transcoder) {
1.34 paf 794:
1.35 paf 795: const char *src_ptr=src.cstr();
1.34 paf 796: size_t src_size=strlen(src_ptr);
797:
1.35 paf 798: String::C dest=Charset::transcode(String::C(src_ptr, src_size),
799: source_transcoder,
800: dest_transcoder);
1.34 paf 801:
1.37 paf 802: return String::Body(dest.str, dest.length);
1.35 paf 803: }
804:
805: String& Charset::transcode(const String& src,
806: const Charset& source_transcoder,
807: const Charset& dest_transcoder) {
808: if(!src.length())
809: return *new String("", 0, false);
1.34 paf 810:
1.37 paf 811: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 812: }
813:
1.35 paf 814: void Charset::transcode(ArrayString& src,
1.34 paf 815: const Charset& source_transcoder,
1.35 paf 816: const Charset& dest_transcoder) {
817: for(size_t i=0; i<src.count(); i++)
818: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 819: }
820:
821: #ifndef DOXYGEN
822: struct Transcode_pair_info {
823: const Charset* source_transcoder;
824: const Charset* dest_transcoder;
825: };
826: #endif
1.40 paf 827: static void transcode_pair(const String::Body /*akey*/,
1.37 paf 828: String::Body& avalue,
1.35 paf 829: Transcode_pair_info* info) {
830: avalue=Charset::transcode(avalue,
831: *info->source_transcoder,
832: *info->dest_transcoder);
1.34 paf 833: }
1.35 paf 834: void Charset::transcode(HashStringString& src,
1.34 paf 835: const Charset& source_transcoder,
1.35 paf 836: const Charset& dest_transcoder) {
837: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
838: src.for_each_ref(transcode_pair, &info);
1.34 paf 839: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main