Annotation of parser3/src/main/pa_charset.C, revision 1.33.2.19.2.16
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.33.2.6 paf 4: Copyright(c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.33.2.19.2.1 (paf 8:): static const char* IDENT_CHARSET_C="$Date: 2003/04/02 10:59:18 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.33.2.13 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
17: // helpers
18:
19: inline void prepare_case_tables(unsigned char *tables) {
20: unsigned char *lcc_table=tables+lcc_offset;
21: unsigned char *fcc_table=tables+fcc_offset;
22: for(int i=0; i<0x100; i++)
23: lcc_table[i]=fcc_table[i]=i;
24: }
25: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
26: unsigned char bit) {
27: unsigned char *ctypes_table=tables+ctypes_offset;
28: ctypes_table[0]=bit;
29: for(; *cstr; cstr++) {
30: unsigned char c=*cstr;
31: ctypes_table[c]|=bit;
32: }
33: }
1.33.2.6 paf 34: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 35: if(!cstr || !*cstr)
36: return 0;
37: if(cstr[1]==0)
1.4 paf 38: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 39:
40: char *error_pos;
1.4 paf 41: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 42: }
1.33.2.6 paf 43: inline bool to_bool(const char* cstr) {
1.1 paf 44: return cstr && *cstr!=0;
45: }
46: static void element2ctypes(unsigned char c, bool belongs,
47: unsigned char *tables, unsigned char bit, int group_offset=-1) {
48: if(!belongs)
49: return;
50:
51: unsigned char *ctypes_table=tables+ctypes_offset;
52:
53: ctypes_table[c]|=bit;
54: if(group_offset>=0)
1.4 paf 55: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 56: }
57: static void element2case(unsigned char from, unsigned char to,
58: unsigned char *tables) {
59: if(!to)
60: return;
61:
62: unsigned char *lcc_table=tables+lcc_offset;
63: unsigned char *fcc_table=tables+fcc_offset;
64: lcc_table[from]=to;
65: fcc_table[from]=to; fcc_table[to]=from;
66: }
67:
68: // methods
69:
70: extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c
1.33.2.19.2.5 (paf 71:: Charset::Charset(const String& aname, const String* afile_spec):
1.33.2.1 paf 72: fname(aname),
1.33.2.19.2.5 (paf 73:: fname_cstr(aname.cstrm()) {
1.1 paf 74:
1.33.2.1 paf 75: for(char *c=fname_cstr; *c; c++)
1.33.2.19.2.5 (paf 76:: *c = toupper(*c);
1.7 paf 77:
1.33.2.3 paf 78: if(afile_spec) {
1.1 paf 79: fisUTF8=false;
1.33.2.19.2.5 (paf 80:: load_definition(*afile_spec);
1.1 paf 81: #ifdef XML
1.33.2.1 paf 82: addEncoding(fname_cstr);
1.1 paf 83: #endif
84: } else {
85: fisUTF8=true;
1.4 paf 86: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.1 paf 87: memcpy(pcre_tables, pcre_default_tables, sizeof(pcre_tables));
88: }
89:
90: #ifdef XML
1.33.2.1 paf 91: initTranscoder(fname, fname_cstr);
1.1 paf 92: #endif
93: }
94:
1.33.2.19.2.1 (paf 95:: void Charset::load_definition(const String& afile_spec) {
1.1 paf 96: // pcre_tables
97: // lowcase, flipcase, bits digit+word+whitespace, masks
98:
99: // must not move this inside of prepare_case_tables
100: // don't know the size there
101: memset(pcre_tables, 0, sizeof(pcre_tables));
102: prepare_case_tables(pcre_tables);
1.4 paf 103: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 104:
105: // charset
1.33.2.12 paf 106: memset(&tables, 0, sizeof(tables));
1.1 paf 107: // strangly vital
1.10 paf 108: tables.toTable[tables.toTableSize].intCh=0;
109: tables.toTable[tables.toTableSize].extCh=(XMLByte)0;
110: tables.toTableSize++;
1.1 paf 111:
112: // loading text
1.33.2.19.2.5 (paf 113:: char *data=file_read_text(UTF8_charset, afile_spec);
1.1 paf 114:
115: // ignore header
116: getrow(&data);
117:
118: // parse cells
119: char *row;
120: while(row=getrow(&data)) {
121: // remove empty&comment lines
122: if(!*row || *row=='#')
123: continue;
124:
125: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
126: unsigned int c=0;
127: char *cell;
128: for(int column=0; cell=lsplit(&row, '\t'); column++) {
129: switch(column) {
130: case 0: c=to_wchar_code(cell); break;
131: // pcre_tables
132: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
133: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
134: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
135: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
136: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
137: case 6: element2case(c, to_wchar_code(cell), pcre_tables); break;
138: case 7:
139: case 8:
140: // charset
1.10 paf 141: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.23 paf 142: throw Exception("parser.runtime",
1.33.2.19.2.5 (paf 143:: &afile_spec,
1.1 paf 144: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
145:
146: XMLCh unicode=(XMLCh)to_wchar_code(cell);
147: if(!unicode && column==7/*unicode1 column*/)
148: unicode=(XMLCh)c;
149: if(unicode) {
1.10 paf 150: if(!tables.fromTable[c])
151: tables.fromTable[c]=unicode;
152: tables.toTable[tables.toTableSize].intCh=unicode;
153: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
154: tables.toTableSize++;
1.1 paf 155: }
156: break;
157: }
158: }
159: };
160:
161: // sort by the Unicode code point
162: sort_ToTable();
163: }
164:
165: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
166: return
167: static_cast<const Charset_TransRec *>(a)->intCh-
168: static_cast<const Charset_TransRec *>(b)->intCh;
169: }
170:
171: void Charset::sort_ToTable() {
1.10 paf 172: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 173: sort_cmp_Trans_rec_intCh);
174: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 175: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 176: //fclose(f);
177: }
178:
1.10 paf 179: static XMLByte xlatOneTo(const XMLCh toXlat,
1.25 paf 180: const Charset::Tables& tables,
181: XMLByte not_found) {
1.1 paf 182: unsigned int lowOfs = 0;
1.10 paf 183: unsigned int hiOfs = tables.toTableSize - 1;
1.1 paf 184: XMLByte curByte = 0;
185: do {
186: // Calc the mid point of the low and high offset.
1.4 paf 187: const unsigned int midOfs =((hiOfs - lowOfs) / 2)+lowOfs;
1.1 paf 188:
189: // If our test char is greater than the mid point char, then
190: // we move up to the upper half. Else we move to the lower
191: // half. If its equal, then its our guy.
1.10 paf 192: if(toXlat>tables.toTable[midOfs].intCh)
1.1 paf 193: lowOfs = midOfs;
1.10 paf 194: else if(toXlat<tables.toTable[midOfs].intCh)
1.1 paf 195: hiOfs = midOfs;
196: else
1.10 paf 197: return tables.toTable[midOfs].extCh;
1.4 paf 198: } while(lowOfs+1<hiOfs);
1.1 paf 199:
1.25 paf 200: return not_found;
1.1 paf 201: }
202:
1.33.2.19.2.1 (paf 203:): String::C Charset::transcode(const String::C src,
204:): const Charset& source_charset,
205:): const Charset& dest_charset) {
206:): if(!src.length)
207:): return String::C(0, 0);
1.4 paf 208:
1.1 paf 209: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
210: default: // 0x00
1.33.2.19.2.1 (paf 211:): return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 212: case 0x01:
1.33.2.19.2.1 (paf 213:): return source_charset.transcodeToUTF8(src);
1.1 paf 214: case 0x10:
1.33.2.19.2.1 (paf 215:): return dest_charset.transcodeFromUTF8(src);
1.1 paf 216: case 0x11:
1.33.2.19.2.1 (paf 217:): return src;
1.1 paf 218: }
219: }
220:
221: // ---------------------------------------------------------------------------
222: // Local static data
223: //
224: // gUTFBytes
225: // A list of counts of trailing bytes for each initial byte in the input.
226: //
227: // gUTFOffsets
228: // A list of values to offset each result char type, according to how
229: // many source bytes when into making it.
230: //
231: // gFirstByteMark
232: // A list of values to mask onto the first byte of an encoded sequence,
233: // indexed by the number of bytes used to create the sequence.
234: // ---------------------------------------------------------------------------
235: static const XMLByte gUTFBytes[0x100] = {
236: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
237: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
238: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
239: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
240: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
241: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
242: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
243: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
244: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
245: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
246: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
247: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
248: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
249: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
250: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
251: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
252: };
253:
254: static const uint gUTFOffsets[6] = {
255: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
256: };
257:
258: static const XMLByte gFirstByteMark[7] = {
259: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
260: };
261:
1.11 paf 262: static int transcodeToUTF8(
1.33.2.19.2.1 (paf 263:): const XMLByte* srcData, size_t& srcLen,
264:): XMLByte *toFill, size_t& toFillLen,
265:): const Charset::Tables& tables) {
1.11 paf 266: const XMLByte* srcPtr=srcData;
267: const XMLByte* srcEnd=srcData+srcLen;
268: XMLByte* outPtr=toFill;
269: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 270:
1.4 paf 271: while(srcPtr<srcEnd) {
1.10 paf 272: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 273: if(!curVal) {
274: // use the replacement character
1.4 paf 275: *outPtr++= '?';
276: srcPtr++;
1.1 paf 277: continue;
278: }
279:
280: // Figure out how many bytes we need
281: unsigned int encodedBytes;
1.4 paf 282: if(curVal<0x80)
1.1 paf 283: encodedBytes = 1;
1.4 paf 284: else if(curVal<0x800)
1.1 paf 285: encodedBytes = 2;
1.4 paf 286: else if(curVal<0x10000)
1.1 paf 287: encodedBytes = 3;
1.4 paf 288: else if(curVal<0x200000)
1.1 paf 289: encodedBytes = 4;
1.4 paf 290: else if(curVal<0x4000000)
1.1 paf 291: encodedBytes = 5;
1.4 paf 292: else if(curVal<= 0x7FFFFFFF)
1.1 paf 293: encodedBytes = 6;
294: else {
295: // use the replacement character
1.4 paf 296: *outPtr++= '?';
297: srcPtr++;
1.1 paf 298: continue;
299: }
300:
1.10 paf 301: // If we cannot fully get this char into the output buffer
302: if (outPtr + encodedBytes > outEnd)
303: break;
1.1 paf 304:
305: // We can do it, so update the source index
306: srcPtr++;
307:
308: // And spit out the bytes. We spit them out in reverse order
309: // here, so bump up the output pointer and work down as we go.
1.4 paf 310: outPtr+= encodedBytes;
1.1 paf 311: switch(encodedBytes) {
1.18 paf 312: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
1.4 paf 313: curVal>>= 6;
1.18 paf 314: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
1.4 paf 315: curVal>>= 6;
1.18 paf 316: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
1.4 paf 317: curVal>>= 6;
1.18 paf 318: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
1.4 paf 319: curVal>>= 6;
1.18 paf 320: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
1.4 paf 321: curVal>>= 6;
1.18 paf 322: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.1 paf 323: }
324:
325: // Add the encoded bytes back in again to indicate we've eaten them
1.4 paf 326: outPtr+= encodedBytes;
1.1 paf 327: }
328:
1.11 paf 329: // Update the bytes eaten
330: srcLen = srcPtr - srcData;
331:
332: // Return the characters read
333: toFillLen = outPtr - toFill;
334:
1.29 paf 335: //return srcPtr==srcEnd?(int)toFillLen:-1;
336: /*
337: xmlCharEncodingInputFunc
338: Returns :
339: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
340: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
341: of ocetes consumed.
342: */
343: return 0;
1.1 paf 344: }
1.26 paf 345: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.30 paf 346: static int transcodeFromUTF8(
1.33.2.14 paf 347: const XMLByte* srcData, size_t& srcLen,
1.11 paf 348: XMLByte* toFill, size_t& toFillLen,
349: const Charset::Tables& tables) {
350: const XMLByte* srcPtr=srcData;
351: const XMLByte* srcEnd=srcData+srcLen;
352: XMLByte* outPtr=toFill;
353: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 354:
1.10 paf 355: // We now loop until we either run out of input data, or room to store
356: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
1.1 paf 357: // Get the next leading byte out
1.33.2.14 paf 358: const XMLByte firstByte =* srcPtr;
1.1 paf 359:
1.4 paf 360: // Special-case ASCII, which is a leading byte value of<= 127
361: if(firstByte<= 127) {
362: *outPtr++= firstByte;
1.1 paf 363: srcPtr++;
364: continue;
365: }
366:
367: // See how many trailing src bytes this sequence is going to require
368: const unsigned int trailingBytes = gUTFBytes[firstByte];
369:
370: // If there are not enough source bytes to do this one, then we
1.4 paf 371: // are done. Note that we done>= here because we are implicitly
1.1 paf 372: // counting the 1 byte we get no matter what.
1.4 paf 373: if(srcPtr+trailingBytes>= srcEnd)
1.1 paf 374: break;
375:
376: // Looks ok, so lets build up the value
377: uint tmpVal=0;
378: switch(trailingBytes) {
379: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
380: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
381: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
382: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
383: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
384: case 0: tmpVal+=*srcPtr++;
385: break;
386:
387: default:
1.23 paf 388: throw Exception(0,
1.33.2.19.2.2 (paf 389:: 0,
1.4 paf 390: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes);
1.1 paf 391: }
392: tmpVal-=gUTFOffsets[trailingBytes];
393:
394: // If it will fit into a single char, then put it in. Otherwise
395: // fail [*encode it as a surrogate pair. If its not valid, use the
396: // replacement char.*]
1.25 paf 397: if(!(tmpVal & 0xFFFF0000)) {
398: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
399: *outPtr++=xlat;
400: else
401: outPtr+=sprintf((char *)outPtr, "&#%d;", tmpVal); // &#decimal;
402: } else
1.23 paf 403: throw Exception(0,
1.33.2.19.2.2 (paf 404:: 0,
1.4 paf 405: "transcodeFromUTF8 error: too big tmpVal(0x%08X)", tmpVal);
1.1 paf 406: }
407:
1.11 paf 408: // Update the bytes eaten
409: srcLen = srcPtr - srcData;
410:
411: // Return the characters read
412: toFillLen = outPtr - toFill;
413:
1.29 paf 414: //return srcPtr==srcEnd?(int)toFillLen:-1;
415: /*
416: xmlCharEncodingOutputFunc
417: Returns :
418: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
419: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
420: of ocetes consumed.
421: */
422: return 0;
1.10 paf 423: }
424:
425: /// @todo not so memory-hungry with prescan
1.33.2.19.2.1 (paf 426:): const String::C Charset::transcodeToUTF8(const String::C src) const {
427:): size_t src_length=src.length;
428:): size_t dest_length=src.length*6/*so that surly enough, max utf8 seq len=6*/;
429:): XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 430:
431: if(::transcodeToUTF8(
1.33.2.19.2.1 (paf 432:): (XMLByte *)src.str, src_length,
433:): dest_body, dest_length,
1.11 paf 434: tables)<0)
1.10 paf 435: throw(0, 0,
436: 0,
1.11 paf 437: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 438:
1.33.2.19.2.1 (paf 439:): dest_body[dest_length]=0; // terminator
440:): return String::C((char*)dest_body, dest_length);
1.10 paf 441: }
1.33.2.19.2.1 (paf 442:): const String::C Charset::transcodeFromUTF8(const String::C src) const {
443:): size_t src_length=src.length;
444:): size_t dest_length=src.length*6/*so that surly enough, "ÿ" has max ratio */;
445:): XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 446:
447: if(::transcodeFromUTF8(
1.33.2.19.2.1 (paf 448:): (XMLByte *)src.str, src_length,
449:): dest_body, dest_length,
1.11 paf 450: tables)<0)
1.10 paf 451: throw(0, 0,
452: 0,
1.33.2.19.2.1 (paf 453:): "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 454:
1.33.2.19.2.1 (paf 455:): dest_body[dest_length]=0; // terminator
456:): return String::C((char*)dest_body, dest_length);
1.1 paf 457: }
458:
459: /// transcode using both charsets
1.33.2.19.2.1 (paf 460:): const String::C Charset::transcodeToCharset(const String::C src,
461:): const Charset& dest_charset) const {
462:): if(&dest_charset==this)
463:): return src;
464:): else {
465:): size_t dest_length=src.length;
466:): XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
467:):
468:): XMLByte* output=dest_body;
469:): const XMLByte* input=(XMLByte *)src.str;
470:): while(char c=*input++) {
471:): XMLCh curVal = tables.fromTable[c];
472:): *output++=curVal?
473:): xlatOneTo(curVal, dest_charset.tables, '?') // OK
474:): :'?'; // use the replacement character
1.6 paf 475: }
1.1 paf 476:
1.33.2.19.2.1 (paf 477:): dest_body[dest_length]=0; // terminator
478:): return String::C((char*)dest_body, dest_length);
1.6 paf 479: }
1.1 paf 480: }
481:
482: #ifdef XML
1.10 paf 483:
1.33.2.19.2.1 (paf 484:): static const Charset::Tables* tables[MAX_CHARSETS];
485:):
486:): #define declareXml256ioFuncs(i) \
487:): static int xml256CharEncodingInputFunc##i( \
488:): unsigned char *out, int *outlen, \
489:): const unsigned char *in, int *inlen) { \
490:): return transcodeToUTF8( \
491:): in, *(size_t*)inlen, \
492:): out, *(size_t*)outlen, \
493:): *tables[i]); \
494:): } \
495:): static int xml256CharEncodingOutputFunc##i( \
496:): unsigned char *out, int *outlen, \
497:): const unsigned char *in, int *inlen) { \
498:): return transcodeFromUTF8( \
499:): in, *(size_t*)inlen, \
500:): out, *(size_t*)outlen, \
501:): *tables[i]); \
502:): }
503:):
504:): declareXml256ioFuncs(0) declareXml256ioFuncs(1)
505:): declareXml256ioFuncs(2) declareXml256ioFuncs(3)
506:): declareXml256ioFuncs(4) declareXml256ioFuncs(5)
507:): declareXml256ioFuncs(6) declareXml256ioFuncs(7)
508:): declareXml256ioFuncs(8) declareXml256ioFuncs(9)
509:):
510:): static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
511:): xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
512:): xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
513:): xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
514:): xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
515:): xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
516:): };
517:): static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
518:): xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
519:): xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
520:): xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
521:): xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
522:): xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
523:): };
524:): static size_t handlers_count=0;
1.10 paf 525:
526: void Charset::addEncoding(char *name_cstr) {
1.33.2.19.2.1 (paf 527:): if(handlers_count==MAX_CHARSETS)
528:): throw Exception(0,
529:): 0,
530:): "already allocated %d handlers, no space for new encoding '%s'",
531:): MAX_CHARSETS, name_cstr);
532:):
1.33.2.19.2.9 (paf 533:: xmlCharEncodingHandler* handler=new(PointerFreeGC) xmlCharEncodingHandler;
1.33.2.19.2.1 (paf 534:): {
535:): handler->name=name_cstr;
536:): handler->input=inputFuncs[handlers_count];
537:): handler->output=outputFuncs[handlers_count];
538:): ::tables[handlers_count]=&tables;
539:): handlers_count++;
540:): }
1.10 paf 541:
542: xmlRegisterCharEncodingHandler(handler);
1.33.2.19.2.1 (paf 543:):
1.10 paf 544: }
545:
1.33.2.19.2.7 (paf 546:: void Charset::initTranscoder(const String& name, const char* name_cstr) {
1.15 paf 547: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.33.2.19.2.7 (paf 548:: transcoder(name); // check right way
1.15 paf 549: }
550:
1.33.2.19.2.7 (paf 551:: xmlCharEncodingHandler& Charset::transcoder(const String& name) {
1.15 paf 552: if(!ftranscoder)
1.23 paf 553: throw Exception("parser.runtime",
1.33.2.19.2.7 (paf 554:: &name,
1.10 paf 555: "unsupported encoding");
1.33.2.14 paf 556: return *ftranscoder;
1.10 paf 557: }
558:
1.33.2.19.2.7 (paf 559:: const char* Charset::transcode_cstr(xmlChar* s) {
1.13 paf 560: if(!s)
1.14 paf 561: return "";
1.8 paf 562:
1.33.2.6 paf 563: int inlen=strlen((const char* )s);
1.8 paf 564: int outlen=inlen+1; // max
1.33.2.19.2.6 (paf 565:: char *out=new(PointerFreeGC) char[outlen];
1.8 paf 566:
1.30 paf 567: int error;
1.33.2.19.2.7 (paf 568:: if(xmlCharEncodingOutputFunc output=transcoder(fname).output) {
1.30 paf 569: error=output(
1.17 paf 570: (unsigned char*)out, &outlen,
1.33.2.19.2.1 (paf 571:): (const unsigned char*)s, &inlen);
1.30 paf 572: } else {
573: memcpy(out, s, outlen=inlen);
574: error=0;
575: }
576: if(error<0)
1.33.2.19.2.2 (paf 577:: throw Exception(0,
578:: 0,
1.30 paf 579: "transcode_cstr failed (%d)", error);
1.8 paf 580:
1.30 paf 581: out[outlen/*surely would be less then on input*/]=0;
1.8 paf 582: return out;
1.14 paf 583: }
1.33.2.19.2.7 (paf 584:: const String& Charset::transcode(xmlChar* s) {
1.33.2.19.2.1 (paf 585:): return *new String(transcode_cstr(s), 0/*auto-size*/, true);
1.14 paf 586: }
1.33.2.19.2.7 (paf 587:: const char* Charset::transcode_cstr(GdomeDOMString* s) {
1.33.2.19.2.3 (paf 588:: return s?transcode_cstr(BAD_CAST s->str):"";
1.1 paf 589: }
1.33.2.19.2.7 (paf 590:: const String& Charset::transcode(GdomeDOMString* s) {
1.33.2.19.2.1 (paf 591:): return *new String(transcode_cstr(s), 0/*auto-size*/, true);
1.1 paf 592: }
593:
1.8 paf 594: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.33.2.16 paf 595: void* Charset::transcode_buf2mchar(transcode_buf_malloc_func malloc_func,
1.33.2.19.2.8 (paf 596:: const char* buf, size_t buf_size) {
1.33.2.14 paf 597: unsigned char* out;
1.30 paf 598: int outlen;
599: int error;
1.33.2.19.2.7 (paf 600:: if(xmlCharEncodingInputFunc input=transcoder(fname).input) {
1.32 paf 601: outlen=buf_size*6/*max*/;
1.33.2.16 paf 602: out=(unsigned char*)malloc_func(outlen+1);
1.30 paf 603: error=input(
1.17 paf 604: out, &outlen,
1.33.2.19.2.1 (paf 605:): (const unsigned char *)buf, (int *)&buf_size);
1.30 paf 606: } else {
607: outlen=buf_size;
1.33.2.16 paf 608: out=(unsigned char*)malloc_func(outlen+1);
1.30 paf 609: memcpy(out, buf, outlen);
610: error=0;
611: }
1.17 paf 612:
1.30 paf 613: if(error<0)
1.33.2.19.2.2 (paf 614:: throw Exception(0,
615:: 0,
1.30 paf 616: "transcode_buf failed (%d)", error);
1.8 paf 617:
1.30 paf 618: out[outlen/*surely would be less then on input*/]=0;
1.33.2.16 paf 619: return out;
620: }
621:
622: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
623: return static_cast<xmlChar*>(transcode_buf2mchar(xmlMalloc, buf, buf_size));
624: }
625: static void* g_malloc_wrapper(size_t size) {
626: return g_malloc(size);
627: }
628: gchar* Charset::transcode_buf2gchar(const char* buf, size_t buf_size) {
629: return static_cast<gchar*>(transcode_buf2mchar(g_malloc_wrapper, buf, buf_size));
1.24 paf 630: }
1.33.2.6 paf 631: GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char* buf, size_t buf_size) {
1.33.2.16 paf 632: return GdomeDOMString_auto_ptr(transcode_buf2gchar(buf, buf_size));
1.1 paf 633: }
1.33.2.19.2.1 (paf 634:: GdomeDOMString_auto_ptr Charset::transcode(const String& s) {
1.33.2.19.2.7 (paf 635:: const char* cstr=s.cstr(String::L_UNSPECIFIED);
1.1 paf 636:
1.24 paf 637: return transcode_buf2dom(cstr, strlen(cstr));
1.33.2.19.2.8 (paf 638:: }
639:: GdomeDOMString_auto_ptr Charset::transcode(const StringBody s) {
640:: const char* cstr=s.cstr();
641::
642:: return transcode_buf2dom(cstr, s.length());
1.1 paf 643: }
644: #endif
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main