Annotation of parser3/src/main/pa_charset.C, revision 1.99
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.90 moko 4: Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.96 moko 11: // we are using some pcre_internal.h stuff as well
12: #include "../lib/pcre/pa_pcre_internal.h"
13:
1.99 ! moko 14: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.98 2015/05/30 22:55:29 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 15:
1.1 paf 16: #ifdef XML
1.8 paf 17: #include "libxml/encoding.h"
1.1 paf 18: #endif
19:
1.46 paf 20: //#define PA_PATCHED_LIBXML_BACKWARD
1.67 misha 21:
22: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 23: #define PRECALCULATE_DEST_LENGTH
1.46 paf 24:
1.38 paf 25: // globals
26:
27: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
28: #include "utf8-to-upper.inc"
29: };
30: Charset::UTF8CaseTable UTF8CaseToUpper={
31: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
32: UTF8CaseToUpperRecords};
33:
34: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
35: #include "utf8-to-lower.inc"
36: };
37: Charset::UTF8CaseTable UTF8CaseToLower={
38: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
39: UTF8CaseToLowerRecords};
40:
1.1 paf 41: // helpers
42:
43: inline void prepare_case_tables(unsigned char *tables) {
44: unsigned char *lcc_table=tables+lcc_offset;
45: unsigned char *fcc_table=tables+fcc_offset;
46: for(int i=0; i<0x100; i++)
1.53 paf 47: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 48: }
1.99 ! moko 49: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr, unsigned char bit) {
1.1 paf 50: unsigned char *ctypes_table=tables+ctypes_offset;
51: ctypes_table[0]=bit;
52: for(; *cstr; cstr++) {
53: unsigned char c=*cstr;
54: ctypes_table[c]|=bit;
55: }
56: }
1.35 paf 57: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 58: if(!cstr || !*cstr)
59: return 0;
60: if(cstr[1]==0)
1.4 paf 61: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 62:
1.91 moko 63: return pa_atoui(cstr,0);
1.1 paf 64: }
1.35 paf 65: inline bool to_bool(const char* cstr) {
1.1 paf 66: return cstr && *cstr!=0;
67: }
1.99 ! moko 68: static void element2ctypes(unsigned char c, bool belongs, unsigned char *tables, unsigned char bit, int group_offset=-1) {
1.1 paf 69: if(!belongs)
70: return;
71:
72: unsigned char *ctypes_table=tables+ctypes_offset;
73:
74: ctypes_table[c]|=bit;
75: if(group_offset>=0)
1.4 paf 76: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 77: }
1.99 ! moko 78: static void element2case(unsigned char from, unsigned char to, unsigned char *tables) {
1.1 paf 79: if(!to)
80: return;
81:
82: unsigned char *lcc_table=tables+lcc_offset;
83: unsigned char *fcc_table=tables+fcc_offset;
84: lcc_table[from]=to;
85: fcc_table[from]=to; fcc_table[to]=from;
86: }
87:
1.95 moko 88: inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
1.93 moko 89: if(prefix) {
1.95 moko 90: strcpy((char *)dest, prefix);
1.93 moko 91: dest+=strlen(prefix);
92: }
93: *dest++=hex_digits[c >> 4];
94: *dest++=hex_digits[c & 0x0F];
1.95 moko 95: return dest;
1.93 moko 96: }
97:
1.95 moko 98: inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
1.93 moko 99: if(prefix) {
1.95 moko 100: strcpy((char *)dest, prefix);
1.93 moko 101: dest+=strlen(prefix);
102: }
103: *dest++=hex_digits[(c >> 12) & 0x0F];
104: *dest++=hex_digits[(c >> 8) & 0x0F];
105: *dest++=hex_digits[(c >> 4) & 0x0F];
106: *dest++=hex_digits[(c) & 0x0F];
1.95 moko 107: return dest;
1.93 moko 108: }
109:
1.1 paf 110: // methods
111:
1.37 paf 112: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 113: FNAME(ANAME),
114: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 115:
1.35 paf 116: if(afile_spec) {
1.1 paf 117: fisUTF8=false;
1.35 paf 118: load_definition(*charsets, *afile_spec);
1.1 paf 119: #ifdef XML
1.35 paf 120: addEncoding(FNAME_CSTR);
1.1 paf 121: #endif
122: } else {
123: fisUTF8=true;
1.4 paf 124: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.96 moko 125: memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
1.1 paf 126: }
127:
128: #ifdef XML
1.35 paf 129: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 130: #endif
131: }
132:
1.35 paf 133: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 134: // pcre_tables
135: // lowcase, flipcase, bits digit+word+whitespace, masks
136:
137: // must not move this inside of prepare_case_tables
138: // don't know the size there
139: memset(pcre_tables, 0, sizeof(pcre_tables));
140: prepare_case_tables(pcre_tables);
1.4 paf 141: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 142:
143: // charset
1.35 paf 144: memset(&tables, 0, sizeof(tables));
1.1 paf 145:
146: // loading text
1.35 paf 147: char *data=file_read_text(charsets, afile_spec);
1.1 paf 148:
149: // ignore header
150: getrow(&data);
151:
152: // parse cells
153: char *row;
1.42 paf 154: while((row=getrow(&data))) {
1.1 paf 155: // remove empty&comment lines
156: if(!*row || *row=='#')
157: continue;
158:
159: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 160: unsigned char c=0;
1.1 paf 161: char *cell;
1.42 paf 162: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 163: switch(column) {
1.53 paf 164: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 165: // pcre_tables
166: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
167: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
168: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
169: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
170: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 171: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 172: case 7:
173: case 8:
174: // charset
1.10 paf 175: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 176: throw Exception(PARSER_RUNTIME,
1.35 paf 177: &afile_spec,
1.1 paf 178: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
179:
180: XMLCh unicode=(XMLCh)to_wchar_code(cell);
181: if(!unicode && column==7/*unicode1 column*/)
182: unicode=(XMLCh)c;
183: if(unicode) {
1.10 paf 184: if(!tables.fromTable[c])
185: tables.fromTable[c]=unicode;
186: tables.toTable[tables.toTableSize].intCh=unicode;
187: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
188: tables.toTableSize++;
1.1 paf 189: }
190: break;
191: }
192: }
193: };
194:
1.87 moko 195: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
196: for(uint i=0; i<0x20; i++)
197: if(!tables.fromTable[i]){
198: tables.fromTable[i]=i;
199: tables.toTable[tables.toTableSize].intCh=i;
200: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
201: tables.toTableSize++;
202: }
203:
1.1 paf 204: // sort by the Unicode code point
205: sort_ToTable();
206: }
207:
208: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
209: return
1.38 paf 210: static_cast<const Charset::Tables::Rec *>(a)->intCh-
211: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 212: }
213:
214: void Charset::sort_ToTable() {
1.92 moko 215: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 216: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 217: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 218: //fclose(f);
219: }
220:
1.60 misha 221: // @todo: precache for spedup searching
1.99 ! moko 222: static XMLByte xlatOneTo(const XMLCh toXlat, const Charset::Tables& tables, XMLByte not_found) {
1.80 misha 223: int lo = 0;
224: int hi = tables.toTableSize - 1;
1.39 paf 225: while(lo<=hi) {
1.35 paf 226: // Calc the mid point of the low and high offset.
1.39 paf 227: const unsigned int i = (lo + hi) / 2;
228:
229: XMLCh cur=tables.toTable[i].intCh;
230: if(toXlat==cur)
231: return tables.toTable[i].extCh;
232: if(toXlat>cur)
233: lo = i+1;
1.1 paf 234: else
1.39 paf 235: hi = i-1;
236: }
1.35 paf 237:
238: return not_found;
1.1 paf 239: }
240:
1.99 ! moko 241: String::C Charset::transcode(const String::C src, const Charset& source_charset, const Charset& dest_charset) {
1.35 paf 242: if(!src.length)
243: return String::C("", 0);
1.4 paf 244:
1.1 paf 245: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
246: default: // 0x00
1.35 paf 247: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 248: case 0x01:
1.35 paf 249: return source_charset.transcodeToUTF8(src);
1.1 paf 250: case 0x10:
1.35 paf 251: return dest_charset.transcodeFromUTF8(src);
1.1 paf 252: case 0x11:
1.35 paf 253: return src;
1.1 paf 254: }
255: }
256:
257: // ---------------------------------------------------------------------------
258: // Local static data
259: //
260: // gUTFBytes
261: // A list of counts of trailing bytes for each initial byte in the input.
262: //
263: // gUTFOffsets
264: // A list of values to offset each result char type, according to how
265: // many source bytes when into making it.
266: //
267: // gFirstByteMark
268: // A list of values to mask onto the first byte of an encoded sequence,
269: // indexed by the number of bytes used to create the sequence.
270: // ---------------------------------------------------------------------------
271: static const XMLByte gUTFBytes[0x100] = {
272: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
273: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
274: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
275: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
276: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
277: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
278: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
279: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
280: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
281: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
282: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
283: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
284: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
285: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
286: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
287: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
288: };
289:
290: static const uint gUTFOffsets[6] = {
1.80 misha 291: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 292: };
293:
294: static const XMLByte gFirstByteMark[7] = {
1.80 misha 295: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 296: };
297:
1.99 ! moko 298: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen, XMLByte *toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 299: const XMLByte* srcPtr=srcData;
300: const XMLByte* srcEnd=srcData+srcLen;
301: XMLByte* outPtr=toFill;
302: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 303:
1.35 paf 304: while(srcPtr<srcEnd) {
305: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 306: if(!curVal) {
1.35 paf 307: // use the replacement character
308: *outPtr++= '?';
309: srcPtr++;
310: continue;
311: }
1.1 paf 312:
1.35 paf 313: // Figure out how many bytes we need
314: unsigned int encodedBytes;
315: if(curVal<0x80)
316: encodedBytes = 1;
317: else if(curVal<0x800)
318: encodedBytes = 2;
319: else if(curVal<0x10000)
320: encodedBytes = 3;
321: else if(curVal<0x200000)
322: encodedBytes = 4;
323: else if(curVal<0x4000000)
324: encodedBytes = 5;
325: else if(curVal<= 0x7FFFFFFF)
326: encodedBytes = 6;
327: else {
328: // use the replacement character
329: *outPtr++= '?';
330: srcPtr++;
331: continue;
332: }
1.11 paf 333:
1.35 paf 334: // If we cannot fully get this char into the output buffer
335: if (outPtr + encodedBytes > outEnd)
336: break;
337:
338: // We can do it, so update the source index
339: srcPtr++;
340:
341: // And spit out the bytes. We spit them out in reverse order
342: // here, so bump up the output pointer and work down as we go.
343: outPtr+= encodedBytes;
344: switch(encodedBytes) {
1.60 misha 345: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
346: curVal>>= 6;
347: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
348: curVal>>= 6;
349: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
350: curVal>>= 6;
351: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
352: curVal>>= 6;
353: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
354: curVal>>= 6;
355: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 356: }
357:
358: // Add the encoded bytes back in again to indicate we've eaten them
359: outPtr+= encodedBytes;
360: }
361:
362: // Update the bytes eaten
363: srcLen = srcPtr - srcData;
364:
365: // Return the characters read
366: toFillLen = outPtr - toFill;
367:
1.29 paf 368: //return srcPtr==srcEnd?(int)toFillLen:-1;
369: /*
370: xmlCharEncodingInputFunc
371: Returns :
372: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
373: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
374: of ocetes consumed.
375: */
376: return 0;
1.1 paf 377: }
1.26 paf 378: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.99 ! moko 379: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen, XMLByte* toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 380: const XMLByte* srcPtr=srcData;
381: const XMLByte* srcEnd=srcData+srcLen;
382: XMLByte* outPtr=toFill;
383: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 384:
1.35 paf 385: // We now loop until we either run out of input data, or room to store
386: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
387: // Get the next leading byte out
388: const XMLByte firstByte =* srcPtr;
389:
390: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 391: if(firstByte<=127) {
1.35 paf 392: *outPtr++= firstByte;
393: srcPtr++;
394: continue;
395: }
396:
397: // See how many trailing src bytes this sequence is going to require
398: const unsigned int trailingBytes = gUTFBytes[firstByte];
399:
400: // If there are not enough source bytes to do this one, then we
401: // are done. Note that we done>= here because we are implicitly
402: // counting the 1 byte we get no matter what.
403: if(srcPtr+trailingBytes>= srcEnd)
404: break;
405:
406: // Looks ok, so lets build up the value
407: uint tmpVal=0;
408: switch(trailingBytes) {
409: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
410: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
411: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
412: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
413: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
414: case 0: tmpVal+=*srcPtr++;
415: break;
416:
417: default:
418: throw Exception(0,
419: 0,
1.49 paf 420: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 421: }
422: tmpVal-=gUTFOffsets[trailingBytes];
423:
424: // If it will fit into a single char, then put it in. Otherwise
425: // fail [*encode it as a surrogate pair. If its not valid, use the
426: // replacement char.*]
427: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 428: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
429: *outPtr++=xlat;
1.49 paf 430: else {
1.50 paf 431: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 432: }
433: } else {
434: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
435: for(uint i=0; i<=trailingBytes; i++)
436: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
437: }
1.1 paf 438: }
1.35 paf 439:
440: // Update the bytes eaten
441: srcLen = srcPtr - srcData;
442:
443: // Return the characters read
444: toFillLen = outPtr - toFill;
1.11 paf 445:
1.29 paf 446: //return srcPtr==srcEnd?(int)toFillLen:-1;
447: /*
448: xmlCharEncodingOutputFunc
449: Returns :
450: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
451: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
452: of ocetes consumed.
453: */
454: return 0;
1.10 paf 455: }
456:
1.85 misha 457: static bool need_escape(XMLByte c){
1.60 misha 458: return
1.66 misha 459: !(
460: (c<=127)
461: && (
1.89 misha 462: pa_isalnum((unsigned char)c)
1.66 misha 463: || strchr("*@-_+./", c)!=0
464: )
465: );
1.60 misha 466: }
467:
1.70 misha 468: // read one UTF8 char and return length of this char (in bytes)
469: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 470: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
471: return 0;
472:
473: firstByte=*srcPtr;
474:
475: if(firstByte<=127){
476: UTF8Char=firstByte;
477: srcPtr++;
478: return 1;
479: }
480:
481: unsigned int trailingBytes=gUTFBytes[firstByte];
482:
483: if(srcPtr+trailingBytes>=srcEnd){
484: return 0; // not enough bytes in source string for reading
485: }
486:
487: uint tmpVal=0;
488: switch(trailingBytes){
489: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
490: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
491: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
492: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
493: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
494: case 0: tmpVal+=*srcPtr++;
495: }
496:
497: tmpVal-=gUTFOffsets[trailingBytes];
498: UTF8Char=tmpVal;
499:
500: return trailingBytes+1;
501: }
502:
1.70 misha 503: // skip UTF8 char and return length of this char (in bytes)
504: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 505: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
506: return 0;
507:
1.63 misha 508: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
509: srcPtr+=trailingBytes;
1.62 misha 510:
511: return trailingBytes;
1.61 misha 512: }
513:
1.85 misha 514: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 515: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 516: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
517: return 0;
518:
519: firstByte=*srcPtr++;
520: UTF8Char=tables.fromTable[firstByte];
521:
522: if(UTF8Char<0x80)
523: return 1;
524: else if(UTF8Char<0x800)
525: return 2;
526: else if(UTF8Char<0x10000)
527: return 3;
528: else if(UTF8Char<0x200000)
529: return 4;
530: else if(UTF8Char<0x4000000)
531: return 5;
532: else if(UTF8Char<= 0x7FFFFFFF)
533: return 6;
534:
535: // will use the replacement character '?'
536: firstByte=0;
537: return 1;
538: }
539:
1.85 misha 540: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
541: size_t dest_length=0;
542:
543: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
544: if(i.getCharSize()==1)
545: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
546: else
547: dest_length+=6; // %uXXXX
1.60 misha 548: }
549:
1.85 misha 550: return dest_length;
1.60 misha 551: }
552:
1.86 moko 553: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
554: const XMLByte* src_end=src+src_length;
555: XMLByte first_byte;
556: XMLCh UTF8_char;
1.85 misha 557: size_t dest_length=0;
558:
1.86 moko 559: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 560: if(char_size==1)
561: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
562: else
563: dest_length+=6; // %uXXXX
1.60 misha 564: }
565:
1.85 misha 566: return dest_length;
567: }
568:
569: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 570: if(!src.length)
1.85 misha 571: return 0;
572:
573: #ifdef PRECALCULATE_DEST_LENGTH
574: if(source_charset.isUTF8())
1.86 moko 575: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 576: else
1.86 moko 577: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 578: #else
579: return src_length*6; // enough for %uXXXX but too memory-hungry
580: #endif
581: }
582:
583: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
584: if(char_size==1) \
585: if(first_byte){ \
586: if(need_escape(first_byte)) \
1.95 moko 587: dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 588: else \
589: *dest_ptr++=first_byte; /*as is*/ \
590: } else \
591: *dest_ptr++='?'; /* replacement char '?' */ \
592: else \
1.95 moko 593: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 594:
595:
596: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
597: XMLByte* dest_ptr=dest;
598:
599: // loop until we either run out of input data
600: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
601: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 602:
1.85 misha 603: return dest_ptr - dest;
1.60 misha 604: }
605:
1.85 misha 606: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
607: const XMLByte* src_end=src+src_length;
608: XMLByte* dest_ptr=dest;
609:
610: XMLByte first_byte;
611: XMLCh UTF8_char;
612: uint char_size;
613:
1.86 moko 614: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 615: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
616:
617: return dest_ptr - dest;
618: }
1.60 misha 619:
620: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 621: if(!src.length)
1.60 misha 622: return String::C("", 0);
623:
1.85 misha 624: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
625: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
626:
627: size_t dest_length;
628: if(source_charset.isUTF8())
1.86 moko 629: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 630: else
1.86 moko 631: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 632:
633: if(dest_length>dest_calculated_length)
634: throw Exception(0, 0, "Charset::escape buffer overflow");
635:
636: dest_body[dest_length]=0; // terminator
637: return String::C((char*)dest_body, dest_length);
638: }
639:
640: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 641: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 642: return String::Body(dest.length ? dest.str:0);
643: }
644:
645: String& Charset::escape(const String& src, const Charset& source_charset) {
646: if(src.is_empty())
647: return *new String();
648:
649: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
650: }
651:
652: inline bool need_json_escape(unsigned char c){
653: return strchr("\n\"\\/\t\r\b\f", c)!=0;
654: }
655:
656: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
657: size_t dest_length=0;
658:
659: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 moko 660: if(i.getCharSize()==1){
661: XMLByte first_byte=i.getFirstByte();
662: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
663: } else
1.85 misha 664: dest_length+=6; // \uXXXX
665: }
666:
667: return dest_length;
668: }
669:
1.86 moko 670: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
671: const XMLByte* src_end=src+src_length;
1.85 misha 672: XMLByte first_byte;
673: XMLCh UTF8_char;
1.60 misha 674: size_t dest_length=0;
675:
1.86 moko 676: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 677: if(char_size==1)
1.93 moko 678: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 679: else
680: dest_length+=6; // \uXXXX
1.60 misha 681: }
1.85 misha 682:
683: return dest_length;
684: }
685:
686: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 687: if(!src.length)
1.85 misha 688: return 0;
689:
690: #ifdef PRECALCULATE_DEST_LENGTH
691: if(source_charset.isUTF8())
1.86 moko 692: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 693: else
1.86 moko 694: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 695: #else
1.85 misha 696: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 697: #endif
1.85 misha 698: }
699:
700: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
701: if(char_size==1) \
702: switch(first_byte){ \
703: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
704: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
705: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
706: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
707: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
708: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
709: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
710: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
711: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.95 moko 712: default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
1.93 moko 713: else *dest_ptr++=first_byte; \
1.85 misha 714: } \
715: else \
1.95 moko 716: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 717:
718:
719: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
720: XMLByte* dest_ptr=dest;
721:
722: // loop until we either run out of input data
723: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
724: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
725:
726: return dest_ptr - dest;
727: }
728:
729: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
730: const XMLByte* src_end=src+src_length;
731: XMLByte* dest_ptr=dest;
732:
733: XMLByte first_byte;
734: XMLCh UTF8_char;
735: uint char_size;
736:
1.86 moko 737: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 738: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
739:
740: return dest_ptr - dest;
741: }
1.60 misha 742:
1.85 misha 743: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 744: if(!src.length)
1.85 misha 745: return String::C("", 0);
1.60 misha 746:
1.85 misha 747: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
748: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
749:
750: size_t dest_length;
751: if(source_charset.isUTF8())
1.86 moko 752: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 753: else
1.86 moko 754: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 755:
1.85 misha 756: if(dest_length>dest_calculated_length)
757: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 758:
759: dest_body[dest_length]=0; // terminator
760: return String::C((char*)dest_body, dest_length);
761: }
1.85 misha 762:
763: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 764: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 765: return String::Body(dest.length ? dest.str:0);
1.64 misha 766: }
767:
1.85 misha 768: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 769: if(src.is_empty())
1.73 misha 770: return *new String();
1.64 misha 771:
1.85 misha 772: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 773: }
1.60 misha 774:
1.35 paf 775: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 776: int src_length=src.length;
1.60 misha 777:
778: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 779: int dest_length=0;
1.60 misha 780: const XMLByte* srcPtr=(XMLByte*)src.str;
781: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 782: XMLByte firstByte;
783: XMLCh UTF8Char;
784: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 785: dest_length+=charSize;
786: #else
1.85 misha 787: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 788: #endif
789:
1.35 paf 790: #ifndef NDEBUG
1.71 misha 791: int saved_dest_length=dest_length;
1.35 paf 792: #endif
793: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 794:
795: if(::transcodeToUTF8(
1.35 paf 796: (XMLByte *)src.str, src_length,
797: dest_body, dest_length,
1.11 paf 798: tables)<0)
1.43 paf 799: throw Exception(0,
1.10 paf 800: 0,
1.11 paf 801: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 802:
1.60 misha 803: assert(dest_length<=saved_dest_length);
804: dest_body[dest_length]=0; // terminator
1.35 paf 805: return String::C((char*)dest_body, dest_length);
1.10 paf 806: }
1.38 paf 807:
808: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 809: int lo = 0;
810: int hi = table.size - 1;
1.39 paf 811: while(lo<=hi) {
1.38 paf 812: // Calc the mid point of the low and high offset.
1.39 paf 813: const unsigned int i = (lo + hi) / 2;
814:
815: XMLCh cur=table.records[i].from;
816: if(src==cur)
817: return table.records[i].to;
818: if(src>cur)
819: lo = i+1;
1.38 paf 820: else
1.39 paf 821: hi = i-1;
822: }
823:
824: // not found
1.38 paf 825: return src;
826: }
827:
1.58 misha 828: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 829: if(!src) {
830: // use the replacement character
831: *outPtr++= '?';
832: return;
833: }
834:
835: // Figure out how many bytes we need
836: unsigned int encodedBytes;
837: if(src<0x80)
838: encodedBytes = 1;
839: else if(src<0x800)
840: encodedBytes = 2;
841: else if(src<0x10000)
842: encodedBytes = 3;
843: else if(src<0x200000)
844: encodedBytes = 4;
845: else if(src<0x4000000)
846: encodedBytes = 5;
847: else if(src<= 0x7FFFFFFF)
848: encodedBytes = 6;
849: else {
850: // use the replacement character
851: *outPtr++= '?';
852: return;
853: }
854:
855: // And spit out the bytes. We spit them out in reverse order
856: // here, so bump up the output pointer and work down as we go.
857: outPtr+= encodedBytes;
858: switch(encodedBytes) {
859: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
860: src>>= 6;
861: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
862: src>>= 6;
863: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
864: src>>= 6;
865: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
866: src>>= 6;
867: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
868: src>>= 6;
869: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
870: }
871:
872: // Add the encoded bytes back in again to indicate we've eaten them
873: outPtr+= encodedBytes;
874: }
875:
1.99 ! moko 876: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr, const Charset::UTF8CaseTable& table) {
1.38 paf 877: store_UTF8(change_case_UTF8(src, table), outPtr);
1.98 moko 878: }
879:
1.99 ! moko 880: void change_case_UTF8(const XMLByte* srcData, size_t srcLen, XMLByte* toFill, size_t toFillLen, const Charset::UTF8CaseTable& table) {
1.38 paf 881: const XMLByte* srcPtr=srcData;
1.44 paf 882: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 883: XMLByte* outPtr=toFill;
1.44 paf 884: XMLByte* outEnd=toFill+toFillLen;
885:
886: // We now loop until we either run out of input data, or room to store
887: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
888: // Get the next leading byte out
889: const XMLByte firstByte =* srcPtr;
1.38 paf 890:
1.60 misha 891: if(firstByte<=127) {
1.38 paf 892: change_case_UTF8(firstByte, outPtr, table);
893: srcPtr++;
894: continue;
895: }
896:
897: // See how many trailing src bytes this sequence is going to require
898: const unsigned int trailingBytes = gUTFBytes[firstByte];
899:
900: // Looks ok, so lets build up the value
901: uint tmpVal=0;
902: switch(trailingBytes) {
903: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
904: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
905: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
906: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
907: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
908: case 0: tmpVal+=*srcPtr++;
909: break;
910:
911: default:
912: throw Exception(0,
913: 0,
914: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
915: }
916: tmpVal-=gUTFOffsets[trailingBytes];
917:
918: // If it will fit into a single char, then put it in. Otherwise
919: // fail [*encode it as a surrogate pair. If its not valid, use the
920: // replacement char.*]
921: if(!(tmpVal & 0xFFFF0000))
922: change_case_UTF8(tmpVal, outPtr, table);
923: else
924: throw Exception(0,
925: 0,
926: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
927: }
928:
929: if(srcPtr!=outPtr)
930: throw Exception(0,
931: 0,
932: "change_case_UTF8 error: end pointers do not match");
933: }
934:
1.60 misha 935: static size_t getDecNumLength(XMLCh UTF8Char){
936: return
937: (UTF8Char < 100)
938: ?2
939: :(UTF8Char < 1000)
940: ?3
941: :(UTF8Char < 10000)
942: ?4
943: :5;
944: }
1.38 paf 945:
1.35 paf 946: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 947: int src_length=src.length;
1.60 misha 948: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 949: int dest_length=0;
1.82 misha 950: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 951: dest_length += ( i.next() & 0xFFFF0000 )
952: ? 3*i.getCharSize() // %XX for each byte
953: : ( xlatOneTo(i.next(), tables, 0) != 0 )
954: ? 1 // can convert it to a single char
955: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 956: }
957: #else
958: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 959: int dest_length=src_length*6;
1.60 misha 960: #endif
961:
1.35 paf 962: #ifndef NDEBUG
1.71 misha 963: int saved_dest_length=dest_length;
1.35 paf 964: #endif
965: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 966:
967: if(::transcodeFromUTF8(
1.82 misha 968: (XMLByte *)src.str, src_length,
1.35 paf 969: dest_body, dest_length,
1.11 paf 970: tables)<0)
1.43 paf 971: throw Exception(0,
1.10 paf 972: 0,
1.35 paf 973: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 974:
1.60 misha 975: assert(dest_length<=saved_dest_length);
976: dest_body[dest_length]=0; // terminator
1.35 paf 977: return String::C((char*)dest_body, dest_length);
1.1 paf 978: }
979:
980: /// transcode using both charsets
1.99 ! moko 981: const String::C Charset::transcodeToCharset(const String::C src, const Charset& dest_charset) const {
1.35 paf 982: if(&dest_charset==this)
983: return src;
984: else {
985: size_t dest_length=src.length;
986: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
987:
988: XMLByte* output=dest_body;
989: const XMLByte* input=(XMLByte *)src.str;
990: while(XMLCh c=*input++) {
991: XMLCh curVal = tables.fromTable[c];
992: *output++=curVal?
993: xlatOneTo(curVal, dest_charset.tables, '?') // OK
994: :'?'; // use the replacement character
1.6 paf 995: }
1.1 paf 996:
1.35 paf 997: dest_body[dest_length]=0; // terminator
998: return String::C((char*)dest_body, dest_length);
1.6 paf 999: }
1.1 paf 1000: }
1001:
1.58 misha 1002: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 1003: if(isUTF8())
1.58 misha 1004: store_UTF8(src, outPtr);
1.59 misha 1005: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 1006: *outPtr++=ch;
1.57 misha 1007: }
1008:
1.1 paf 1009: #ifdef XML
1.10 paf 1010:
1.35 paf 1011: static const Charset::Tables* tables[MAX_CHARSETS];
1012:
1.46 paf 1013: #ifdef PA_PATCHED_LIBXML_BACKWARD
1014:
1015: #define declareXml256ioFuncs(i) \
1016: static int xml256CharEncodingInputFunc##i( \
1017: unsigned char *out, int *outlen, \
1018: const unsigned char *in, int *inlen, void*) { \
1019: return transcodeToUTF8( \
1.71 misha 1020: in, *inlen, \
1021: out, *outlen, \
1.46 paf 1022: *tables[i]); \
1023: } \
1024: static int xml256CharEncodingOutputFunc##i( \
1025: unsigned char *out, int *outlen, \
1026: const unsigned char *in, int *inlen, void*) { \
1027: return transcodeFromUTF8( \
1.71 misha 1028: in, *inlen, \
1029: out, *outlen, \
1.46 paf 1030: *tables[i]); \
1031: }
1032:
1033: #else
1034:
1.35 paf 1035: #define declareXml256ioFuncs(i) \
1036: static int xml256CharEncodingInputFunc##i( \
1037: unsigned char *out, int *outlen, \
1038: const unsigned char *in, int *inlen) { \
1039: return transcodeToUTF8( \
1.71 misha 1040: in, *inlen, \
1041: out, *outlen, \
1.35 paf 1042: *tables[i]); \
1043: } \
1044: static int xml256CharEncodingOutputFunc##i( \
1045: unsigned char *out, int *outlen, \
1046: const unsigned char *in, int *inlen) { \
1047: return transcodeFromUTF8( \
1.71 misha 1048: in, *inlen, \
1049: out, *outlen, \
1.35 paf 1050: *tables[i]); \
1051: }
1052:
1.46 paf 1053: #endif
1054:
1055:
1.35 paf 1056: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1057: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1058: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1059: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1060: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1061:
1062: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1063: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1064: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1065: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1066: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1067: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1068: };
1069: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1070: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1071: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1072: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1073: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1074: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1075: };
1076: static size_t handlers_count=0;
1.10 paf 1077:
1078: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1079: if(handlers_count==MAX_CHARSETS)
1080: throw Exception(0,
1081: 0,
1082: "already allocated %d handlers, no space for new encoding '%s'",
1083: MAX_CHARSETS, name_cstr);
1084:
1.97 moko 1085: xmlCharEncodingHandler* handler=new xmlCharEncodingHandler;
1.35 paf 1086: {
1087: handler->name=name_cstr;
1088: handler->input=inputFuncs[handlers_count];
1089: handler->output=outputFuncs[handlers_count];
1090: ::tables[handlers_count]=&tables;
1091: handlers_count++;
1092: }
1.10 paf 1093:
1094: xmlRegisterCharEncodingHandler(handler);
1.35 paf 1095:
1.10 paf 1096: }
1097:
1.37 paf 1098: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 1099: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 1100: transcoder(NAME); // check right way
1.15 paf 1101: }
1102:
1.37 paf 1103: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 1104: if(!ftranscoder)
1.56 misha 1105: throw Exception(PARSER_RUNTIME,
1.35 paf 1106: new String(NAME, String::L_TAINTED),
1.10 paf 1107: "unsupported encoding");
1.35 paf 1108: return *ftranscoder;
1.10 paf 1109: }
1110:
1.54 paf 1111: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1112: if(!s)
1.35 paf 1113: return String::C("", 0);
1.8 paf 1114:
1.35 paf 1115: int inlen=strlen((const char*)s);
1.51 paf 1116: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1117: #ifndef NDEBUG
1118: int saved_outlen=outlen;
1119: #endif
1120: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1121:
1.30 paf 1122: int error;
1.35 paf 1123: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1124: error=output(
1.17 paf 1125: (unsigned char*)out, &outlen,
1.46 paf 1126: (const unsigned char*)s, &inlen
1127: #ifdef PA_PATCHED_LIBXML_BACKWARD
1128: ,0
1129: #endif
1130: );
1.30 paf 1131: } else {
1132: memcpy(out, s, outlen=inlen);
1133: error=0;
1134: }
1135: if(error<0)
1.23 paf 1136: throw Exception(0,
1.8 paf 1137: 0,
1.30 paf 1138: "transcode_cstr failed (%d)", error);
1.8 paf 1139:
1.35 paf 1140: assert(outlen<=saved_outlen); out[outlen]=0;
1141: return String::C(out, outlen);
1.14 paf 1142: }
1.54 paf 1143: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1144: String::C cstr=transcode_cstr(s);
1.75 misha 1145: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1146: }
1147:
1.8 paf 1148: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1149: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1150: xmlChar* out;
1.30 paf 1151: int outlen;
1152: int error;
1.35 paf 1153: #ifndef NDEBUG
1154: int saved_outlen;
1155: #endif
1156: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1157: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1158: #ifndef NDEBUG
1159: saved_outlen=outlen;
1160: #endif
1.47 paf 1161: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1162: error=input(
1.17 paf 1163: out, &outlen,
1.46 paf 1164: (const unsigned char*)buf, (int*)&buf_size
1165: #ifdef PA_PATCHED_LIBXML_BACKWARD
1166: ,0
1167: #endif
1168: );
1.30 paf 1169: } else {
1170: outlen=buf_size;
1.35 paf 1171: #ifndef NDEBUG
1172: saved_outlen=outlen;
1173: #endif
1174: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1175: memcpy(out, buf, outlen);
1176: error=0;
1177: }
1.17 paf 1178:
1.30 paf 1179: if(error<0)
1.23 paf 1180: throw Exception(0,
1.8 paf 1181: 0,
1.30 paf 1182: "transcode_buf failed (%d)", error);
1.8 paf 1183:
1.35 paf 1184: assert(outlen<=saved_outlen); out[outlen]=0;
1185: return out;
1.24 paf 1186: }
1.1 paf 1187:
1.79 misha 1188: xmlChar* Charset::transcode(const String& s) {
1189: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1190: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1191: }
1.35 paf 1192:
1.79 misha 1193: xmlChar* Charset::transcode(const String::Body s) {
1194: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1195: }
1.36 paf 1196: #endif
1.34 paf 1197:
1.99 ! moko 1198: String::Body Charset::transcode(const String::Body src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.34 paf 1199:
1.35 paf 1200: const char *src_ptr=src.cstr();
1.83 misha 1201: size_t src_size=src.length();
1.34 paf 1202:
1.77 misha 1203: String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);
1.34 paf 1204:
1.77 misha 1205: return String::Body(dest.length ? dest.str:0);
1.35 paf 1206: }
1207:
1.99 ! moko 1208: String& Charset::transcode(const String& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.72 misha 1209: if(src.is_empty())
1.73 misha 1210: return *new String();
1.34 paf 1211:
1.37 paf 1212: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1213: }
1214:
1.99 ! moko 1215: void Charset::transcode(ArrayString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1216: for(size_t i=0; i<src.count(); i++)
1217: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1218: }
1219:
1220: #ifndef DOXYGEN
1221: struct Transcode_pair_info {
1222: const Charset* source_transcoder;
1223: const Charset* dest_transcoder;
1224: };
1225: #endif
1.99 ! moko 1226: static void transcode_pair(HashStringValue::key_type /*akey*/, String::Body& avalue, Transcode_pair_info* info) {
! 1227: avalue=Charset::transcode(avalue, *info->source_transcoder, *info->dest_transcoder);
1.34 paf 1228: }
1.61 misha 1229:
1.99 ! moko 1230: void Charset::transcode(HashStringString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1231: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1232: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1233: }
1.61 misha 1234:
1235: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1236: const XMLByte* ptr=srcBegin;
1.70 misha 1237: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1238:
1239: return ptr-srcBegin;
1240: }
1241:
1242: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1243: size_t charPos=0;
1244: const XMLByte* ptr=srcBegin;
1245: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1246: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1247: if(ptr>ptrEnd)
1248: return charPos;
1249: charPos++;
1250: }
1251:
1252: // scan till end but position in bytes still too low
1253: throw Exception(0,
1254: 0,
1255: "Error convertion byte pos to char pos");
1256: }
1257:
1258: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1259: size_t size=0;
1.70 misha 1260: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1261: size++;
1262:
1263: return size;
1264: }
1.80 misha 1265:
1.84 misha 1266: unsigned int lengthUTF8Char(const XMLByte c){
1267: return gUTFBytes[c]+1;
1268: }
1269:
1.94 moko 1270: const char *fixUTF8(const char *src){
1271: if(src && *src){
1272: size_t length=strlen(src);
1273:
1274: int error_offset;
1.96 moko 1275: if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
1.94 moko 1276:
1277: char *result=(char *)pa_malloc_atomic(length+1);
1278: char *dst=result;
1279:
1280: do {
1281:
1282: if(error_offset){
1283: strncpy(dst, src, error_offset);
1284: dst+=error_offset;
1285:
1286: src+=error_offset;
1287: length-=error_offset;
1288:
1289: }
1290:
1291: *dst++='?';
1292: src++;
1293: length--;
1294:
1.96 moko 1295: } while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));
1.94 moko 1296:
1297: if(length){
1298: strcpy(dst, src);
1299: } else {
1300: *dst='\0';
1301: }
1302:
1303: return result;
1304: }
1305: }
1306: return src;
1307: }
1308:
1.80 misha 1309: bool UTF8_string_iterator::has_next(){
1310: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1311: return fcharSize!=0;
1312: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main