Annotation of parser3/src/main/pa_charset.C, revision 1.119
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.118 moko 4: Copyright (c) 2001-2026 Art. Lebedev Studio (https://www.artlebedev.com)
1.112 moko 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.119 ! moko 10: #include "pa_file.h"
1.1 paf 11:
1.96 moko 12: // we are using some pcre_internal.h stuff as well
13: #include "../lib/pcre/pa_pcre_internal.h"
14:
1.119 ! moko 15: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.118 2026/04/25 13:38:46 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 16:
1.1 paf 17: #ifdef XML
1.115 moko 18: #include "libxml/xmlmemory.h"
1.8 paf 19: #include "libxml/encoding.h"
1.1 paf 20: #endif
21:
1.67 misha 22: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 23: #define PRECALCULATE_DEST_LENGTH
1.46 paf 24:
1.38 paf 25: // globals
26:
27: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
28: #include "utf8-to-upper.inc"
29: };
30: Charset::UTF8CaseTable UTF8CaseToUpper={
31: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
32: UTF8CaseToUpperRecords};
33:
34: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
35: #include "utf8-to-lower.inc"
36: };
37: Charset::UTF8CaseTable UTF8CaseToLower={
38: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
39: UTF8CaseToLowerRecords};
40:
1.1 paf 41: // helpers
42:
43: inline void prepare_case_tables(unsigned char *tables) {
44: unsigned char *lcc_table=tables+lcc_offset;
45: unsigned char *fcc_table=tables+fcc_offset;
46: for(int i=0; i<0x100; i++)
1.53 paf 47: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 48: }
1.99 moko 49: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr, unsigned char bit) {
1.1 paf 50: unsigned char *ctypes_table=tables+ctypes_offset;
51: ctypes_table[0]=bit;
52: for(; *cstr; cstr++) {
53: unsigned char c=*cstr;
54: ctypes_table[c]|=bit;
55: }
56: }
1.35 paf 57: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 58: if(!cstr || !*cstr)
59: return 0;
60: if(cstr[1]==0)
1.4 paf 61: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 62:
1.91 moko 63: return pa_atoui(cstr,0);
1.1 paf 64: }
1.35 paf 65: inline bool to_bool(const char* cstr) {
1.1 paf 66: return cstr && *cstr!=0;
67: }
1.99 moko 68: static void element2ctypes(unsigned char c, bool belongs, unsigned char *tables, unsigned char bit, int group_offset=-1) {
1.1 paf 69: if(!belongs)
70: return;
71:
72: unsigned char *ctypes_table=tables+ctypes_offset;
73:
74: ctypes_table[c]|=bit;
75: if(group_offset>=0)
1.4 paf 76: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 77: }
1.99 moko 78: static void element2case(unsigned char from, unsigned char to, unsigned char *tables) {
1.1 paf 79: if(!to)
80: return;
81:
82: unsigned char *lcc_table=tables+lcc_offset;
83: unsigned char *fcc_table=tables+fcc_offset;
84: lcc_table[from]=to;
85: fcc_table[from]=to; fcc_table[to]=from;
86: }
87:
1.95 moko 88: inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
1.93 moko 89: if(prefix) {
1.95 moko 90: strcpy((char *)dest, prefix);
1.93 moko 91: dest+=strlen(prefix);
92: }
93: *dest++=hex_digits[c >> 4];
94: *dest++=hex_digits[c & 0x0F];
1.95 moko 95: return dest;
1.93 moko 96: }
97:
1.95 moko 98: inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
1.93 moko 99: if(prefix) {
1.95 moko 100: strcpy((char *)dest, prefix);
1.93 moko 101: dest+=strlen(prefix);
102: }
103: *dest++=hex_digits[(c >> 12) & 0x0F];
104: *dest++=hex_digits[(c >> 8) & 0x0F];
105: *dest++=hex_digits[(c >> 4) & 0x0F];
106: *dest++=hex_digits[(c) & 0x0F];
1.95 moko 107: return dest;
1.93 moko 108: }
109:
1.1 paf 110: // methods
111:
1.103 moko 112: Charset::Charset(Request_charsets* acharsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 113: FNAME(ANAME),
114: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 115:
1.35 paf 116: if(afile_spec) {
1.1 paf 117: fisUTF8=false;
1.103 moko 118: load_definition(*acharsets, *afile_spec);
1.1 paf 119: #ifdef XML
1.35 paf 120: addEncoding(FNAME_CSTR);
1.1 paf 121: #endif
122: } else {
123: fisUTF8=true;
1.4 paf 124: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.96 moko 125: memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
1.1 paf 126: }
127: }
128:
1.104 moko 129: void Charset::load_definition(Request_charsets& acharsets, const String& afile_spec) {
1.1 paf 130: // pcre_tables
131: // lowcase, flipcase, bits digit+word+whitespace, masks
132:
133: // must not move this inside of prepare_case_tables
134: // don't know the size there
135: memset(pcre_tables, 0, sizeof(pcre_tables));
136: prepare_case_tables(pcre_tables);
1.4 paf 137: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 138:
139: // charset
1.35 paf 140: memset(&tables, 0, sizeof(tables));
1.1 paf 141:
142: // loading text
1.104 moko 143: char *data=file_read_text(acharsets, afile_spec);
1.1 paf 144:
145: // ignore header
146: getrow(&data);
147:
148: // parse cells
149: char *row;
1.42 paf 150: while((row=getrow(&data))) {
1.1 paf 151: // remove empty&comment lines
152: if(!*row || *row=='#')
153: continue;
154:
155: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 156: unsigned char c=0;
1.1 paf 157: char *cell;
1.42 paf 158: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 159: switch(column) {
1.53 paf 160: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 161: // pcre_tables
162: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
163: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
164: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
165: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
166: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 167: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 168: case 7:
169: case 8:
170: // charset
1.10 paf 171: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.113 moko 172: throw Exception(PARSER_RUNTIME, &afile_spec, "charset must contain not more than %d unicode values", MAX_CHARSET_UNI_CODES);
1.1 paf 173:
174: XMLCh unicode=(XMLCh)to_wchar_code(cell);
175: if(!unicode && column==7/*unicode1 column*/)
176: unicode=(XMLCh)c;
177: if(unicode) {
1.10 paf 178: if(!tables.fromTable[c])
179: tables.fromTable[c]=unicode;
180: tables.toTable[tables.toTableSize].intCh=unicode;
181: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
182: tables.toTableSize++;
1.1 paf 183: }
184: break;
185: }
186: }
187: };
188:
1.87 moko 189: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
190: for(uint i=0; i<0x20; i++)
191: if(!tables.fromTable[i]){
192: tables.fromTable[i]=i;
193: tables.toTable[tables.toTableSize].intCh=i;
194: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
195: tables.toTableSize++;
196: }
197:
1.1 paf 198: // sort by the Unicode code point
199: sort_ToTable();
200: }
201:
202: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
203: return
1.38 paf 204: static_cast<const Charset::Tables::Rec *>(a)->intCh-
205: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 206: }
207:
208: void Charset::sort_ToTable() {
1.92 moko 209: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 210: }
211:
1.60 misha 212: // @todo: precache for spedup searching
1.99 moko 213: static XMLByte xlatOneTo(const XMLCh toXlat, const Charset::Tables& tables, XMLByte not_found) {
1.80 misha 214: int lo = 0;
215: int hi = tables.toTableSize - 1;
1.39 paf 216: while(lo<=hi) {
1.35 paf 217: // Calc the mid point of the low and high offset.
1.39 paf 218: const unsigned int i = (lo + hi) / 2;
219:
220: XMLCh cur=tables.toTable[i].intCh;
221: if(toXlat==cur)
222: return tables.toTable[i].extCh;
223: if(toXlat>cur)
224: lo = i+1;
1.1 paf 225: else
1.39 paf 226: hi = i-1;
227: }
1.35 paf 228:
229: return not_found;
1.1 paf 230: }
231:
1.99 moko 232: String::C Charset::transcode(const String::C src, const Charset& source_charset, const Charset& dest_charset) {
1.35 paf 233: if(!src.length)
234: return String::C("", 0);
1.4 paf 235:
1.1 paf 236: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
237: default: // 0x00
1.35 paf 238: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 239: case 0x01:
1.35 paf 240: return source_charset.transcodeToUTF8(src);
1.1 paf 241: case 0x10:
1.35 paf 242: return dest_charset.transcodeFromUTF8(src);
1.1 paf 243: case 0x11:
1.35 paf 244: return src;
1.1 paf 245: }
246: }
247:
248: // ---------------------------------------------------------------------------
249: // Local static data
250: //
251: // gUTFBytes
252: // A list of counts of trailing bytes for each initial byte in the input.
253: //
254: // gUTFOffsets
255: // A list of values to offset each result char type, according to how
256: // many source bytes when into making it.
257: //
258: // gFirstByteMark
259: // A list of values to mask onto the first byte of an encoded sequence,
260: // indexed by the number of bytes used to create the sequence.
261: // ---------------------------------------------------------------------------
262: static const XMLByte gUTFBytes[0x100] = {
263: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
264: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
265: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
266: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
267: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
268: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
269: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
270: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
271: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
272: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
273: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
274: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
275: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
276: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
277: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
278: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
279: };
280:
281: static const uint gUTFOffsets[6] = {
1.80 misha 282: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 283: };
284:
285: static const XMLByte gFirstByteMark[7] = {
1.80 misha 286: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 287: };
288:
1.99 moko 289: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen, XMLByte *toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 290: const XMLByte* srcPtr=srcData;
291: const XMLByte* srcEnd=srcData+srcLen;
292: XMLByte* outPtr=toFill;
293: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 294:
1.35 paf 295: while(srcPtr<srcEnd) {
296: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 297: if(!curVal) {
1.35 paf 298: // use the replacement character
299: *outPtr++= '?';
300: srcPtr++;
301: continue;
302: }
1.1 paf 303:
1.35 paf 304: // Figure out how many bytes we need
305: unsigned int encodedBytes;
306: if(curVal<0x80)
307: encodedBytes = 1;
308: else if(curVal<0x800)
309: encodedBytes = 2;
310: else if(curVal<0x10000)
311: encodedBytes = 3;
312: else if(curVal<0x200000)
313: encodedBytes = 4;
314: else if(curVal<0x4000000)
315: encodedBytes = 5;
316: else if(curVal<= 0x7FFFFFFF)
317: encodedBytes = 6;
318: else {
319: // use the replacement character
320: *outPtr++= '?';
321: srcPtr++;
322: continue;
323: }
1.11 paf 324:
1.35 paf 325: // If we cannot fully get this char into the output buffer
326: if (outPtr + encodedBytes > outEnd)
327: break;
328:
329: // We can do it, so update the source index
330: srcPtr++;
331:
332: // And spit out the bytes. We spit them out in reverse order
333: // here, so bump up the output pointer and work down as we go.
334: outPtr+= encodedBytes;
335: switch(encodedBytes) {
1.60 misha 336: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
337: curVal>>= 6;
338: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
339: curVal>>= 6;
340: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
341: curVal>>= 6;
342: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
343: curVal>>= 6;
344: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
345: curVal>>= 6;
346: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 347: }
348:
349: // Add the encoded bytes back in again to indicate we've eaten them
350: outPtr+= encodedBytes;
351: }
352:
353: // Update the bytes eaten
354: srcLen = srcPtr - srcData;
355:
356: // Return the characters read
357: toFillLen = outPtr - toFill;
358:
1.29 paf 359: //return srcPtr==srcEnd?(int)toFillLen:-1;
360: /*
361: xmlCharEncodingInputFunc
362: Returns :
363: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
364: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
365: of ocetes consumed.
366: */
367: return 0;
1.1 paf 368: }
1.26 paf 369: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.99 moko 370: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen, XMLByte* toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 371: const XMLByte* srcPtr=srcData;
372: const XMLByte* srcEnd=srcData+srcLen;
373: XMLByte* outPtr=toFill;
374: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 375:
1.35 paf 376: // We now loop until we either run out of input data, or room to store
377: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
378: // Get the next leading byte out
379: const XMLByte firstByte =* srcPtr;
380:
381: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 382: if(firstByte<=127) {
1.35 paf 383: *outPtr++= firstByte;
384: srcPtr++;
385: continue;
386: }
387:
388: // See how many trailing src bytes this sequence is going to require
389: const unsigned int trailingBytes = gUTFBytes[firstByte];
390:
391: // If there are not enough source bytes to do this one, then we
392: // are done. Note that we done>= here because we are implicitly
393: // counting the 1 byte we get no matter what.
394: if(srcPtr+trailingBytes>= srcEnd)
395: break;
396:
397: // Looks ok, so lets build up the value
398: uint tmpVal=0;
399: switch(trailingBytes) {
400: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
401: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
402: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
403: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
404: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
405: case 0: tmpVal+=*srcPtr++;
406: break;
407:
408: default:
1.100 moko 409: throw Exception(0, 0, "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 410: }
411: tmpVal-=gUTFOffsets[trailingBytes];
412:
413: // If it will fit into a single char, then put it in. Otherwise
414: // fail [*encode it as a surrogate pair. If its not valid, use the
415: // replacement char.*]
416: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 417: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
418: *outPtr++=xlat;
1.49 paf 419: else {
1.50 paf 420: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 421: }
422: } else {
423: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
424: for(uint i=0; i<=trailingBytes; i++)
425: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
426: }
1.1 paf 427: }
1.35 paf 428:
429: // Update the bytes eaten
430: srcLen = srcPtr - srcData;
431:
432: // Return the characters read
433: toFillLen = outPtr - toFill;
1.11 paf 434:
1.29 paf 435: //return srcPtr==srcEnd?(int)toFillLen:-1;
436: /*
437: xmlCharEncodingOutputFunc
438: Returns :
439: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
440: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
441: of ocetes consumed.
442: */
443: return 0;
1.10 paf 444: }
445:
1.85 misha 446: static bool need_escape(XMLByte c){
1.60 misha 447: return
1.66 misha 448: !(
449: (c<=127)
450: && (
1.89 misha 451: pa_isalnum((unsigned char)c)
1.66 misha 452: || strchr("*@-_+./", c)!=0
453: )
454: );
1.60 misha 455: }
456:
1.70 misha 457: // read one UTF8 char and return length of this char (in bytes)
458: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 459: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
460: return 0;
461:
462: firstByte=*srcPtr;
463:
464: if(firstByte<=127){
465: UTF8Char=firstByte;
466: srcPtr++;
467: return 1;
468: }
469:
470: unsigned int trailingBytes=gUTFBytes[firstByte];
471:
472: if(srcPtr+trailingBytes>=srcEnd){
473: return 0; // not enough bytes in source string for reading
474: }
475:
476: uint tmpVal=0;
477: switch(trailingBytes){
478: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
479: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
480: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
481: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
482: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
483: case 0: tmpVal+=*srcPtr++;
484: }
485:
486: tmpVal-=gUTFOffsets[trailingBytes];
487: UTF8Char=tmpVal;
488:
489: return trailingBytes+1;
490: }
491:
1.70 misha 492: // skip UTF8 char and return length of this char (in bytes)
493: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 494: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
495: return 0;
496:
1.63 misha 497: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
498: srcPtr+=trailingBytes;
1.62 misha 499:
500: return trailingBytes;
1.61 misha 501: }
502:
1.85 misha 503: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 504: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 505: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
506: return 0;
507:
508: firstByte=*srcPtr++;
509: UTF8Char=tables.fromTable[firstByte];
510:
511: if(UTF8Char<0x80)
512: return 1;
513: else if(UTF8Char<0x800)
514: return 2;
515: else if(UTF8Char<0x10000)
516: return 3;
517: else if(UTF8Char<0x200000)
518: return 4;
519: else if(UTF8Char<0x4000000)
520: return 5;
521: else if(UTF8Char<= 0x7FFFFFFF)
522: return 6;
523:
524: // will use the replacement character '?'
525: firstByte=0;
526: return 1;
527: }
528:
1.85 misha 529: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
530: size_t dest_length=0;
531:
532: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
533: if(i.getCharSize()==1)
534: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
535: else
536: dest_length+=6; // %uXXXX
1.60 misha 537: }
538:
1.85 misha 539: return dest_length;
1.60 misha 540: }
541:
1.86 moko 542: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
543: const XMLByte* src_end=src+src_length;
544: XMLByte first_byte;
545: XMLCh UTF8_char;
1.85 misha 546: size_t dest_length=0;
547:
1.86 moko 548: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 549: if(char_size==1)
550: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
551: else
552: dest_length+=6; // %uXXXX
1.60 misha 553: }
554:
1.85 misha 555: return dest_length;
556: }
557:
558: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 559: if(!src.length)
1.85 misha 560: return 0;
561:
562: #ifdef PRECALCULATE_DEST_LENGTH
563: if(source_charset.isUTF8())
1.86 moko 564: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 565: else
1.86 moko 566: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 567: #else
568: return src_length*6; // enough for %uXXXX but too memory-hungry
569: #endif
570: }
571:
572: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
573: if(char_size==1) \
574: if(first_byte){ \
575: if(need_escape(first_byte)) \
1.95 moko 576: dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 577: else \
578: *dest_ptr++=first_byte; /*as is*/ \
579: } else \
580: *dest_ptr++='?'; /* replacement char '?' */ \
581: else \
1.95 moko 582: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 583:
584:
585: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
586: XMLByte* dest_ptr=dest;
587:
588: // loop until we either run out of input data
589: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
590: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 591:
1.85 misha 592: return dest_ptr - dest;
1.60 misha 593: }
594:
1.85 misha 595: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
596: const XMLByte* src_end=src+src_length;
597: XMLByte* dest_ptr=dest;
598:
599: XMLByte first_byte;
600: XMLCh UTF8_char;
601: uint char_size;
602:
1.86 moko 603: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 604: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
605:
606: return dest_ptr - dest;
607: }
1.60 misha 608:
609: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 610: if(!src.length)
1.60 misha 611: return String::C("", 0);
612:
1.85 misha 613: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
614: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
615:
616: size_t dest_length;
617: if(source_charset.isUTF8())
1.86 moko 618: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 619: else
1.86 moko 620: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 621:
622: if(dest_length>dest_calculated_length)
623: throw Exception(0, 0, "Charset::escape buffer overflow");
624:
625: dest_body[dest_length]=0; // terminator
626: return String::C((char*)dest_body, dest_length);
627: }
628:
629: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 630: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 631: return String::Body(dest.length ? dest.str:0);
632: }
633:
634: String& Charset::escape(const String& src, const Charset& source_charset) {
635: if(src.is_empty())
636: return *new String();
637:
638: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
639: }
640:
641: inline bool need_json_escape(unsigned char c){
642: return strchr("\n\"\\/\t\r\b\f", c)!=0;
643: }
644:
645: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
646: size_t dest_length=0;
647:
648: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 moko 649: if(i.getCharSize()==1){
650: XMLByte first_byte=i.getFirstByte();
651: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
652: } else
1.85 misha 653: dest_length+=6; // \uXXXX
654: }
655:
656: return dest_length;
657: }
658:
1.86 moko 659: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
660: const XMLByte* src_end=src+src_length;
1.85 misha 661: XMLByte first_byte;
662: XMLCh UTF8_char;
1.60 misha 663: size_t dest_length=0;
664:
1.86 moko 665: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 666: if(char_size==1)
1.93 moko 667: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 668: else
669: dest_length+=6; // \uXXXX
1.60 misha 670: }
1.85 misha 671:
672: return dest_length;
673: }
674:
675: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 676: if(!src.length)
1.85 misha 677: return 0;
678:
679: #ifdef PRECALCULATE_DEST_LENGTH
680: if(source_charset.isUTF8())
1.86 moko 681: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 682: else
1.86 moko 683: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 684: #else
1.85 misha 685: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 686: #endif
1.85 misha 687: }
688:
689: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
690: if(char_size==1) \
691: switch(first_byte){ \
692: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
693: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
694: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
695: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
696: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
697: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
698: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
699: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
700: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.95 moko 701: default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
1.93 moko 702: else *dest_ptr++=first_byte; \
1.85 misha 703: } \
704: else \
1.95 moko 705: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 706:
707:
708: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
709: XMLByte* dest_ptr=dest;
710:
711: // loop until we either run out of input data
712: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
713: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
714:
715: return dest_ptr - dest;
716: }
717:
718: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
719: const XMLByte* src_end=src+src_length;
720: XMLByte* dest_ptr=dest;
721:
722: XMLByte first_byte;
723: XMLCh UTF8_char;
724: uint char_size;
725:
1.86 moko 726: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 727: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
728:
729: return dest_ptr - dest;
730: }
1.60 misha 731:
1.85 misha 732: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 733: if(!src.length)
1.85 misha 734: return String::C("", 0);
1.60 misha 735:
1.85 misha 736: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
737: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
738:
739: size_t dest_length;
740: if(source_charset.isUTF8())
1.86 moko 741: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 742: else
1.86 moko 743: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 744:
1.85 misha 745: if(dest_length>dest_calculated_length)
746: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 747:
748: dest_body[dest_length]=0; // terminator
749: return String::C((char*)dest_body, dest_length);
750: }
1.85 misha 751:
752: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 753: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 754: return String::Body(dest.length ? dest.str:0);
1.64 misha 755: }
756:
1.85 misha 757: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 758: if(src.is_empty())
1.73 misha 759: return *new String();
1.64 misha 760:
1.85 misha 761: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 762: }
1.60 misha 763:
1.35 paf 764: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 765: int src_length=src.length;
1.60 misha 766:
767: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 768: int dest_length=0;
1.60 misha 769: const XMLByte* srcPtr=(XMLByte*)src.str;
770: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 771: XMLByte firstByte;
772: XMLCh UTF8Char;
773: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 774: dest_length+=charSize;
775: #else
1.85 misha 776: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 777: #endif
778:
1.35 paf 779: #ifndef NDEBUG
1.71 misha 780: int saved_dest_length=dest_length;
1.35 paf 781: #endif
782: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 783:
784: if(::transcodeToUTF8(
1.35 paf 785: (XMLByte *)src.str, src_length,
786: dest_body, dest_length,
1.11 paf 787: tables)<0)
1.100 moko 788: throw Exception(0, 0, "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 789:
1.60 misha 790: assert(dest_length<=saved_dest_length);
791: dest_body[dest_length]=0; // terminator
1.35 paf 792: return String::C((char*)dest_body, dest_length);
1.10 paf 793: }
1.38 paf 794:
795: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 796: int lo = 0;
797: int hi = table.size - 1;
1.39 paf 798: while(lo<=hi) {
1.38 paf 799: // Calc the mid point of the low and high offset.
1.39 paf 800: const unsigned int i = (lo + hi) / 2;
801:
802: XMLCh cur=table.records[i].from;
803: if(src==cur)
804: return table.records[i].to;
805: if(src>cur)
806: lo = i+1;
1.38 paf 807: else
1.39 paf 808: hi = i-1;
809: }
810:
811: // not found
1.38 paf 812: return src;
813: }
814:
1.58 misha 815: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 816: if(!src) {
817: // use the replacement character
818: *outPtr++= '?';
819: return;
820: }
821:
822: // Figure out how many bytes we need
823: unsigned int encodedBytes;
824: if(src<0x80)
825: encodedBytes = 1;
826: else if(src<0x800)
827: encodedBytes = 2;
828: else if(src<0x10000)
829: encodedBytes = 3;
830: else if(src<0x200000)
831: encodedBytes = 4;
832: else if(src<0x4000000)
833: encodedBytes = 5;
834: else if(src<= 0x7FFFFFFF)
835: encodedBytes = 6;
836: else {
837: // use the replacement character
838: *outPtr++= '?';
839: return;
840: }
841:
842: // And spit out the bytes. We spit them out in reverse order
843: // here, so bump up the output pointer and work down as we go.
844: outPtr+= encodedBytes;
845: switch(encodedBytes) {
846: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
847: src>>= 6;
848: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
849: src>>= 6;
850: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
851: src>>= 6;
852: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
853: src>>= 6;
854: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
855: src>>= 6;
856: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
857: }
858:
859: // Add the encoded bytes back in again to indicate we've eaten them
860: outPtr+= encodedBytes;
861: }
862:
1.99 moko 863: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr, const Charset::UTF8CaseTable& table) {
1.38 paf 864: store_UTF8(change_case_UTF8(src, table), outPtr);
1.98 moko 865: }
866:
1.99 moko 867: void change_case_UTF8(const XMLByte* srcData, size_t srcLen, XMLByte* toFill, size_t toFillLen, const Charset::UTF8CaseTable& table) {
1.38 paf 868: const XMLByte* srcPtr=srcData;
1.44 paf 869: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 870: XMLByte* outPtr=toFill;
1.44 paf 871: XMLByte* outEnd=toFill+toFillLen;
872:
873: // We now loop until we either run out of input data, or room to store
874: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
875: // Get the next leading byte out
876: const XMLByte firstByte =* srcPtr;
1.38 paf 877:
1.60 misha 878: if(firstByte<=127) {
1.38 paf 879: change_case_UTF8(firstByte, outPtr, table);
880: srcPtr++;
881: continue;
882: }
883:
884: // See how many trailing src bytes this sequence is going to require
885: const unsigned int trailingBytes = gUTFBytes[firstByte];
886:
887: // Looks ok, so lets build up the value
888: uint tmpVal=0;
889: switch(trailingBytes) {
890: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
891: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
892: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
893: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
894: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
895: case 0: tmpVal+=*srcPtr++;
896: break;
897:
898: default:
1.100 moko 899: throw Exception(0, 0, "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
1.38 paf 900: }
901: tmpVal-=gUTFOffsets[trailingBytes];
902:
903: // If it will fit into a single char, then put it in. Otherwise
904: // fail [*encode it as a surrogate pair. If its not valid, use the
905: // replacement char.*]
906: if(!(tmpVal & 0xFFFF0000))
907: change_case_UTF8(tmpVal, outPtr, table);
908: else
1.100 moko 909: throw Exception(0, 0, "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
1.38 paf 910: }
911:
912: if(srcPtr!=outPtr)
1.100 moko 913: throw Exception(0, 0, "change_case_UTF8 error: end pointers do not match");
1.38 paf 914: }
915:
1.60 misha 916: static size_t getDecNumLength(XMLCh UTF8Char){
917: return
918: (UTF8Char < 100)
919: ?2
920: :(UTF8Char < 1000)
921: ?3
922: :(UTF8Char < 10000)
923: ?4
924: :5;
925: }
1.38 paf 926:
1.35 paf 927: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 928: int src_length=src.length;
1.60 misha 929: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 930: int dest_length=0;
1.82 misha 931: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 932: dest_length += ( i.next() & 0xFFFF0000 )
933: ? 3*i.getCharSize() // %XX for each byte
934: : ( xlatOneTo(i.next(), tables, 0) != 0 )
935: ? 1 // can convert it to a single char
936: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 937: }
938: #else
939: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 940: int dest_length=src_length*6;
1.60 misha 941: #endif
942:
1.35 paf 943: #ifndef NDEBUG
1.71 misha 944: int saved_dest_length=dest_length;
1.35 paf 945: #endif
946: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 947:
948: if(::transcodeFromUTF8(
1.82 misha 949: (XMLByte *)src.str, src_length,
1.35 paf 950: dest_body, dest_length,
1.11 paf 951: tables)<0)
1.100 moko 952: throw Exception(0, 0, "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 953:
1.60 misha 954: assert(dest_length<=saved_dest_length);
955: dest_body[dest_length]=0; // terminator
1.35 paf 956: return String::C((char*)dest_body, dest_length);
1.1 paf 957: }
958:
959: /// transcode using both charsets
1.99 moko 960: const String::C Charset::transcodeToCharset(const String::C src, const Charset& dest_charset) const {
1.35 paf 961: if(&dest_charset==this)
962: return src;
963: else {
964: size_t dest_length=src.length;
965: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
966:
967: XMLByte* output=dest_body;
968: const XMLByte* input=(XMLByte *)src.str;
969: while(XMLCh c=*input++) {
970: XMLCh curVal = tables.fromTable[c];
971: *output++=curVal?
972: xlatOneTo(curVal, dest_charset.tables, '?') // OK
973: :'?'; // use the replacement character
1.6 paf 974: }
1.1 paf 975:
1.35 paf 976: dest_body[dest_length]=0; // terminator
977: return String::C((char*)dest_body, dest_length);
1.6 paf 978: }
1.1 paf 979: }
980:
1.58 misha 981: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 982: if(isUTF8())
1.58 misha 983: store_UTF8(src, outPtr);
1.59 misha 984: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 985: *outPtr++=ch;
1.57 misha 986: }
987:
1.1 paf 988: #ifdef XML
1.10 paf 989:
1.35 paf 990: static const Charset::Tables* tables[MAX_CHARSETS];
1.46 paf 991:
1.35 paf 992: #define declareXml256ioFuncs(i) \
993: static int xml256CharEncodingInputFunc##i( \
994: unsigned char *out, int *outlen, \
995: const unsigned char *in, int *inlen) { \
996: return transcodeToUTF8( \
1.71 misha 997: in, *inlen, \
998: out, *outlen, \
1.35 paf 999: *tables[i]); \
1000: } \
1001: static int xml256CharEncodingOutputFunc##i( \
1002: unsigned char *out, int *outlen, \
1003: const unsigned char *in, int *inlen) { \
1004: return transcodeFromUTF8( \
1.71 misha 1005: in, *inlen, \
1006: out, *outlen, \
1.35 paf 1007: *tables[i]); \
1008: }
1009:
1010: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1011: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1012: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1013: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1014: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1015:
1016: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1017: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1018: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1019: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1020: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1021: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1022: };
1023: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1024: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1025: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1026: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1027: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1028: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1029: };
1030: static size_t handlers_count=0;
1.10 paf 1031:
1032: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1033: if(handlers_count==MAX_CHARSETS)
1.100 moko 1034: throw Exception(0, 0, "already allocated %d handlers, no space for new encoding '%s'", MAX_CHARSETS, name_cstr);
1.35 paf 1035:
1.116 moko 1036: ftranscoder_input=inputFuncs[handlers_count];
1037: ftranscoder_output=outputFuncs[handlers_count];
1038: ::tables[handlers_count++]=&tables;
1039:
1040: xmlCharEncodingHandler* handler=xmlNewCharEncodingHandler(name_cstr, ftranscoder_input, ftranscoder_output);
1041: if(!handler)
1.117 moko 1042: throw Exception(0, new String(name_cstr, String::L_TAINTED), "unable to register XML encoding handler");
1.10 paf 1043: xmlRegisterCharEncodingHandler(handler);
1044: }
1045:
1.54 paf 1046: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1047: if(!s)
1.35 paf 1048: return String::C("", 0);
1.8 paf 1049:
1.35 paf 1050: int inlen=strlen((const char*)s);
1.51 paf 1051: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1052: #ifndef NDEBUG
1053: int saved_outlen=outlen;
1054: #endif
1055: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1056:
1.30 paf 1057: int error;
1.116 moko 1058: if(!fisUTF8) {
1059: error=ftranscoder_output((unsigned char*)out, &outlen, (const unsigned char*)s, &inlen);
1.30 paf 1060: } else {
1061: memcpy(out, s, outlen=inlen);
1062: error=0;
1063: }
1064: if(error<0)
1.100 moko 1065: throw Exception(0, 0, "transcode_cstr failed (%d)", error);
1.8 paf 1066:
1.35 paf 1067: assert(outlen<=saved_outlen); out[outlen]=0;
1068: return String::C(out, outlen);
1.14 paf 1069: }
1.54 paf 1070: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1071: String::C cstr=transcode_cstr(s);
1.75 misha 1072: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1073: }
1074:
1.8 paf 1075: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1076: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1077: xmlChar* out;
1.30 paf 1078: int outlen;
1079: int error;
1.35 paf 1080: #ifndef NDEBUG
1081: int saved_outlen;
1082: #endif
1.116 moko 1083: if(!fisUTF8) {
1.51 paf 1084: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1085: #ifndef NDEBUG
1086: saved_outlen=outlen;
1087: #endif
1.47 paf 1088: out=(xmlChar*)xmlMalloc(outlen+1);
1.116 moko 1089: error=ftranscoder_input(out, &outlen, (const unsigned char*)buf, (int*)&buf_size);
1.30 paf 1090: } else {
1091: outlen=buf_size;
1.35 paf 1092: #ifndef NDEBUG
1093: saved_outlen=outlen;
1094: #endif
1095: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1096: memcpy(out, buf, outlen);
1097: error=0;
1098: }
1.17 paf 1099:
1.30 paf 1100: if(error<0)
1.100 moko 1101: throw Exception(0, 0, "transcode_buf failed (%d)", error);
1.8 paf 1102:
1.35 paf 1103: assert(outlen<=saved_outlen); out[outlen]=0;
1104: return out;
1.24 paf 1105: }
1.1 paf 1106:
1.79 misha 1107: xmlChar* Charset::transcode(const String& s) {
1108: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1109: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1110: }
1.35 paf 1111:
1.79 misha 1112: xmlChar* Charset::transcode(const String::Body s) {
1113: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1114: }
1.36 paf 1115: #endif
1.34 paf 1116:
1.99 moko 1117: String::Body Charset::transcode(const String::Body src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.101 moko 1118: return String::Body(Charset::transcode(String::C(src.cstr(), src.length()), source_transcoder, dest_transcoder));
1.35 paf 1119: }
1120:
1.99 moko 1121: String& Charset::transcode(const String& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.72 misha 1122: if(src.is_empty())
1.73 misha 1123: return *new String();
1.34 paf 1124:
1.37 paf 1125: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1126: }
1127:
1.99 moko 1128: void Charset::transcode(ArrayString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1129: for(size_t i=0; i<src.count(); i++)
1130: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1131: }
1132:
1133: #ifndef DOXYGEN
1134: struct Transcode_pair_info {
1135: const Charset* source_transcoder;
1136: const Charset* dest_transcoder;
1137: };
1138: #endif
1.99 moko 1139: static void transcode_pair(HashStringValue::key_type /*akey*/, String::Body& avalue, Transcode_pair_info* info) {
1140: avalue=Charset::transcode(avalue, *info->source_transcoder, *info->dest_transcoder);
1.34 paf 1141: }
1.61 misha 1142:
1.99 moko 1143: void Charset::transcode(HashStringString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1144: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1145: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1146: }
1.61 misha 1147:
1148: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1149: const XMLByte* ptr=srcBegin;
1.70 misha 1150: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1151:
1152: return ptr-srcBegin;
1153: }
1154:
1155: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1156: size_t charPos=0;
1157: const XMLByte* ptr=srcBegin;
1158: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1159: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1160: if(ptr>ptrEnd)
1161: return charPos;
1162: charPos++;
1163: }
1164:
1165: // scan till end but position in bytes still too low
1.107 moko 1166: throw Exception(0, 0, "Error conversion byte pos to char pos");
1.61 misha 1167: }
1168:
1169: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1170: size_t size=0;
1.70 misha 1171: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1172: size++;
1173:
1174: return size;
1175: }
1.80 misha 1176:
1.84 misha 1177: unsigned int lengthUTF8Char(const XMLByte c){
1178: return gUTFBytes[c]+1;
1179: }
1180:
1.94 moko 1181: const char *fixUTF8(const char *src){
1182: if(src && *src){
1183: size_t length=strlen(src);
1184:
1185: int error_offset;
1.96 moko 1186: if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
1.94 moko 1187:
1188: char *result=(char *)pa_malloc_atomic(length+1);
1189: char *dst=result;
1190:
1191: do {
1192:
1193: if(error_offset){
1.110 moko 1194: memcpy(dst, src, error_offset);
1.94 moko 1195: dst+=error_offset;
1196:
1197: src+=error_offset;
1198: length-=error_offset;
1199:
1200: }
1201:
1202: *dst++='?';
1203: src++;
1204: length--;
1205:
1.96 moko 1206: } while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));
1.94 moko 1207:
1208: if(length){
1209: strcpy(dst, src);
1210: } else {
1211: *dst='\0';
1212: }
1213:
1214: return result;
1215: }
1216: }
1217: return src;
1218: }
1219:
1.80 misha 1220: bool UTF8_string_iterator::has_next(){
1221: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1222: return fcharSize!=0;
1223: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main