Annotation of parser3/src/main/pa_charset.C, revision 1.116
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.114 moko 4: Copyright (c) 2001-2024 Art. Lebedev Studio (http://www.artlebedev.com)
1.112 moko 5: Authors: Konstantin Morshnev <moko@design.ru>, Alexandr Petrosian <paf@design.ru>
1.27 paf 6: */
1.1 paf 7:
8: #include "pa_charset.h"
1.35 paf 9: #include "pa_charsets.h"
1.1 paf 10:
1.96 moko 11: // we are using some pcre_internal.h stuff as well
12: #include "../lib/pcre/pa_pcre_internal.h"
13:
1.116 ! moko 14: volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.115 2024/12/11 21:57:35 moko Exp $" IDENT_PA_CHARSET_H;
1.90 moko 15:
1.1 paf 16: #ifdef XML
1.115 moko 17: #include "libxml/xmlmemory.h"
1.8 paf 18: #include "libxml/encoding.h"
1.1 paf 19: #endif
20:
1.67 misha 21: // reduce memory usage by pre-calculation utf-8 string length
1.60 misha 22: #define PRECALCULATE_DEST_LENGTH
1.46 paf 23:
1.38 paf 24: // globals
25:
26: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
27: #include "utf8-to-upper.inc"
28: };
29: Charset::UTF8CaseTable UTF8CaseToUpper={
30: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
31: UTF8CaseToUpperRecords};
32:
33: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
34: #include "utf8-to-lower.inc"
35: };
36: Charset::UTF8CaseTable UTF8CaseToLower={
37: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
38: UTF8CaseToLowerRecords};
39:
1.1 paf 40: // helpers
41:
42: inline void prepare_case_tables(unsigned char *tables) {
43: unsigned char *lcc_table=tables+lcc_offset;
44: unsigned char *fcc_table=tables+fcc_offset;
45: for(int i=0; i<0x100; i++)
1.53 paf 46: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 47: }
1.99 moko 48: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr, unsigned char bit) {
1.1 paf 49: unsigned char *ctypes_table=tables+ctypes_offset;
50: ctypes_table[0]=bit;
51: for(; *cstr; cstr++) {
52: unsigned char c=*cstr;
53: ctypes_table[c]|=bit;
54: }
55: }
1.35 paf 56: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 57: if(!cstr || !*cstr)
58: return 0;
59: if(cstr[1]==0)
1.4 paf 60: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 61:
1.91 moko 62: return pa_atoui(cstr,0);
1.1 paf 63: }
1.35 paf 64: inline bool to_bool(const char* cstr) {
1.1 paf 65: return cstr && *cstr!=0;
66: }
1.99 moko 67: static void element2ctypes(unsigned char c, bool belongs, unsigned char *tables, unsigned char bit, int group_offset=-1) {
1.1 paf 68: if(!belongs)
69: return;
70:
71: unsigned char *ctypes_table=tables+ctypes_offset;
72:
73: ctypes_table[c]|=bit;
74: if(group_offset>=0)
1.4 paf 75: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 76: }
1.99 moko 77: static void element2case(unsigned char from, unsigned char to, unsigned char *tables) {
1.1 paf 78: if(!to)
79: return;
80:
81: unsigned char *lcc_table=tables+lcc_offset;
82: unsigned char *fcc_table=tables+fcc_offset;
83: lcc_table[from]=to;
84: fcc_table[from]=to; fcc_table[to]=from;
85: }
86:
1.95 moko 87: inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
1.93 moko 88: if(prefix) {
1.95 moko 89: strcpy((char *)dest, prefix);
1.93 moko 90: dest+=strlen(prefix);
91: }
92: *dest++=hex_digits[c >> 4];
93: *dest++=hex_digits[c & 0x0F];
1.95 moko 94: return dest;
1.93 moko 95: }
96:
1.95 moko 97: inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
1.93 moko 98: if(prefix) {
1.95 moko 99: strcpy((char *)dest, prefix);
1.93 moko 100: dest+=strlen(prefix);
101: }
102: *dest++=hex_digits[(c >> 12) & 0x0F];
103: *dest++=hex_digits[(c >> 8) & 0x0F];
104: *dest++=hex_digits[(c >> 4) & 0x0F];
105: *dest++=hex_digits[(c) & 0x0F];
1.95 moko 106: return dest;
1.93 moko 107: }
108:
1.1 paf 109: // methods
110:
1.103 moko 111: Charset::Charset(Request_charsets* acharsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 112: FNAME(ANAME),
113: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 114:
1.35 paf 115: if(afile_spec) {
1.1 paf 116: fisUTF8=false;
1.103 moko 117: load_definition(*acharsets, *afile_spec);
1.1 paf 118: #ifdef XML
1.35 paf 119: addEncoding(FNAME_CSTR);
1.1 paf 120: #endif
121: } else {
122: fisUTF8=true;
1.4 paf 123: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.96 moko 124: memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
1.1 paf 125: }
126: }
127:
1.104 moko 128: void Charset::load_definition(Request_charsets& acharsets, const String& afile_spec) {
1.1 paf 129: // pcre_tables
130: // lowcase, flipcase, bits digit+word+whitespace, masks
131:
132: // must not move this inside of prepare_case_tables
133: // don't know the size there
134: memset(pcre_tables, 0, sizeof(pcre_tables));
135: prepare_case_tables(pcre_tables);
1.4 paf 136: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 137:
138: // charset
1.35 paf 139: memset(&tables, 0, sizeof(tables));
1.1 paf 140:
141: // loading text
1.104 moko 142: char *data=file_read_text(acharsets, afile_spec);
1.1 paf 143:
144: // ignore header
145: getrow(&data);
146:
147: // parse cells
148: char *row;
1.42 paf 149: while((row=getrow(&data))) {
1.1 paf 150: // remove empty&comment lines
151: if(!*row || *row=='#')
152: continue;
153:
154: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 155: unsigned char c=0;
1.1 paf 156: char *cell;
1.42 paf 157: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 158: switch(column) {
1.53 paf 159: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 160: // pcre_tables
161: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
162: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
163: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
164: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
165: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 166: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 167: case 7:
168: case 8:
169: // charset
1.10 paf 170: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.113 moko 171: throw Exception(PARSER_RUNTIME, &afile_spec, "charset must contain not more than %d unicode values", MAX_CHARSET_UNI_CODES);
1.1 paf 172:
173: XMLCh unicode=(XMLCh)to_wchar_code(cell);
174: if(!unicode && column==7/*unicode1 column*/)
175: unicode=(XMLCh)c;
176: if(unicode) {
1.10 paf 177: if(!tables.fromTable[c])
178: tables.fromTable[c]=unicode;
179: tables.toTable[tables.toTableSize].intCh=unicode;
180: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
181: tables.toTableSize++;
1.1 paf 182: }
183: break;
184: }
185: }
186: };
187:
1.87 moko 188: // parser charset tables declare only white-space before 0x20, thus adding the missing chars
189: for(uint i=0; i<0x20; i++)
190: if(!tables.fromTable[i]){
191: tables.fromTable[i]=i;
192: tables.toTable[tables.toTableSize].intCh=i;
193: tables.toTable[tables.toTableSize].extCh=(XMLByte)i;
194: tables.toTableSize++;
195: }
196:
1.1 paf 197: // sort by the Unicode code point
198: sort_ToTable();
199: }
200:
201: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
202: return
1.38 paf 203: static_cast<const Charset::Tables::Rec *>(a)->intCh-
204: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 205: }
206:
207: void Charset::sort_ToTable() {
1.92 moko 208: qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
1.1 paf 209: }
210:
1.60 misha 211: // @todo: precache for spedup searching
1.99 moko 212: static XMLByte xlatOneTo(const XMLCh toXlat, const Charset::Tables& tables, XMLByte not_found) {
1.80 misha 213: int lo = 0;
214: int hi = tables.toTableSize - 1;
1.39 paf 215: while(lo<=hi) {
1.35 paf 216: // Calc the mid point of the low and high offset.
1.39 paf 217: const unsigned int i = (lo + hi) / 2;
218:
219: XMLCh cur=tables.toTable[i].intCh;
220: if(toXlat==cur)
221: return tables.toTable[i].extCh;
222: if(toXlat>cur)
223: lo = i+1;
1.1 paf 224: else
1.39 paf 225: hi = i-1;
226: }
1.35 paf 227:
228: return not_found;
1.1 paf 229: }
230:
1.99 moko 231: String::C Charset::transcode(const String::C src, const Charset& source_charset, const Charset& dest_charset) {
1.35 paf 232: if(!src.length)
233: return String::C("", 0);
1.4 paf 234:
1.1 paf 235: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
236: default: // 0x00
1.35 paf 237: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 238: case 0x01:
1.35 paf 239: return source_charset.transcodeToUTF8(src);
1.1 paf 240: case 0x10:
1.35 paf 241: return dest_charset.transcodeFromUTF8(src);
1.1 paf 242: case 0x11:
1.35 paf 243: return src;
1.1 paf 244: }
245: }
246:
247: // ---------------------------------------------------------------------------
248: // Local static data
249: //
250: // gUTFBytes
251: // A list of counts of trailing bytes for each initial byte in the input.
252: //
253: // gUTFOffsets
254: // A list of values to offset each result char type, according to how
255: // many source bytes when into making it.
256: //
257: // gFirstByteMark
258: // A list of values to mask onto the first byte of an encoded sequence,
259: // indexed by the number of bytes used to create the sequence.
260: // ---------------------------------------------------------------------------
261: static const XMLByte gUTFBytes[0x100] = {
262: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
263: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
264: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
265: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
266: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
267: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
268: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
269: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
270: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
271: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
272: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
273: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
274: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
275: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
276: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
277: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
278: };
279:
280: static const uint gUTFOffsets[6] = {
1.80 misha 281: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
1.1 paf 282: };
283:
284: static const XMLByte gFirstByteMark[7] = {
1.80 misha 285: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
1.1 paf 286: };
287:
1.99 moko 288: static int transcodeToUTF8(const XMLByte* srcData, int& srcLen, XMLByte *toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 289: const XMLByte* srcPtr=srcData;
290: const XMLByte* srcEnd=srcData+srcLen;
291: XMLByte* outPtr=toFill;
292: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 293:
1.35 paf 294: while(srcPtr<srcEnd) {
295: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 296: if(!curVal) {
1.35 paf 297: // use the replacement character
298: *outPtr++= '?';
299: srcPtr++;
300: continue;
301: }
1.1 paf 302:
1.35 paf 303: // Figure out how many bytes we need
304: unsigned int encodedBytes;
305: if(curVal<0x80)
306: encodedBytes = 1;
307: else if(curVal<0x800)
308: encodedBytes = 2;
309: else if(curVal<0x10000)
310: encodedBytes = 3;
311: else if(curVal<0x200000)
312: encodedBytes = 4;
313: else if(curVal<0x4000000)
314: encodedBytes = 5;
315: else if(curVal<= 0x7FFFFFFF)
316: encodedBytes = 6;
317: else {
318: // use the replacement character
319: *outPtr++= '?';
320: srcPtr++;
321: continue;
322: }
1.11 paf 323:
1.35 paf 324: // If we cannot fully get this char into the output buffer
325: if (outPtr + encodedBytes > outEnd)
326: break;
327:
328: // We can do it, so update the source index
329: srcPtr++;
330:
331: // And spit out the bytes. We spit them out in reverse order
332: // here, so bump up the output pointer and work down as we go.
333: outPtr+= encodedBytes;
334: switch(encodedBytes) {
1.60 misha 335: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
336: curVal>>= 6;
337: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
338: curVal>>= 6;
339: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
340: curVal>>= 6;
341: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
342: curVal>>= 6;
343: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
344: curVal>>= 6;
345: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 346: }
347:
348: // Add the encoded bytes back in again to indicate we've eaten them
349: outPtr+= encodedBytes;
350: }
351:
352: // Update the bytes eaten
353: srcLen = srcPtr - srcData;
354:
355: // Return the characters read
356: toFillLen = outPtr - toFill;
357:
1.29 paf 358: //return srcPtr==srcEnd?(int)toFillLen:-1;
359: /*
360: xmlCharEncodingInputFunc
361: Returns :
362: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
363: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
364: of ocetes consumed.
365: */
366: return 0;
1.1 paf 367: }
1.26 paf 368: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.99 moko 369: static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen, XMLByte* toFill, int& toFillLen, const Charset::Tables& tables) {
1.11 paf 370: const XMLByte* srcPtr=srcData;
371: const XMLByte* srcEnd=srcData+srcLen;
372: XMLByte* outPtr=toFill;
373: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 374:
1.35 paf 375: // We now loop until we either run out of input data, or room to store
376: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
377: // Get the next leading byte out
378: const XMLByte firstByte =* srcPtr;
379:
380: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 381: if(firstByte<=127) {
1.35 paf 382: *outPtr++= firstByte;
383: srcPtr++;
384: continue;
385: }
386:
387: // See how many trailing src bytes this sequence is going to require
388: const unsigned int trailingBytes = gUTFBytes[firstByte];
389:
390: // If there are not enough source bytes to do this one, then we
391: // are done. Note that we done>= here because we are implicitly
392: // counting the 1 byte we get no matter what.
393: if(srcPtr+trailingBytes>= srcEnd)
394: break;
395:
396: // Looks ok, so lets build up the value
397: uint tmpVal=0;
398: switch(trailingBytes) {
399: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
400: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
401: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
402: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
403: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
404: case 0: tmpVal+=*srcPtr++;
405: break;
406:
407: default:
1.100 moko 408: throw Exception(0, 0, "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 409: }
410: tmpVal-=gUTFOffsets[trailingBytes];
411:
412: // If it will fit into a single char, then put it in. Otherwise
413: // fail [*encode it as a surrogate pair. If its not valid, use the
414: // replacement char.*]
415: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 416: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
417: *outPtr++=xlat;
1.49 paf 418: else {
1.50 paf 419: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 420: }
421: } else {
422: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
423: for(uint i=0; i<=trailingBytes; i++)
424: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
425: }
1.1 paf 426: }
1.35 paf 427:
428: // Update the bytes eaten
429: srcLen = srcPtr - srcData;
430:
431: // Return the characters read
432: toFillLen = outPtr - toFill;
1.11 paf 433:
1.29 paf 434: //return srcPtr==srcEnd?(int)toFillLen:-1;
435: /*
436: xmlCharEncodingOutputFunc
437: Returns :
438: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
439: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
440: of ocetes consumed.
441: */
442: return 0;
1.10 paf 443: }
444:
1.85 misha 445: static bool need_escape(XMLByte c){
1.60 misha 446: return
1.66 misha 447: !(
448: (c<=127)
449: && (
1.89 misha 450: pa_isalnum((unsigned char)c)
1.66 misha 451: || strchr("*@-_+./", c)!=0
452: )
453: );
1.60 misha 454: }
455:
1.70 misha 456: // read one UTF8 char and return length of this char (in bytes)
457: static unsigned int readUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 458: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
459: return 0;
460:
461: firstByte=*srcPtr;
462:
463: if(firstByte<=127){
464: UTF8Char=firstByte;
465: srcPtr++;
466: return 1;
467: }
468:
469: unsigned int trailingBytes=gUTFBytes[firstByte];
470:
471: if(srcPtr+trailingBytes>=srcEnd){
472: return 0; // not enough bytes in source string for reading
473: }
474:
475: uint tmpVal=0;
476: switch(trailingBytes){
477: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
478: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
479: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
480: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
481: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
482: case 0: tmpVal+=*srcPtr++;
483: }
484:
485: tmpVal-=gUTFOffsets[trailingBytes];
486: UTF8Char=tmpVal;
487:
488: return trailingBytes+1;
489: }
490:
1.70 misha 491: // skip UTF8 char and return length of this char (in bytes)
492: static unsigned int skipUTF8Char(const XMLByte*& srcPtr, const XMLByte* srcEnd){
1.62 misha 493: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
494: return 0;
495:
1.63 misha 496: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
497: srcPtr+=trailingBytes;
1.62 misha 498:
499: return trailingBytes;
1.61 misha 500: }
501:
1.85 misha 502: // read non-UTF8 char, and return number of bytes needed for storing this char in UTF8
1.61 misha 503: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 504: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
505: return 0;
506:
507: firstByte=*srcPtr++;
508: UTF8Char=tables.fromTable[firstByte];
509:
510: if(UTF8Char<0x80)
511: return 1;
512: else if(UTF8Char<0x800)
513: return 2;
514: else if(UTF8Char<0x10000)
515: return 3;
516: else if(UTF8Char<0x200000)
517: return 4;
518: else if(UTF8Char<0x4000000)
519: return 5;
520: else if(UTF8Char<= 0x7FFFFFFF)
521: return 6;
522:
523: // will use the replacement character '?'
524: firstByte=0;
525: return 1;
526: }
527:
1.85 misha 528: size_t Charset::calc_escaped_length_UTF8(XMLByte* src, size_t src_length){
529: size_t dest_length=0;
530:
531: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
532: if(i.getCharSize()==1)
533: dest_length+=!need_escape(i.getFirstByte())?1/*as-is*/:3/*%XX*/;
534: else
535: dest_length+=6; // %uXXXX
1.60 misha 536: }
537:
1.85 misha 538: return dest_length;
1.60 misha 539: }
540:
1.86 moko 541: size_t Charset::calc_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
542: const XMLByte* src_end=src+src_length;
543: XMLByte first_byte;
544: XMLCh UTF8_char;
1.85 misha 545: size_t dest_length=0;
546:
1.86 moko 547: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 548: if(char_size==1)
549: dest_length+=(!first_byte/*replacement char '?'*/ || !need_escape(first_byte))?1:3/*'%XX'*/;
550: else
551: dest_length+=6; // %uXXXX
1.60 misha 552: }
553:
1.85 misha 554: return dest_length;
555: }
556:
557: size_t Charset::calc_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 558: if(!src.length)
1.85 misha 559: return 0;
560:
561: #ifdef PRECALCULATE_DEST_LENGTH
562: if(source_charset.isUTF8())
1.86 moko 563: return calc_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 564: else
1.86 moko 565: return calc_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.85 misha 566: #else
567: return src_length*6; // enough for %uXXXX but too memory-hungry
568: #endif
569: }
570:
571: #define escape_char(dest_ptr, char_size, first_byte, UTF8_char) \
572: if(char_size==1) \
573: if(first_byte){ \
574: if(need_escape(first_byte)) \
1.95 moko 575: dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
1.85 misha 576: else \
577: *dest_ptr++=first_byte; /*as is*/ \
578: } else \
579: *dest_ptr++='?'; /* replacement char '?' */ \
580: else \
1.95 moko 581: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
1.85 misha 582:
583:
584: size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
585: XMLByte* dest_ptr=dest;
586:
587: // loop until we either run out of input data
588: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
589: escape_char(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
1.60 misha 590:
1.85 misha 591: return dest_ptr - dest;
1.60 misha 592: }
593:
1.85 misha 594: size_t Charset::escape(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
595: const XMLByte* src_end=src+src_length;
596: XMLByte* dest_ptr=dest;
597:
598: XMLByte first_byte;
599: XMLCh UTF8_char;
600: uint char_size;
601:
1.86 moko 602: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 603: escape_char(dest_ptr, char_size, first_byte, UTF8_char)
604:
605: return dest_ptr - dest;
606: }
1.60 misha 607:
608: String::C Charset::escape(const String::C src, const Charset& source_charset){
1.86 moko 609: if(!src.length)
1.60 misha 610: return String::C("", 0);
611:
1.85 misha 612: size_t dest_calculated_length=calc_escaped_length(src, source_charset);
613: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
614:
615: size_t dest_length;
616: if(source_charset.isUTF8())
1.86 moko 617: dest_length=escape_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 618: else
1.86 moko 619: dest_length=escape((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.85 misha 620:
621: if(dest_length>dest_calculated_length)
622: throw Exception(0, 0, "Charset::escape buffer overflow");
623:
624: dest_body[dest_length]=0; // terminator
625: return String::C((char*)dest_body, dest_length);
626: }
627:
628: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
1.86 moko 629: String::C dest=Charset::escape(String::C(src.cstr(), src.length()), source_charset);
1.85 misha 630: return String::Body(dest.length ? dest.str:0);
631: }
632:
633: String& Charset::escape(const String& src, const Charset& source_charset) {
634: if(src.is_empty())
635: return *new String();
636:
637: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
638: }
639:
640: inline bool need_json_escape(unsigned char c){
641: return strchr("\n\"\\/\t\r\b\f", c)!=0;
642: }
643:
644: size_t Charset::calc_JSON_escaped_length_UTF8(XMLByte* src, size_t src_length){
645: size_t dest_length=0;
646:
647: for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
1.93 moko 648: if(i.getCharSize()==1){
649: XMLByte first_byte=i.getFirstByte();
650: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
651: } else
1.85 misha 652: dest_length+=6; // \uXXXX
653: }
654:
655: return dest_length;
656: }
657:
1.86 moko 658: size_t Charset::calc_JSON_escaped_length(const XMLByte* src, size_t src_length, const Charset::Tables& tables){
659: const XMLByte* src_end=src+src_length;
1.85 misha 660: XMLByte first_byte;
661: XMLCh UTF8_char;
1.60 misha 662: size_t dest_length=0;
663:
1.86 moko 664: while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
1.85 misha 665: if(char_size==1)
1.93 moko 666: dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
1.85 misha 667: else
668: dest_length+=6; // \uXXXX
1.60 misha 669: }
1.85 misha 670:
671: return dest_length;
672: }
673:
674: size_t Charset::calc_JSON_escaped_length(const String::C src, const Charset& source_charset){
1.86 moko 675: if(!src.length)
1.85 misha 676: return 0;
677:
678: #ifdef PRECALCULATE_DEST_LENGTH
679: if(source_charset.isUTF8())
1.86 moko 680: return calc_JSON_escaped_length_UTF8((XMLByte *)src.str, src.length);
1.85 misha 681: else
1.86 moko 682: return calc_JSON_escaped_length((XMLByte *)src.str, src.length, source_charset.tables);
1.60 misha 683: #else
1.85 misha 684: return src_length*6; // enough for \uXXXX but too memory-hungry
1.60 misha 685: #endif
1.85 misha 686: }
687:
688: #define escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char) \
689: if(char_size==1) \
690: switch(first_byte){ \
691: case '\n': *dest_ptr++='\\'; *dest_ptr++='n'; break; \
692: case '"' : *dest_ptr++='\\'; *dest_ptr++='"'; break; \
693: case '\\': *dest_ptr++='\\'; *dest_ptr++='\\'; break; \
694: case '/' : *dest_ptr++='\\'; *dest_ptr++='/'; break; \
695: case '\t': *dest_ptr++='\\'; *dest_ptr++='t'; break; \
696: case '\r': *dest_ptr++='\\'; *dest_ptr++='r'; break; \
697: case '\b': *dest_ptr++='\\'; *dest_ptr++='b'; break; \
698: case '\f': *dest_ptr++='\\'; *dest_ptr++='f'; break; \
699: case 0 : *dest_ptr++='?'; break; /*replacement char*/ \
1.95 moko 700: default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
1.93 moko 701: else *dest_ptr++=first_byte; \
1.85 misha 702: } \
703: else \
1.95 moko 704: dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
1.85 misha 705:
706:
707: size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
708: XMLByte* dest_ptr=dest;
709:
710: // loop until we either run out of input data
711: for(UTF8_string_iterator i((XMLByte *)src, src_length); i.has_next(); )
712: escape_char_JSON(dest_ptr, i.getCharSize(), i.getFirstByte(), i.next())
713:
714: return dest_ptr - dest;
715: }
716:
717: size_t Charset::escape_JSON(const XMLByte* src, size_t src_length, XMLByte* dest, const Charset::Tables& tables) {
718: const XMLByte* src_end=src+src_length;
719: XMLByte* dest_ptr=dest;
720:
721: XMLByte first_byte;
722: XMLCh UTF8_char;
723: uint char_size;
724:
1.86 moko 725: while(char_size=readChar(src, src_end, first_byte, UTF8_char, tables))
1.85 misha 726: escape_char_JSON(dest_ptr, char_size, first_byte, UTF8_char)
727:
728: return dest_ptr - dest;
729: }
1.60 misha 730:
1.85 misha 731: String::C Charset::escape_JSON(const String::C src, const Charset& source_charset){
1.86 moko 732: if(!src.length)
1.85 misha 733: return String::C("", 0);
1.60 misha 734:
1.85 misha 735: size_t dest_calculated_length=calc_JSON_escaped_length(src, source_charset);
736: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_calculated_length+1/*terminator*/];
737:
738: size_t dest_length;
739: if(source_charset.isUTF8())
1.86 moko 740: dest_length=escape_JSON_UTF8((XMLByte *)src.str, src.length, dest_body);
1.85 misha 741: else
1.86 moko 742: dest_length=escape_JSON((XMLByte *)src.str, src.length, dest_body, source_charset.tables);
1.60 misha 743:
1.85 misha 744: if(dest_length>dest_calculated_length)
745: throw Exception(0, 0, "Charset::escape_JSON buffer overflow");
1.60 misha 746:
747: dest_body[dest_length]=0; // terminator
748: return String::C((char*)dest_body, dest_length);
749: }
1.85 misha 750:
751: String::Body Charset::escape_JSON(const String::Body src, const Charset& source_charset) {
1.86 moko 752: String::C dest=Charset::escape_JSON(String::C(src.cstr(), src.length()), source_charset);
1.77 misha 753: return String::Body(dest.length ? dest.str:0);
1.64 misha 754: }
755:
1.85 misha 756: String& Charset::escape_JSON(const String& src, const Charset& source_charset) {
1.72 misha 757: if(src.is_empty())
1.73 misha 758: return *new String();
1.64 misha 759:
1.85 misha 760: return *new String(escape_JSON((String::Body)src, source_charset), String::L_CLEAN);
1.64 misha 761: }
1.60 misha 762:
1.35 paf 763: const String::C Charset::transcodeToUTF8(const String::C src) const {
1.71 misha 764: int src_length=src.length;
1.60 misha 765:
766: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 767: int dest_length=0;
1.60 misha 768: const XMLByte* srcPtr=(XMLByte*)src.str;
769: const XMLByte* srcEnd=srcPtr+src_length;
1.69 misha 770: XMLByte firstByte;
771: XMLCh UTF8Char;
772: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
1.60 misha 773: dest_length+=charSize;
774: #else
1.85 misha 775: int dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hungry
1.60 misha 776: #endif
777:
1.35 paf 778: #ifndef NDEBUG
1.71 misha 779: int saved_dest_length=dest_length;
1.35 paf 780: #endif
781: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 782:
783: if(::transcodeToUTF8(
1.35 paf 784: (XMLByte *)src.str, src_length,
785: dest_body, dest_length,
1.11 paf 786: tables)<0)
1.100 moko 787: throw Exception(0, 0, "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 788:
1.60 misha 789: assert(dest_length<=saved_dest_length);
790: dest_body[dest_length]=0; // terminator
1.35 paf 791: return String::C((char*)dest_body, dest_length);
1.10 paf 792: }
1.38 paf 793:
794: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.80 misha 795: int lo = 0;
796: int hi = table.size - 1;
1.39 paf 797: while(lo<=hi) {
1.38 paf 798: // Calc the mid point of the low and high offset.
1.39 paf 799: const unsigned int i = (lo + hi) / 2;
800:
801: XMLCh cur=table.records[i].from;
802: if(src==cur)
803: return table.records[i].to;
804: if(src>cur)
805: lo = i+1;
1.38 paf 806: else
1.39 paf 807: hi = i-1;
808: }
809:
810: // not found
1.38 paf 811: return src;
812: }
813:
1.58 misha 814: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 815: if(!src) {
816: // use the replacement character
817: *outPtr++= '?';
818: return;
819: }
820:
821: // Figure out how many bytes we need
822: unsigned int encodedBytes;
823: if(src<0x80)
824: encodedBytes = 1;
825: else if(src<0x800)
826: encodedBytes = 2;
827: else if(src<0x10000)
828: encodedBytes = 3;
829: else if(src<0x200000)
830: encodedBytes = 4;
831: else if(src<0x4000000)
832: encodedBytes = 5;
833: else if(src<= 0x7FFFFFFF)
834: encodedBytes = 6;
835: else {
836: // use the replacement character
837: *outPtr++= '?';
838: return;
839: }
840:
841: // And spit out the bytes. We spit them out in reverse order
842: // here, so bump up the output pointer and work down as we go.
843: outPtr+= encodedBytes;
844: switch(encodedBytes) {
845: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
846: src>>= 6;
847: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
848: src>>= 6;
849: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
850: src>>= 6;
851: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
852: src>>= 6;
853: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
854: src>>= 6;
855: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
856: }
857:
858: // Add the encoded bytes back in again to indicate we've eaten them
859: outPtr+= encodedBytes;
860: }
861:
1.99 moko 862: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr, const Charset::UTF8CaseTable& table) {
1.38 paf 863: store_UTF8(change_case_UTF8(src, table), outPtr);
1.98 moko 864: }
865:
1.99 moko 866: void change_case_UTF8(const XMLByte* srcData, size_t srcLen, XMLByte* toFill, size_t toFillLen, const Charset::UTF8CaseTable& table) {
1.38 paf 867: const XMLByte* srcPtr=srcData;
1.44 paf 868: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 869: XMLByte* outPtr=toFill;
1.44 paf 870: XMLByte* outEnd=toFill+toFillLen;
871:
872: // We now loop until we either run out of input data, or room to store
873: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
874: // Get the next leading byte out
875: const XMLByte firstByte =* srcPtr;
1.38 paf 876:
1.60 misha 877: if(firstByte<=127) {
1.38 paf 878: change_case_UTF8(firstByte, outPtr, table);
879: srcPtr++;
880: continue;
881: }
882:
883: // See how many trailing src bytes this sequence is going to require
884: const unsigned int trailingBytes = gUTFBytes[firstByte];
885:
886: // Looks ok, so lets build up the value
887: uint tmpVal=0;
888: switch(trailingBytes) {
889: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
890: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
891: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
892: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
893: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
894: case 0: tmpVal+=*srcPtr++;
895: break;
896:
897: default:
1.100 moko 898: throw Exception(0, 0, "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
1.38 paf 899: }
900: tmpVal-=gUTFOffsets[trailingBytes];
901:
902: // If it will fit into a single char, then put it in. Otherwise
903: // fail [*encode it as a surrogate pair. If its not valid, use the
904: // replacement char.*]
905: if(!(tmpVal & 0xFFFF0000))
906: change_case_UTF8(tmpVal, outPtr, table);
907: else
1.100 moko 908: throw Exception(0, 0, "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
1.38 paf 909: }
910:
911: if(srcPtr!=outPtr)
1.100 moko 912: throw Exception(0, 0, "change_case_UTF8 error: end pointers do not match");
1.38 paf 913: }
914:
1.60 misha 915: static size_t getDecNumLength(XMLCh UTF8Char){
916: return
917: (UTF8Char < 100)
918: ?2
919: :(UTF8Char < 1000)
920: ?3
921: :(UTF8Char < 10000)
922: ?4
923: :5;
924: }
1.38 paf 925:
1.35 paf 926: const String::C Charset::transcodeFromUTF8(const String::C src) const {
1.82 misha 927: int src_length=src.length;
1.60 misha 928: #ifdef PRECALCULATE_DEST_LENGTH
1.71 misha 929: int dest_length=0;
1.82 misha 930: for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){
1.88 misha 931: dest_length += ( i.next() & 0xFFFF0000 )
932: ? 3*i.getCharSize() // %XX for each byte
933: : ( xlatOneTo(i.next(), tables, 0) != 0 )
934: ? 1 // can convert it to a single char
935: : 3+getDecNumLength( i.next() ); // print char as &#XX;, &#XXX;, &#XXXX; or &#XXXXX;
1.60 misha 936: }
937: #else
938: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
1.82 misha 939: int dest_length=src_length*6;
1.60 misha 940: #endif
941:
1.35 paf 942: #ifndef NDEBUG
1.71 misha 943: int saved_dest_length=dest_length;
1.35 paf 944: #endif
945: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 946:
947: if(::transcodeFromUTF8(
1.82 misha 948: (XMLByte *)src.str, src_length,
1.35 paf 949: dest_body, dest_length,
1.11 paf 950: tables)<0)
1.100 moko 951: throw Exception(0, 0, "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 952:
1.60 misha 953: assert(dest_length<=saved_dest_length);
954: dest_body[dest_length]=0; // terminator
1.35 paf 955: return String::C((char*)dest_body, dest_length);
1.1 paf 956: }
957:
958: /// transcode using both charsets
1.99 moko 959: const String::C Charset::transcodeToCharset(const String::C src, const Charset& dest_charset) const {
1.35 paf 960: if(&dest_charset==this)
961: return src;
962: else {
963: size_t dest_length=src.length;
964: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
965:
966: XMLByte* output=dest_body;
967: const XMLByte* input=(XMLByte *)src.str;
968: while(XMLCh c=*input++) {
969: XMLCh curVal = tables.fromTable[c];
970: *output++=curVal?
971: xlatOneTo(curVal, dest_charset.tables, '?') // OK
972: :'?'; // use the replacement character
1.6 paf 973: }
1.1 paf 974:
1.35 paf 975: dest_body[dest_length]=0; // terminator
976: return String::C((char*)dest_body, dest_length);
1.6 paf 977: }
1.1 paf 978: }
979:
1.58 misha 980: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 981: if(isUTF8())
1.58 misha 982: store_UTF8(src, outPtr);
1.59 misha 983: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 984: *outPtr++=ch;
1.57 misha 985: }
986:
1.1 paf 987: #ifdef XML
1.10 paf 988:
1.35 paf 989: static const Charset::Tables* tables[MAX_CHARSETS];
1.46 paf 990:
1.35 paf 991: #define declareXml256ioFuncs(i) \
992: static int xml256CharEncodingInputFunc##i( \
993: unsigned char *out, int *outlen, \
994: const unsigned char *in, int *inlen) { \
995: return transcodeToUTF8( \
1.71 misha 996: in, *inlen, \
997: out, *outlen, \
1.35 paf 998: *tables[i]); \
999: } \
1000: static int xml256CharEncodingOutputFunc##i( \
1001: unsigned char *out, int *outlen, \
1002: const unsigned char *in, int *inlen) { \
1003: return transcodeFromUTF8( \
1.71 misha 1004: in, *inlen, \
1005: out, *outlen, \
1.35 paf 1006: *tables[i]); \
1007: }
1008:
1009: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
1010: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
1011: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
1012: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
1013: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
1014:
1015: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
1016: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
1017: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
1018: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
1019: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
1020: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
1021: };
1022: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
1023: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
1024: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
1025: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
1026: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
1027: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
1028: };
1029: static size_t handlers_count=0;
1.10 paf 1030:
1031: void Charset::addEncoding(char *name_cstr) {
1.35 paf 1032: if(handlers_count==MAX_CHARSETS)
1.100 moko 1033: throw Exception(0, 0, "already allocated %d handlers, no space for new encoding '%s'", MAX_CHARSETS, name_cstr);
1.35 paf 1034:
1.116 ! moko 1035: ftranscoder_input=inputFuncs[handlers_count];
! 1036: ftranscoder_output=outputFuncs[handlers_count];
! 1037: ::tables[handlers_count++]=&tables;
! 1038:
! 1039: xmlCharEncodingHandler* handler=xmlNewCharEncodingHandler(name_cstr, ftranscoder_input, ftranscoder_output);
! 1040: if(!handler)
! 1041: throw Exception(0, new String(NAME, String::L_TAINTED), "unable to register XML encoding handler");
1.10 paf 1042: xmlRegisterCharEncodingHandler(handler);
1043: }
1044:
1.54 paf 1045: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1046: if(!s)
1.35 paf 1047: return String::C("", 0);
1.8 paf 1048:
1.35 paf 1049: int inlen=strlen((const char*)s);
1.51 paf 1050: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1051: #ifndef NDEBUG
1052: int saved_outlen=outlen;
1053: #endif
1054: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1055:
1.30 paf 1056: int error;
1.116 ! moko 1057: if(!fisUTF8) {
! 1058: error=ftranscoder_output((unsigned char*)out, &outlen, (const unsigned char*)s, &inlen);
1.30 paf 1059: } else {
1060: memcpy(out, s, outlen=inlen);
1061: error=0;
1062: }
1063: if(error<0)
1.100 moko 1064: throw Exception(0, 0, "transcode_cstr failed (%d)", error);
1.8 paf 1065:
1.35 paf 1066: assert(outlen<=saved_outlen); out[outlen]=0;
1067: return String::C(out, outlen);
1.14 paf 1068: }
1.54 paf 1069: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1070: String::C cstr=transcode_cstr(s);
1.75 misha 1071: return *new String(cstr.str, String::L_TAINTED);
1.1 paf 1072: }
1073:
1.8 paf 1074: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1075: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1076: xmlChar* out;
1.30 paf 1077: int outlen;
1078: int error;
1.35 paf 1079: #ifndef NDEBUG
1080: int saved_outlen;
1081: #endif
1.116 ! moko 1082: if(!fisUTF8) {
1.51 paf 1083: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1084: #ifndef NDEBUG
1085: saved_outlen=outlen;
1086: #endif
1.47 paf 1087: out=(xmlChar*)xmlMalloc(outlen+1);
1.116 ! moko 1088: error=ftranscoder_input(out, &outlen, (const unsigned char*)buf, (int*)&buf_size);
1.30 paf 1089: } else {
1090: outlen=buf_size;
1.35 paf 1091: #ifndef NDEBUG
1092: saved_outlen=outlen;
1093: #endif
1094: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1095: memcpy(out, buf, outlen);
1096: error=0;
1097: }
1.17 paf 1098:
1.30 paf 1099: if(error<0)
1.100 moko 1100: throw Exception(0, 0, "transcode_buf failed (%d)", error);
1.8 paf 1101:
1.35 paf 1102: assert(outlen<=saved_outlen); out[outlen]=0;
1103: return out;
1.24 paf 1104: }
1.1 paf 1105:
1.79 misha 1106: xmlChar* Charset::transcode(const String& s) {
1107: String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS);
1108: return transcode_buf2xchar(sbody.cstr(), sbody.length());
1.1 paf 1109: }
1.35 paf 1110:
1.79 misha 1111: xmlChar* Charset::transcode(const String::Body s) {
1112: return transcode_buf2xchar(s.cstr(), s.length());
1.35 paf 1113: }
1.36 paf 1114: #endif
1.34 paf 1115:
1.99 moko 1116: String::Body Charset::transcode(const String::Body src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.101 moko 1117: return String::Body(Charset::transcode(String::C(src.cstr(), src.length()), source_transcoder, dest_transcoder));
1.35 paf 1118: }
1119:
1.99 moko 1120: String& Charset::transcode(const String& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.72 misha 1121: if(src.is_empty())
1.73 misha 1122: return *new String();
1.34 paf 1123:
1.37 paf 1124: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1125: }
1126:
1.99 moko 1127: void Charset::transcode(ArrayString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1128: for(size_t i=0; i<src.count(); i++)
1129: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1130: }
1131:
1132: #ifndef DOXYGEN
1133: struct Transcode_pair_info {
1134: const Charset* source_transcoder;
1135: const Charset* dest_transcoder;
1136: };
1137: #endif
1.99 moko 1138: static void transcode_pair(HashStringValue::key_type /*akey*/, String::Body& avalue, Transcode_pair_info* info) {
1139: avalue=Charset::transcode(avalue, *info->source_transcoder, *info->dest_transcoder);
1.34 paf 1140: }
1.61 misha 1141:
1.99 moko 1142: void Charset::transcode(HashStringString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
1.35 paf 1143: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1144: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1145: }
1.61 misha 1146:
1147: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1148: const XMLByte* ptr=srcBegin;
1.70 misha 1149: while(charPos-- && skipUTF8Char(ptr, srcEnd));
1.61 misha 1150:
1151: return ptr-srcBegin;
1152: }
1153:
1154: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1155: size_t charPos=0;
1156: const XMLByte* ptr=srcBegin;
1157: const XMLByte* ptrEnd=srcBegin+bytePos;
1.70 misha 1158: while(skipUTF8Char(ptr, srcEnd)){
1.61 misha 1159: if(ptr>ptrEnd)
1160: return charPos;
1161: charPos++;
1162: }
1163:
1164: // scan till end but position in bytes still too low
1.107 moko 1165: throw Exception(0, 0, "Error conversion byte pos to char pos");
1.61 misha 1166: }
1167:
1168: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1169: size_t size=0;
1.70 misha 1170: while(skipUTF8Char(srcBegin, srcEnd))
1.61 misha 1171: size++;
1172:
1173: return size;
1174: }
1.80 misha 1175:
1.84 misha 1176: unsigned int lengthUTF8Char(const XMLByte c){
1177: return gUTFBytes[c]+1;
1178: }
1179:
1.94 moko 1180: const char *fixUTF8(const char *src){
1181: if(src && *src){
1182: size_t length=strlen(src);
1183:
1184: int error_offset;
1.96 moko 1185: if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
1.94 moko 1186:
1187: char *result=(char *)pa_malloc_atomic(length+1);
1188: char *dst=result;
1189:
1190: do {
1191:
1192: if(error_offset){
1.110 moko 1193: memcpy(dst, src, error_offset);
1.94 moko 1194: dst+=error_offset;
1195:
1196: src+=error_offset;
1197: length-=error_offset;
1198:
1199: }
1200:
1201: *dst++='?';
1202: src++;
1203: length--;
1204:
1.96 moko 1205: } while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));
1.94 moko 1206:
1207: if(length){
1208: strcpy(dst, src);
1209: } else {
1210: *dst='\0';
1211: }
1212:
1213: return result;
1214: }
1215: }
1216: return src;
1217: }
1218:
1.80 misha 1219: bool UTF8_string_iterator::has_next(){
1220: fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
1221: return fcharSize!=0;
1222: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main