Annotation of parser3/src/main/pa_charset.C, revision 1.66
1.1 paf 1: /** @file
2: Parser: Charset connection implementation.
3:
1.52 paf 4: Copyright(c) 2001-2005 ArtLebedev Group (http://www.artlebedev.com)
1.4 paf 5: Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
1.27 paf 6: */
1.1 paf 7:
1.66 ! misha 8: static const char * const IDENT_CHARSET_C="$Date: 2008-08-15 15:30:21 $";
1.1 paf 9:
10: #include "pa_charset.h"
1.35 paf 11: #include "pa_charsets.h"
1.1 paf 12:
13: #ifdef XML
1.8 paf 14: #include "libxml/encoding.h"
1.1 paf 15: #endif
16:
1.46 paf 17: //#define PA_PATCHED_LIBXML_BACKWARD
1.60 misha 18: #define PRECALCULATE_DEST_LENGTH
1.46 paf 19:
1.38 paf 20: // globals
21:
22: Charset::UTF8CaseTable::Rec UTF8CaseToUpperRecords[]={
23: #include "utf8-to-upper.inc"
24: };
25: Charset::UTF8CaseTable UTF8CaseToUpper={
26: sizeof(UTF8CaseToUpperRecords)/sizeof(Charset::UTF8CaseTable::Rec),
27: UTF8CaseToUpperRecords};
28:
29: Charset::UTF8CaseTable::Rec UTF8CaseToLowerRecords[]={
30: #include "utf8-to-lower.inc"
31: };
32: Charset::UTF8CaseTable UTF8CaseToLower={
33: sizeof(UTF8CaseToLowerRecords)/sizeof(Charset::UTF8CaseTable::Rec),
34: UTF8CaseToLowerRecords};
35:
1.1 paf 36: // helpers
37:
38: inline void prepare_case_tables(unsigned char *tables) {
39: unsigned char *lcc_table=tables+lcc_offset;
40: unsigned char *fcc_table=tables+fcc_offset;
41: for(int i=0; i<0x100; i++)
1.53 paf 42: lcc_table[i]=fcc_table[i]=(unsigned char)i;
1.1 paf 43: }
44: inline void cstr2ctypes(unsigned char *tables, const unsigned char *cstr,
45: unsigned char bit) {
46: unsigned char *ctypes_table=tables+ctypes_offset;
47: ctypes_table[0]=bit;
48: for(; *cstr; cstr++) {
49: unsigned char c=*cstr;
50: ctypes_table[c]|=bit;
51: }
52: }
1.35 paf 53: inline unsigned int to_wchar_code(const char* cstr) {
1.1 paf 54: if(!cstr || !*cstr)
55: return 0;
56: if(cstr[1]==0)
1.4 paf 57: return(unsigned int)(unsigned char)cstr[0];
1.1 paf 58:
59: char *error_pos;
1.4 paf 60: return(unsigned int)strtol(cstr, &error_pos, 0);
1.1 paf 61: }
1.35 paf 62: inline bool to_bool(const char* cstr) {
1.1 paf 63: return cstr && *cstr!=0;
64: }
65: static void element2ctypes(unsigned char c, bool belongs,
66: unsigned char *tables, unsigned char bit, int group_offset=-1) {
67: if(!belongs)
68: return;
69:
70: unsigned char *ctypes_table=tables+ctypes_offset;
71:
72: ctypes_table[c]|=bit;
73: if(group_offset>=0)
1.4 paf 74: tables[cbits_offset+group_offset+c/8] |= 1<<(c%8);
1.1 paf 75: }
76: static void element2case(unsigned char from, unsigned char to,
77: unsigned char *tables) {
78: if(!to)
79: return;
80:
81: unsigned char *lcc_table=tables+lcc_offset;
82: unsigned char *fcc_table=tables+fcc_offset;
83: lcc_table[from]=to;
84: fcc_table[from]=to; fcc_table[to]=from;
85: }
86:
87: // methods
88:
1.37 paf 89: Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
1.35 paf 90: FNAME(ANAME),
91: FNAME_CSTR(ANAME.cstrm()) {
1.7 paf 92:
1.35 paf 93: if(afile_spec) {
1.1 paf 94: fisUTF8=false;
1.35 paf 95: load_definition(*charsets, *afile_spec);
1.1 paf 96: #ifdef XML
1.35 paf 97: addEncoding(FNAME_CSTR);
1.1 paf 98: #endif
99: } else {
100: fisUTF8=true;
1.4 paf 101: // grab default onces [for UTF-8 so to be able to make a-z =>A-Z
1.65 misha 102: memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
1.1 paf 103: }
104:
105: #ifdef XML
1.35 paf 106: initTranscoder(FNAME, FNAME_CSTR);
1.1 paf 107: #endif
108: }
109:
1.35 paf 110: void Charset::load_definition(Request_charsets& charsets, const String& afile_spec) {
1.1 paf 111: // pcre_tables
112: // lowcase, flipcase, bits digit+word+whitespace, masks
113:
114: // must not move this inside of prepare_case_tables
115: // don't know the size there
116: memset(pcre_tables, 0, sizeof(pcre_tables));
117: prepare_case_tables(pcre_tables);
1.4 paf 118: cstr2ctypes(pcre_tables,(const unsigned char *)"*+?{^.$|()[", ctype_meta);
1.1 paf 119:
120: // charset
1.35 paf 121: memset(&tables, 0, sizeof(tables));
1.1 paf 122:
123: // loading text
1.35 paf 124: char *data=file_read_text(charsets, afile_spec);
1.1 paf 125:
126: // ignore header
127: getrow(&data);
128:
129: // parse cells
130: char *row;
1.42 paf 131: while((row=getrow(&data))) {
1.1 paf 132: // remove empty&comment lines
133: if(!*row || *row=='#')
134: continue;
135:
136: // char white-space digit hex-digit letter word lowercase unicode1 unicode2
1.53 paf 137: unsigned char c=0;
1.1 paf 138: char *cell;
1.42 paf 139: for(int column=0; (cell=lsplit(&row, '\t')); column++) {
1.1 paf 140: switch(column) {
1.53 paf 141: case 0: c=(unsigned char)to_wchar_code(cell); break;
1.1 paf 142: // pcre_tables
143: case 1: element2ctypes(c, to_bool(cell), pcre_tables, ctype_space, cbit_space); break;
144: case 2: element2ctypes(c, to_bool(cell), pcre_tables, ctype_digit, cbit_digit); break;
145: case 3: element2ctypes(c, to_bool(cell), pcre_tables, ctype_xdigit); break;
146: case 4: element2ctypes(c, to_bool(cell), pcre_tables, ctype_letter); break;
147: case 5: element2ctypes(c, to_bool(cell), pcre_tables, ctype_word, cbit_word); break;
1.53 paf 148: case 6: element2case(c, (unsigned char)to_wchar_code(cell), pcre_tables); break;
1.1 paf 149: case 7:
150: case 8:
151: // charset
1.10 paf 152: if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
1.56 misha 153: throw Exception(PARSER_RUNTIME,
1.35 paf 154: &afile_spec,
1.1 paf 155: "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
156:
157: XMLCh unicode=(XMLCh)to_wchar_code(cell);
158: if(!unicode && column==7/*unicode1 column*/)
159: unicode=(XMLCh)c;
160: if(unicode) {
1.10 paf 161: if(!tables.fromTable[c])
162: tables.fromTable[c]=unicode;
163: tables.toTable[tables.toTableSize].intCh=unicode;
164: tables.toTable[tables.toTableSize].extCh=(XMLByte)c;
165: tables.toTableSize++;
1.1 paf 166: }
167: break;
168: }
169: }
170: };
171:
172: // sort by the Unicode code point
173: sort_ToTable();
174: }
175:
176: static int sort_cmp_Trans_rec_intCh(const void *a, const void *b) {
177: return
1.38 paf 178: static_cast<const Charset::Tables::Rec *>(a)->intCh-
179: static_cast<const Charset::Tables::Rec *>(b)->intCh;
1.1 paf 180: }
181:
182: void Charset::sort_ToTable() {
1.10 paf 183: _qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable),
1.1 paf 184: sort_cmp_Trans_rec_intCh);
185: //FILE *f=fopen("c:\\temp\\a", "wb");
1.10 paf 186: //fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
1.1 paf 187: //fclose(f);
188: }
189:
1.60 misha 190: // @todo: precache for spedup searching
1.10 paf 191: static XMLByte xlatOneTo(const XMLCh toXlat,
1.35 paf 192: const Charset::Tables& tables,
193: XMLByte not_found) {
1.39 paf 194: int lo = 0;
195: int hi = tables.toTableSize - 1;
196: while(lo<=hi) {
1.35 paf 197: // Calc the mid point of the low and high offset.
1.39 paf 198: const unsigned int i = (lo + hi) / 2;
199:
200: XMLCh cur=tables.toTable[i].intCh;
201: if(toXlat==cur)
202: return tables.toTable[i].extCh;
203: if(toXlat>cur)
204: lo = i+1;
1.1 paf 205: else
1.39 paf 206: hi = i-1;
207: }
1.35 paf 208:
209: return not_found;
1.1 paf 210: }
211:
1.35 paf 212: String::C Charset::transcode(const String::C src,
213: const Charset& source_charset,
214: const Charset& dest_charset) {
215: if(!src.length)
216: return String::C("", 0);
1.4 paf 217:
1.1 paf 218: switch((source_charset.isUTF8()?0x10:0x00)|(dest_charset.isUTF8()?0x01:0x00)) {
219: default: // 0x00
1.35 paf 220: return source_charset.transcodeToCharset(src, dest_charset);
1.1 paf 221: case 0x01:
1.35 paf 222: return source_charset.transcodeToUTF8(src);
1.1 paf 223: case 0x10:
1.35 paf 224: return dest_charset.transcodeFromUTF8(src);
1.1 paf 225: case 0x11:
1.35 paf 226: return src;
1.1 paf 227: }
228: }
229:
230: // ---------------------------------------------------------------------------
231: // Local static data
232: //
233: // gUTFBytes
234: // A list of counts of trailing bytes for each initial byte in the input.
235: //
236: // gUTFOffsets
237: // A list of values to offset each result char type, according to how
238: // many source bytes when into making it.
239: //
240: // gFirstByteMark
241: // A list of values to mask onto the first byte of an encoded sequence,
242: // indexed by the number of bytes used to create the sequence.
243: // ---------------------------------------------------------------------------
244: static const XMLByte gUTFBytes[0x100] = {
245: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
246: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
247: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
248: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
249: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
250: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
251: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
252: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
253: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
254: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
255: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
256: , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
257: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
258: , 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
259: , 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
260: , 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
261: };
262:
263: static const uint gUTFOffsets[6] = {
264: 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080
265: };
266:
267: static const XMLByte gFirstByteMark[7] = {
268: 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
269: };
270:
1.35 paf 271: static int transcodeToUTF8(const XMLByte* srcData, size_t& srcLen,
272: XMLByte *toFill, size_t& toFillLen,
273: const Charset::Tables& tables) {
1.11 paf 274: const XMLByte* srcPtr=srcData;
275: const XMLByte* srcEnd=srcData+srcLen;
276: XMLByte* outPtr=toFill;
277: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 278:
1.35 paf 279: while(srcPtr<srcEnd) {
280: uint curVal = tables.fromTable[*srcPtr];
1.1 paf 281: if(!curVal) {
1.35 paf 282: // use the replacement character
283: *outPtr++= '?';
284: srcPtr++;
285: continue;
286: }
1.1 paf 287:
1.35 paf 288: // Figure out how many bytes we need
289: unsigned int encodedBytes;
290: if(curVal<0x80)
291: encodedBytes = 1;
292: else if(curVal<0x800)
293: encodedBytes = 2;
294: else if(curVal<0x10000)
295: encodedBytes = 3;
296: else if(curVal<0x200000)
297: encodedBytes = 4;
298: else if(curVal<0x4000000)
299: encodedBytes = 5;
300: else if(curVal<= 0x7FFFFFFF)
301: encodedBytes = 6;
302: else {
303: // use the replacement character
304: *outPtr++= '?';
305: srcPtr++;
306: continue;
307: }
1.11 paf 308:
1.35 paf 309: // If we cannot fully get this char into the output buffer
310: if (outPtr + encodedBytes > outEnd)
311: break;
312:
313: // We can do it, so update the source index
314: srcPtr++;
315:
316: // And spit out the bytes. We spit them out in reverse order
317: // here, so bump up the output pointer and work down as we go.
318: outPtr+= encodedBytes;
319: switch(encodedBytes) {
1.60 misha 320: case 6: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
321: curVal>>= 6;
322: case 5: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
323: curVal>>= 6;
324: case 4: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
325: curVal>>= 6;
326: case 3: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
327: curVal>>= 6;
328: case 2: *--outPtr = XMLByte((curVal | 0x80UL) & 0xBFUL);
329: curVal>>= 6;
330: case 1: *--outPtr = XMLByte(curVal | gFirstByteMark[encodedBytes]);
1.35 paf 331: }
332:
333: // Add the encoded bytes back in again to indicate we've eaten them
334: outPtr+= encodedBytes;
335: }
336:
337: // Update the bytes eaten
338: srcLen = srcPtr - srcData;
339:
340: // Return the characters read
341: toFillLen = outPtr - toFill;
342:
1.29 paf 343: //return srcPtr==srcEnd?(int)toFillLen:-1;
344: /*
345: xmlCharEncodingInputFunc
346: Returns :
347: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
348: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
349: of ocetes consumed.
350: */
351: return 0;
1.1 paf 352: }
1.26 paf 353: /// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
1.35 paf 354: static int transcodeFromUTF8(const XMLByte* srcData, size_t& srcLen,
355: XMLByte* toFill, size_t& toFillLen,
356: const Charset::Tables& tables) {
1.11 paf 357: const XMLByte* srcPtr=srcData;
358: const XMLByte* srcEnd=srcData+srcLen;
359: XMLByte* outPtr=toFill;
360: XMLByte* outEnd=toFill+toFillLen;
1.1 paf 361:
1.35 paf 362: // We now loop until we either run out of input data, or room to store
363: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
364: // Get the next leading byte out
365: const XMLByte firstByte =* srcPtr;
366:
367: // Special-case ASCII, which is a leading byte value of<= 127
1.60 misha 368: if(firstByte<=127) {
1.35 paf 369: *outPtr++= firstByte;
370: srcPtr++;
371: continue;
372: }
373:
374: // See how many trailing src bytes this sequence is going to require
375: const unsigned int trailingBytes = gUTFBytes[firstByte];
376:
377: // If there are not enough source bytes to do this one, then we
378: // are done. Note that we done>= here because we are implicitly
379: // counting the 1 byte we get no matter what.
380: if(srcPtr+trailingBytes>= srcEnd)
381: break;
382:
383: // Looks ok, so lets build up the value
384: uint tmpVal=0;
385: switch(trailingBytes) {
386: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
387: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
388: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
389: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
390: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
391: case 0: tmpVal+=*srcPtr++;
392: break;
393:
394: default:
395: throw Exception(0,
396: 0,
1.49 paf 397: "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
1.35 paf 398: }
399: tmpVal-=gUTFOffsets[trailingBytes];
400:
401: // If it will fit into a single char, then put it in. Otherwise
402: // fail [*encode it as a surrogate pair. If its not valid, use the
403: // replacement char.*]
404: if(!(tmpVal & 0xFFFF0000)) {
1.25 paf 405: if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
406: *outPtr++=xlat;
1.49 paf 407: else {
1.50 paf 408: outPtr+=sprintf((char *)outPtr, "&#%u;", tmpVal); // &#decimal;
1.49 paf 409: }
410: } else {
411: const XMLByte* recoverPtr=srcPtr-trailingBytes-1;
412: for(uint i=0; i<=trailingBytes; i++)
413: outPtr+=sprintf((char*)outPtr, "%%%02X", *recoverPtr++);
414: }
1.1 paf 415: }
1.35 paf 416:
417: // Update the bytes eaten
418: srcLen = srcPtr - srcData;
419:
420: // Return the characters read
421: toFillLen = outPtr - toFill;
1.11 paf 422:
1.29 paf 423: //return srcPtr==srcEnd?(int)toFillLen:-1;
424: /*
425: xmlCharEncodingOutputFunc
426: Returns :
427: the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
428: number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
429: of ocetes consumed.
430: */
431: return 0;
1.10 paf 432: }
433:
1.60 misha 434: static bool is_escaped(char c){
435: return
1.66 ! misha 436: !(
! 437: (c<=127)
! 438: && (
! 439: ((c>='0') && (c<='9'))
! 440: || ((c>='A') && (c<='Z'))
! 441: || ((c>='a') && (c<='z'))
! 442: || strchr("*@-_+./", c)!=0
! 443: )
! 444: );
1.60 misha 445: }
446:
447: // read one utf8 character, return number of bytes needed for store it
1.61 misha 448: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char){
1.60 misha 449: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
450: return 0;
451:
452: firstByte=*srcPtr;
453:
454: if(firstByte<=127){
455: UTF8Char=firstByte;
456: srcPtr++;
457: return 1;
458: }
459:
460: unsigned int trailingBytes=gUTFBytes[firstByte];
461:
462: if(srcPtr+trailingBytes>=srcEnd){
463: return 0; // not enough bytes in source string for reading
464: }
465:
466: uint tmpVal=0;
467: switch(trailingBytes){
468: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
469: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
470: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
471: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
472: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
473: case 0: tmpVal+=*srcPtr++;
474: }
475:
476: tmpVal-=gUTFOffsets[trailingBytes];
477: UTF8Char=tmpVal;
478:
479: return trailingBytes+1;
480: }
481:
1.62 misha 482: static unsigned int skipChar(const XMLByte*& srcPtr, const XMLByte* srcEnd){
483: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
484: return 0;
485:
1.63 misha 486: unsigned int trailingBytes=gUTFBytes[*srcPtr]+1;
487: srcPtr+=trailingBytes;
1.62 misha 488:
489: return trailingBytes;
1.61 misha 490: }
491:
1.60 misha 492: // read char, return number of bytes needed for store it as UTF8
1.61 misha 493: static unsigned int readChar(const XMLByte*& srcPtr, const XMLByte* srcEnd, XMLByte& firstByte, XMLCh& UTF8Char, const Charset::Tables& tables){
1.60 misha 494: if(!srcPtr || !*srcPtr || srcPtr>=srcEnd)
495: return 0;
496:
497: firstByte=*srcPtr++;
498: UTF8Char=tables.fromTable[firstByte];
499:
500: if(UTF8Char<0x80)
501: return 1;
502: else if(UTF8Char<0x800)
503: return 2;
504: else if(UTF8Char<0x10000)
505: return 3;
506: else if(UTF8Char<0x200000)
507: return 4;
508: else if(UTF8Char<0x4000000)
509: return 5;
510: else if(UTF8Char<= 0x7FFFFFFF)
511: return 6;
512:
513: // will use the replacement character '?'
514: firstByte=0;
515: return 1;
516: }
517:
518: static int escape(const XMLByte* srcData, size_t& srcLen,
519: XMLByte* toFill, size_t& toFillLen) {
520: const XMLByte* srcPtr=srcData;
521: const XMLByte* srcEnd=srcData+srcLen;
522: XMLByte* outPtr=toFill;
523: XMLByte* outEnd=toFill+toFillLen;
524: XMLByte firstByte;
525: XMLCh UTF8Char;
526: uint charSize;
527:
528: // loop until we either run out of input data, or room to store
529: while((outPtr < outEnd) && (charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char))){
530: if(charSize==1){
531: if(is_escaped(firstByte)) // %XX
532: outPtr+=sprintf((char*)outPtr, "%%%02X", firstByte);
533: else
534: *outPtr++=firstByte;
535: } else
536: outPtr+=sprintf((char*)outPtr, "%%u%04X", UTF8Char); // %uXXXX
537: }
538:
539: // Update the bytes eaten
540: srcLen=srcPtr-srcData;
541:
542: // Return the characters read
543: toFillLen=outPtr-toFill;
544:
545: return 0;
546: }
547:
548: static int escape(const XMLByte* srcData, size_t& srcLen,
549: XMLByte *toFill, size_t& toFillLen,
550: const Charset::Tables& tables) {
551: const XMLByte* srcPtr=srcData;
552: const XMLByte* srcEnd=srcData+srcLen;
553: XMLByte* outPtr=toFill;
554: //XMLByte* outEnd=toFill+toFillLen;
555: XMLByte firstByte;
556: XMLCh UTF8Char;
557: uint charSize;
558:
559: while(charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables)){
560: if(charSize==1){
561: if(firstByte){
562: if(is_escaped(firstByte)) // %XX
563: outPtr+=sprintf((char*)outPtr, "%%%02X", firstByte);
564: else
565: *outPtr++=firstByte;
566: } else // add replacement char '?'
567: *outPtr++='?';
568: } else
569: outPtr+=sprintf((char*)outPtr, "%%u%04X", UTF8Char); // %uXXXX
570: }
571:
572: // Update the bytes eaten
573: srcLen = srcPtr - srcData;
574:
575: // Return the characters read
576: toFillLen = outPtr - toFill;
577:
578: return 0;
579: }
580:
581:
582: String::C Charset::escape(const String::C src, const Charset& source_charset){
583: size_t src_length=src.length;
584: if(!src_length)
585: return String::C("", 0);
586:
587: #ifdef PRECALCULATE_DEST_LENGTH
588: size_t dest_length=0;
589: const XMLByte* srcPtr=(XMLByte*)src.str;
590: const XMLByte* srcEnd=srcPtr+src_length;
591: XMLByte firstByte;
592: XMLCh UTF8Char;
593:
594: if(source_charset.isUTF8()){
595: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char)){
596: if(charSize==1)
597: dest_length+=!is_escaped(firstByte)?1:3/*%XX*/;
598: else
599: dest_length+=6; // '%uXXXX'
600: }
601: } else {
602: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, source_charset.tables)){
603: if(charSize==1)
604: dest_length+=(!firstByte/*replacement char '?'*/ || !is_escaped(firstByte))?1:3/*'%XX'*/;
605: else
606: dest_length+=6; // '%uXXXX'
607: }
608: }
609: #else
610: size_t dest_length=src_length*6; // enough for %uXXXX but too memory-hungry
611: #endif
612:
613: //throw Exception(0,0,"%u",dest_length);
614:
615: #ifndef NDEBUG
616: size_t saved_dest_length=dest_length;
617: #endif
618: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
619:
620: int status;
621: if(source_charset.isUTF8()){
622: status=::escape((XMLByte *)src.str, src_length, dest_body, dest_length);
623: } else {
624: status=::escape((XMLByte *)src.str, src_length, dest_body, dest_length, source_charset.tables);
625: }
626:
627: if(status<0)
628: throw Exception(0,
629: 0,
630: "Charset::escapeString buffer overflow");
631:
632: assert(dest_length<=saved_dest_length);
633: dest_body[dest_length]=0; // terminator
634: return String::C((char*)dest_body, dest_length);
635: }
636:
1.64 misha 637: String::Body Charset::escape(const String::Body src, const Charset& source_charset) {
638: const char *src_ptr=src.cstr();
639: size_t src_size=strlen(src_ptr);
640:
641: String::C dest=Charset::escape(String::C(src_ptr, src_size),
642: source_charset);
643:
644: return String::Body(dest.str, dest.length);
645: }
646:
647: String& Charset::escape(const String& src, const Charset& source_charset) {
648: if(!src.length())
649: return *new String("", 0, false);
650:
651: return *new String(escape((String::Body)src, source_charset), String::L_CLEAN);
652: }
1.60 misha 653:
1.35 paf 654: const String::C Charset::transcodeToUTF8(const String::C src) const {
655: size_t src_length=src.length;
1.60 misha 656:
657: #ifdef PRECALCULATE_DEST_LENGTH
658: size_t dest_length=0;
659: const XMLByte* srcPtr=(XMLByte*)src.str;
660: const XMLByte* srcEnd=srcPtr+src_length;
661: XMLByte firstByte;
662: XMLCh UTF8Char;
663: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, tables))
664: dest_length+=charSize;
665: #else
666: size_t dest_length=src_length*6; // so that surly enough (max utf8 seq len=6) but too memory-hyngry
667: #endif
668:
669: //throw Exception(0,0,"%u",dest_length);
670:
1.35 paf 671: #ifndef NDEBUG
672: size_t saved_dest_length=dest_length;
673: #endif
674: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 675:
676: if(::transcodeToUTF8(
1.35 paf 677: (XMLByte *)src.str, src_length,
678: dest_body, dest_length,
1.11 paf 679: tables)<0)
1.43 paf 680: throw Exception(0,
1.10 paf 681: 0,
1.11 paf 682: "Charset::transcodeToUTF8 buffer overflow");
1.10 paf 683:
1.60 misha 684: assert(dest_length<=saved_dest_length);
685: dest_body[dest_length]=0; // terminator
1.35 paf 686: return String::C((char*)dest_body, dest_length);
1.10 paf 687: }
1.38 paf 688:
689: static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) {
1.39 paf 690: int lo = 0;
691: int hi = table.size - 1;
692: while(lo<=hi) {
1.38 paf 693: // Calc the mid point of the low and high offset.
1.39 paf 694: const unsigned int i = (lo + hi) / 2;
695:
696: XMLCh cur=table.records[i].from;
697: if(src==cur)
698: return table.records[i].to;
699: if(src>cur)
700: lo = i+1;
1.38 paf 701: else
1.39 paf 702: hi = i-1;
703: }
704:
705: // not found
1.38 paf 706: return src;
707: }
708:
1.58 misha 709: static void store_UTF8(XMLCh src, XMLByte*& outPtr){
1.38 paf 710: if(!src) {
711: // use the replacement character
712: *outPtr++= '?';
713: return;
714: }
715:
716: // Figure out how many bytes we need
717: unsigned int encodedBytes;
718: if(src<0x80)
719: encodedBytes = 1;
720: else if(src<0x800)
721: encodedBytes = 2;
722: else if(src<0x10000)
723: encodedBytes = 3;
724: else if(src<0x200000)
725: encodedBytes = 4;
726: else if(src<0x4000000)
727: encodedBytes = 5;
728: else if(src<= 0x7FFFFFFF)
729: encodedBytes = 6;
730: else {
731: // use the replacement character
732: *outPtr++= '?';
733: return;
734: }
735:
736: // And spit out the bytes. We spit them out in reverse order
737: // here, so bump up the output pointer and work down as we go.
738: outPtr+= encodedBytes;
739: switch(encodedBytes) {
740: case 6: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
741: src>>= 6;
742: case 5: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
743: src>>= 6;
744: case 4: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
745: src>>= 6;
746: case 3: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
747: src>>= 6;
748: case 2: *--outPtr = XMLByte((src | 0x80UL) & 0xBFUL);
749: src>>= 6;
750: case 1: *--outPtr = XMLByte(src | gFirstByteMark[encodedBytes]);
751: }
752:
753: // Add the encoded bytes back in again to indicate we've eaten them
754: outPtr+= encodedBytes;
755: }
756:
757: static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,
758: const Charset::UTF8CaseTable& table) {
759: store_UTF8(change_case_UTF8(src, table), outPtr);
760: };
1.44 paf 761: void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
762: XMLByte* toFill, size_t toFillLen,
763: const Charset::UTF8CaseTable& table) {
1.38 paf 764: const XMLByte* srcPtr=srcData;
1.44 paf 765: const XMLByte* srcEnd=srcData+srcLen;
1.38 paf 766: XMLByte* outPtr=toFill;
1.44 paf 767: XMLByte* outEnd=toFill+toFillLen;
768:
769: // We now loop until we either run out of input data, or room to store
770: while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
771: // Get the next leading byte out
772: const XMLByte firstByte =* srcPtr;
1.38 paf 773:
1.60 misha 774: if(firstByte<=127) {
1.38 paf 775: change_case_UTF8(firstByte, outPtr, table);
776: srcPtr++;
777: continue;
778: }
779:
780: // See how many trailing src bytes this sequence is going to require
781: const unsigned int trailingBytes = gUTFBytes[firstByte];
782:
783: // Looks ok, so lets build up the value
784: uint tmpVal=0;
785: switch(trailingBytes) {
786: case 5: tmpVal+=*srcPtr++; tmpVal<<=6;
787: case 4: tmpVal+=*srcPtr++; tmpVal<<=6;
788: case 3: tmpVal+=*srcPtr++; tmpVal<<=6;
789: case 2: tmpVal+=*srcPtr++; tmpVal<<=6;
790: case 1: tmpVal+=*srcPtr++; tmpVal<<=6;
791: case 0: tmpVal+=*srcPtr++;
792: break;
793:
794: default:
795: throw Exception(0,
796: 0,
797: "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
798: }
799: tmpVal-=gUTFOffsets[trailingBytes];
800:
801: // If it will fit into a single char, then put it in. Otherwise
802: // fail [*encode it as a surrogate pair. If its not valid, use the
803: // replacement char.*]
804: if(!(tmpVal & 0xFFFF0000))
805: change_case_UTF8(tmpVal, outPtr, table);
806: else
807: throw Exception(0,
808: 0,
809: "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
810: }
811:
812: if(srcPtr!=outPtr)
813: throw Exception(0,
814: 0,
815: "change_case_UTF8 error: end pointers do not match");
816: }
817:
1.60 misha 818: static size_t getDecNumLength(XMLCh UTF8Char){
819: return
820: (UTF8Char < 100)
821: ?2
822: :(UTF8Char < 1000)
823: ?3
824: :(UTF8Char < 10000)
825: ?4
826: :5;
827: }
1.38 paf 828:
1.35 paf 829: const String::C Charset::transcodeFromUTF8(const String::C src) const {
830: size_t src_length=src.length;
1.60 misha 831:
832: #ifdef PRECALCULATE_DEST_LENGTH
833: size_t dest_length=0;
834: const XMLByte* srcPtr=(XMLByte*)src.str;
835: const XMLByte* srcEnd=srcPtr+src_length;
836: XMLByte firstByte;
837: XMLCh UTF8Char;
838: while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char)){
839: if(charSize==1)
840: dest_length++;
841: else
842: dest_length+=(UTF8Char & 0xFFFF0000)
843: ?charSize*3 // '%XX' for each byte
844: :(xlatOneTo(UTF8Char, tables, 0)!=0)
845: ?1 // can convert it to single char
846: :getDecNumLength(UTF8Char)+3; // &#XX; - &#XXXXX;
847: }
848: #else
849: // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;')
850: size_t dest_length=src_length*6;
851: #endif
852:
853: //throw Exception(0,0,"%u",dest_length);
854:
1.35 paf 855: #ifndef NDEBUG
856: size_t saved_dest_length=dest_length;
857: #endif
858: XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
1.11 paf 859:
860: if(::transcodeFromUTF8(
1.35 paf 861: (XMLByte *)src.str, src_length,
862: dest_body, dest_length,
1.11 paf 863: tables)<0)
1.43 paf 864: throw Exception(0,
1.10 paf 865: 0,
1.35 paf 866: "Charset::transcodeFromUTF8 buffer overflow");
1.10 paf 867:
1.60 misha 868: assert(dest_length<=saved_dest_length);
869: dest_body[dest_length]=0; // terminator
1.35 paf 870: return String::C((char*)dest_body, dest_length);
1.1 paf 871: }
872:
873: /// transcode using both charsets
1.35 paf 874: const String::C Charset::transcodeToCharset(const String::C src,
875: const Charset& dest_charset) const {
876: if(&dest_charset==this)
877: return src;
878: else {
879: size_t dest_length=src.length;
880: XMLByte* dest_body=new(PointerFreeGC) XMLByte[dest_length+1/*for terminator*/];
881:
882: XMLByte* output=dest_body;
883: const XMLByte* input=(XMLByte *)src.str;
884: while(XMLCh c=*input++) {
885: XMLCh curVal = tables.fromTable[c];
886: *output++=curVal?
887: xlatOneTo(curVal, dest_charset.tables, '?') // OK
888: :'?'; // use the replacement character
1.6 paf 889: }
1.1 paf 890:
1.35 paf 891: dest_body[dest_length]=0; // terminator
892: return String::C((char*)dest_body, dest_length);
1.6 paf 893: }
1.1 paf 894: }
895:
1.58 misha 896: void Charset::store_Char(XMLByte*& outPtr, XMLCh src, XMLByte not_found){
1.59 misha 897: if(isUTF8())
1.58 misha 898: store_UTF8(src, outPtr);
1.59 misha 899: else if(char ch=xlatOneTo(src, tables, not_found))
1.58 misha 900: *outPtr++=ch;
1.57 misha 901: }
902:
1.1 paf 903: #ifdef XML
1.10 paf 904:
1.35 paf 905: static const Charset::Tables* tables[MAX_CHARSETS];
906:
1.46 paf 907: #ifdef PA_PATCHED_LIBXML_BACKWARD
908:
909: #define declareXml256ioFuncs(i) \
910: static int xml256CharEncodingInputFunc##i( \
911: unsigned char *out, int *outlen, \
912: const unsigned char *in, int *inlen, void*) { \
913: return transcodeToUTF8( \
914: in, *(size_t*)inlen, \
915: out, *(size_t*)outlen, \
916: *tables[i]); \
917: } \
918: static int xml256CharEncodingOutputFunc##i( \
919: unsigned char *out, int *outlen, \
920: const unsigned char *in, int *inlen, void*) { \
921: return transcodeFromUTF8( \
922: in, *(size_t*)inlen, \
923: out, *(size_t*)outlen, \
924: *tables[i]); \
925: }
926:
927: #else
928:
1.35 paf 929: #define declareXml256ioFuncs(i) \
930: static int xml256CharEncodingInputFunc##i( \
931: unsigned char *out, int *outlen, \
932: const unsigned char *in, int *inlen) { \
933: return transcodeToUTF8( \
934: in, *(size_t*)inlen, \
935: out, *(size_t*)outlen, \
936: *tables[i]); \
937: } \
938: static int xml256CharEncodingOutputFunc##i( \
939: unsigned char *out, int *outlen, \
940: const unsigned char *in, int *inlen) { \
941: return transcodeFromUTF8( \
942: in, *(size_t*)inlen, \
943: out, *(size_t*)outlen, \
944: *tables[i]); \
945: }
946:
1.46 paf 947: #endif
948:
949:
1.35 paf 950: declareXml256ioFuncs(0) declareXml256ioFuncs(1)
951: declareXml256ioFuncs(2) declareXml256ioFuncs(3)
952: declareXml256ioFuncs(4) declareXml256ioFuncs(5)
953: declareXml256ioFuncs(6) declareXml256ioFuncs(7)
954: declareXml256ioFuncs(8) declareXml256ioFuncs(9)
955:
956: static xmlCharEncodingInputFunc inputFuncs[MAX_CHARSETS]={
957: xml256CharEncodingInputFunc0, xml256CharEncodingInputFunc1,
958: xml256CharEncodingInputFunc2, xml256CharEncodingInputFunc3,
959: xml256CharEncodingInputFunc4, xml256CharEncodingInputFunc5,
960: xml256CharEncodingInputFunc6, xml256CharEncodingInputFunc7,
961: xml256CharEncodingInputFunc8, xml256CharEncodingInputFunc9
962: };
963: static xmlCharEncodingOutputFunc outputFuncs[MAX_CHARSETS]={
964: xml256CharEncodingOutputFunc0, xml256CharEncodingOutputFunc1,
965: xml256CharEncodingOutputFunc2, xml256CharEncodingOutputFunc3,
966: xml256CharEncodingOutputFunc4, xml256CharEncodingOutputFunc5,
967: xml256CharEncodingOutputFunc6, xml256CharEncodingOutputFunc7,
968: xml256CharEncodingOutputFunc8, xml256CharEncodingOutputFunc9
969: };
970: static size_t handlers_count=0;
1.10 paf 971:
972: void Charset::addEncoding(char *name_cstr) {
1.35 paf 973: if(handlers_count==MAX_CHARSETS)
974: throw Exception(0,
975: 0,
976: "already allocated %d handlers, no space for new encoding '%s'",
977: MAX_CHARSETS, name_cstr);
978:
1.45 paf 979: xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
1.35 paf 980: {
981: handler->name=name_cstr;
982: handler->input=inputFuncs[handlers_count];
983: handler->output=outputFuncs[handlers_count];
984: ::tables[handlers_count]=&tables;
985: handlers_count++;
986: }
1.10 paf 987:
988: xmlRegisterCharEncodingHandler(handler);
1.35 paf 989:
1.10 paf 990: }
991:
1.37 paf 992: void Charset::initTranscoder(const String::Body NAME, const char* name_cstr) {
1.15 paf 993: ftranscoder=xmlFindCharEncodingHandler(name_cstr);
1.35 paf 994: transcoder(NAME); // check right way
1.15 paf 995: }
996:
1.37 paf 997: xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
1.15 paf 998: if(!ftranscoder)
1.56 misha 999: throw Exception(PARSER_RUNTIME,
1.35 paf 1000: new String(NAME, String::L_TAINTED),
1.10 paf 1001: "unsupported encoding");
1.35 paf 1002: return *ftranscoder;
1.10 paf 1003: }
1004:
1.54 paf 1005: String::C Charset::transcode_cstr(const xmlChar* s) {
1.13 paf 1006: if(!s)
1.35 paf 1007: return String::C("", 0);
1.8 paf 1008:
1.35 paf 1009: int inlen=strlen((const char*)s);
1.51 paf 1010: int outlen=inlen*6/*strlen("ÿ")*/; // max
1.35 paf 1011: #ifndef NDEBUG
1012: int saved_outlen=outlen;
1013: #endif
1014: char *out=new(PointerFreeGC) char[outlen+1];
1.8 paf 1015:
1.30 paf 1016: int error;
1.35 paf 1017: if(xmlCharEncodingOutputFunc output=transcoder(FNAME).output) {
1.30 paf 1018: error=output(
1.17 paf 1019: (unsigned char*)out, &outlen,
1.46 paf 1020: (const unsigned char*)s, &inlen
1021: #ifdef PA_PATCHED_LIBXML_BACKWARD
1022: ,0
1023: #endif
1024: );
1.30 paf 1025: } else {
1026: memcpy(out, s, outlen=inlen);
1027: error=0;
1028: }
1029: if(error<0)
1.23 paf 1030: throw Exception(0,
1.8 paf 1031: 0,
1.30 paf 1032: "transcode_cstr failed (%d)", error);
1.8 paf 1033:
1.35 paf 1034: assert(outlen<=saved_outlen); out[outlen]=0;
1035: return String::C(out, outlen);
1.14 paf 1036: }
1.54 paf 1037: const String& Charset::transcode(const xmlChar* s) {
1.35 paf 1038: String::C cstr=transcode_cstr(s);
1039: return *new String(cstr.str, cstr.length, true);
1.1 paf 1040: }
1041:
1.8 paf 1042: /// @test less memory using -maybe- xmlParserInputBufferCreateMem
1.35 paf 1043: xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
1044: xmlChar* out;
1.30 paf 1045: int outlen;
1046: int error;
1.35 paf 1047: #ifndef NDEBUG
1048: int saved_outlen;
1049: #endif
1050: if(xmlCharEncodingInputFunc input=transcoder(FNAME).input) {
1.51 paf 1051: outlen=buf_size*6/*max UTF8 bytes per char*/;
1.35 paf 1052: #ifndef NDEBUG
1053: saved_outlen=outlen;
1054: #endif
1.47 paf 1055: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1056: error=input(
1.17 paf 1057: out, &outlen,
1.46 paf 1058: (const unsigned char*)buf, (int*)&buf_size
1059: #ifdef PA_PATCHED_LIBXML_BACKWARD
1060: ,0
1061: #endif
1062: );
1.30 paf 1063: } else {
1064: outlen=buf_size;
1.35 paf 1065: #ifndef NDEBUG
1066: saved_outlen=outlen;
1067: #endif
1068: out=(xmlChar*)xmlMalloc(outlen+1);
1.30 paf 1069: memcpy(out, buf, outlen);
1070: error=0;
1071: }
1.17 paf 1072:
1.30 paf 1073: if(error<0)
1.23 paf 1074: throw Exception(0,
1.8 paf 1075: 0,
1.30 paf 1076: "transcode_buf failed (%d)", error);
1.8 paf 1077:
1.35 paf 1078: assert(outlen<=saved_outlen); out[outlen]=0;
1079: return out;
1.24 paf 1080: }
1.54 paf 1081: xmlChar* Charset::transcode(const String& s) {
1.35 paf 1082: const char* cstr=s.cstr(String::L_UNSPECIFIED);
1.1 paf 1083:
1.54 paf 1084: return transcode_buf2xchar(cstr, strlen(cstr));
1.1 paf 1085: }
1.54 paf 1086: xmlChar* Charset::transcode(const String::Body s) {
1.35 paf 1087: const char* cstr=s.cstr();
1088:
1.54 paf 1089: return transcode_buf2xchar(cstr, s.length());
1.35 paf 1090: }
1.36 paf 1091: #endif
1.34 paf 1092:
1.37 paf 1093: String::Body Charset::transcode(const String::Body src,
1.34 paf 1094: const Charset& source_transcoder,
1.35 paf 1095: const Charset& dest_transcoder) {
1.34 paf 1096:
1.35 paf 1097: const char *src_ptr=src.cstr();
1.34 paf 1098: size_t src_size=strlen(src_ptr);
1099:
1.35 paf 1100: String::C dest=Charset::transcode(String::C(src_ptr, src_size),
1101: source_transcoder,
1102: dest_transcoder);
1.34 paf 1103:
1.37 paf 1104: return String::Body(dest.str, dest.length);
1.35 paf 1105: }
1106:
1107: String& Charset::transcode(const String& src,
1108: const Charset& source_transcoder,
1109: const Charset& dest_transcoder) {
1110: if(!src.length())
1111: return *new String("", 0, false);
1.34 paf 1112:
1.37 paf 1113: return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
1.34 paf 1114: }
1115:
1.35 paf 1116: void Charset::transcode(ArrayString& src,
1.34 paf 1117: const Charset& source_transcoder,
1.35 paf 1118: const Charset& dest_transcoder) {
1119: for(size_t i=0; i<src.count(); i++)
1120: src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
1.34 paf 1121: }
1122:
1123: #ifndef DOXYGEN
1124: struct Transcode_pair_info {
1125: const Charset* source_transcoder;
1126: const Charset* dest_transcoder;
1127: };
1128: #endif
1.40 paf 1129: static void transcode_pair(const String::Body /*akey*/,
1.37 paf 1130: String::Body& avalue,
1.35 paf 1131: Transcode_pair_info* info) {
1132: avalue=Charset::transcode(avalue,
1133: *info->source_transcoder,
1134: *info->dest_transcoder);
1.34 paf 1135: }
1.61 misha 1136:
1.35 paf 1137: void Charset::transcode(HashStringString& src,
1.34 paf 1138: const Charset& source_transcoder,
1.35 paf 1139: const Charset& dest_transcoder) {
1140: Transcode_pair_info info={&source_transcoder, &dest_transcoder};
1.55 paf 1141: src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
1.34 paf 1142: }
1.61 misha 1143:
1144: size_t getUTF8BytePos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t charPos){
1145: const XMLByte* ptr=srcBegin;
1.62 misha 1146: while(charPos-- && skipChar(ptr, srcEnd));
1.61 misha 1147:
1148: return ptr-srcBegin;
1149: }
1150:
1151: size_t getUTF8CharPos(const XMLByte* srcBegin, const XMLByte* srcEnd, size_t bytePos){
1152: size_t charPos=0;
1153: const XMLByte* ptr=srcBegin;
1154: const XMLByte* ptrEnd=srcBegin+bytePos;
1.62 misha 1155: while(skipChar(ptr, srcEnd)){
1.61 misha 1156: if(ptr>ptrEnd)
1157: return charPos;
1158: charPos++;
1159: }
1160:
1161: // scan till end but position in bytes still too low
1162: throw Exception(0,
1163: 0,
1164: "Error convertion byte pos to char pos");
1165: }
1166:
1167: size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
1168: size_t size=0;
1.62 misha 1169: while(skipChar(srcBegin, srcEnd))
1.61 misha 1170: size++;
1171:
1172: return size;
1173: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pa_charset.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / main