--- parser3/src/main/pa_charset.C 2009/05/26 21:33:05 1.77 +++ parser3/src/main/pa_charset.C 2009/11/06 05:01:18 1.84 @@ -5,7 +5,7 @@ Author: Alexander Petrosyan(http://paf.design.ru) */ -static const char * const IDENT_CHARSET_C="$Date: 2009/05/26 21:33:05 $"; +static const char * const IDENT_CHARSET_C="$Date: 2009/11/06 05:01:18 $"; #include "pa_charset.h" #include "pa_charsets.h" @@ -193,8 +193,8 @@ void Charset::sort_ToTable() { static XMLByte xlatOneTo(const XMLCh toXlat, const Charset::Tables& tables, XMLByte not_found) { - int lo = 0; - int hi = tables.toTableSize - 1; + int lo = 0; + int hi = tables.toTableSize - 1; while(lo<=hi) { // Calc the mid point of the low and high offset. const unsigned int i = (lo + hi) / 2; @@ -263,11 +263,11 @@ static const XMLByte gUTFBytes[0x100] = }; static const uint gUTFOffsets[6] = { - 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080 + 0, 0x3080, 0xE2080, 0x3C82080, 0xFA082080, 0x82082080 }; static const XMLByte gFirstByteMark[7] = { - 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC + 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; static int transcodeToUTF8(const XMLByte* srcData, int& srcLen, @@ -519,26 +519,20 @@ static unsigned int readChar(const XMLBy } static int escapeUTF8(const XMLByte* srcData, size_t& srcLen, - XMLByte* toFill, size_t& toFillLen) { + XMLByte* toFill, size_t& toFillLen) { const XMLByte* srcPtr=srcData; - const XMLByte* srcEnd=srcData+srcLen; XMLByte* outPtr=toFill; XMLByte* outEnd=toFill+toFillLen; - XMLByte firstByte; - XMLCh UTF8Char; - uint charSize; - // loop until we either run out of input data, or room to store - while((outPtr < outEnd) && (charSize=readUTF8Char(srcPtr, srcEnd, firstByte, UTF8Char))){ - if(charSize==1){ - if(isEscaped(firstByte)) // %XX - outPtr+=sprintf((char*)outPtr, "%%%02X", firstByte); + for(UTF8_string_iterator i((XMLByte *)srcPtr, srcLen); (outPtr < outEnd) && i.has_next(); ){ + if(i.getCharSize()==1){ + if(isEscaped(i.getFirstByte())) // %XX + outPtr+=sprintf((char*)outPtr, "%%%02X", i.getFirstByte()); else - *outPtr++=firstByte; + *outPtr++=i.getFirstByte(); } else - outPtr+=sprintf((char*)outPtr, "%%u%04X", UTF8Char); // %uXXXX + outPtr+=sprintf((char*)outPtr, "%%u%04X", i.next()); // %uXXXX } - // Update the bytes eaten srcLen=srcPtr-srcData; @@ -588,19 +582,19 @@ String::C Charset::escape(const String:: #ifdef PRECALCULATE_DEST_LENGTH size_t dest_length=0; - const XMLByte* srcPtr=(XMLByte*)src.str; - const XMLByte* srcEnd=srcPtr+src_length; - XMLByte firstByte; - XMLCh UTF8Char; if(source_charset.isUTF8()){ - while(uint charSize=readUTF8Char(srcPtr, srcEnd, firstByte, UTF8Char)){ - if(charSize==1) - dest_length+=!isEscaped(firstByte)?1:3/*%XX*/; + for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){ + if(i.getCharSize()==1) + dest_length+=!isEscaped(i.getFirstByte())?1/*as-is*/:3/*%XX*/; else - dest_length+=6; // '%uXXXX' + dest_length+=6/*%uXXXX*/; } } else { + const XMLByte* srcPtr=(XMLByte*)src.str; + const XMLByte* srcEnd=srcPtr+src_length; + XMLByte firstByte; + XMLCh UTF8Char; while(uint charSize=readChar(srcPtr, srcEnd, firstByte, UTF8Char, source_charset.tables)){ if(charSize==1) dest_length+=(!firstByte/*replacement char '?'*/ || !isEscaped(firstByte))?1:3/*'%XX'*/; @@ -638,7 +632,7 @@ String::C Charset::escape(const String:: String::Body Charset::escape(const String::Body src, const Charset& source_charset) { const char *src_ptr=src.cstr(); - size_t src_size=strlen(src_ptr); + size_t src_size=src.length(); String::C dest=Charset::escape(String::C(src_ptr, src_size), source_charset); @@ -688,8 +682,8 @@ const String::C Charset::transcodeToUTF8 } static XMLCh change_case_UTF8(const XMLCh src, const Charset::UTF8CaseTable& table) { - int lo = 0; - int hi = table.size - 1; + int lo = 0; + int hi = table.size - 1; while(lo<=hi) { // Calc the mid point of the low and high offset. const unsigned int i = (lo + hi) / 2; @@ -829,22 +823,17 @@ static size_t getDecNumLength(XMLCh UTF8 const String::C Charset::transcodeFromUTF8(const String::C src) const { int src_length=src.length; - #ifdef PRECALCULATE_DEST_LENGTH int dest_length=0; - const XMLByte* srcPtr=(XMLByte*)src.str; - const XMLByte* srcEnd=srcPtr+src_length; - XMLByte firstByte; - XMLCh UTF8Char; - while(uint charSize=readUTF8Char(srcPtr, srcEnd, firstByte, UTF8Char)){ - if(charSize==1) + for(UTF8_string_iterator i((XMLByte *)src.str, src_length); i.has_next(); ){ + if(i.getCharSize()==1) dest_length++; else - dest_length+=(UTF8Char & 0xFFFF0000) - ?charSize*3 // '%XX' for each byte - :(xlatOneTo(UTF8Char, tables, 0)!=0) + dest_length+=(i.next() & 0xFFFF0000) + ?i.getCharSize()*3 // '%XX' for each byte + :(xlatOneTo(i.next(), tables, 0)!=0) ?1 // can convert it to single char - :getDecNumLength(UTF8Char)+3; // &#XX; - &#XXXXX; + :getDecNumLength(i.next())+3; // &#XX; - &#XXXXX; } #else // so that surly enough, "&#XXX;" has max ratio (huh? 8 bytes needed for '&#XXXXX;') @@ -871,7 +860,7 @@ const String::C Charset::transcodeFromUT /// transcode using both charsets const String::C Charset::transcodeToCharset(const String::C src, - const Charset& dest_charset) const { + const Charset& dest_charset) const { if(&dest_charset==this) return src; else { @@ -1077,15 +1066,14 @@ xmlChar* Charset::transcode_buf2xchar(co assert(outlen<=saved_outlen); out[outlen]=0; return out; } -xmlChar* Charset::transcode(const String& s) { - const char* cstr=s.cstr(String::L_UNSPECIFIED); - return transcode_buf2xchar(cstr, strlen(cstr)); +xmlChar* Charset::transcode(const String& s) { + String::Body sbody=s.cstr_to_string_body_untaint(String::L_AS_IS); + return transcode_buf2xchar(sbody.cstr(), sbody.length()); } -xmlChar* Charset::transcode(const String::Body s) { - const char* cstr=s.cstr(); - return transcode_buf2xchar(cstr, s.length()); +xmlChar* Charset::transcode(const String::Body s) { + return transcode_buf2xchar(s.cstr(), s.length()); } #endif @@ -1094,7 +1082,7 @@ String::Body Charset::transcode(const St const Charset& dest_transcoder) { const char *src_ptr=src.cstr(); - size_t src_size=strlen(src_ptr); + size_t src_size=src.length(); String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder); @@ -1168,3 +1156,12 @@ size_t lengthUTF8(const XMLByte* srcBegi return size; } + +unsigned int lengthUTF8Char(const XMLByte c){ + return gUTFBytes[c]+1; +} + +bool UTF8_string_iterator::has_next(){ + fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char); + return fcharSize!=0; +}