parser3/src/main/pa_charset.C - diff

Return to pa_charset.C CVS log

Up to [parser3project] / parser3 / src / main

Diff for /parser3/src/main/pa_charset.C between versions 1.92 and 1.101

version 1.92, 2013/07/30 12:36:22	version 1.101, 2015/10/09 12:15:14
Line 8	Line 8
#include "pa_charset.h"	#include "pa_charset.h"
#include "pa_charsets.h"	#include "pa_charsets.h"

	// we are using some pcre_internal.h stuff as well
	#include "../lib/pcre/pa_pcre_internal.h"

volatile const char * IDENT_PA_CHARSET_C="$Id$" IDENT_PA_CHARSET_H;	volatile const char * IDENT_PA_CHARSET_C="$Id$" IDENT_PA_CHARSET_H;

#ifdef XML	#ifdef XML
Line 43 inline void prepare_case_tables(unsigned	Line 46 inline void prepare_case_tables(unsigned
for(int i=0; i<0x100; i++)	for(int i=0; i<0x100; i++)
lcc_table[i]=fcc_table[i]=(unsigned char)i;	lcc_table[i]=fcc_table[i]=(unsigned char)i;
}	}
inline void cstr2ctypes(unsigned char tables, const unsigned char cstr,	inline void cstr2ctypes(unsigned char tables, const unsigned char cstr, unsigned char bit) {
unsigned char bit) {
unsigned char *ctypes_table=tables+ctypes_offset;	unsigned char *ctypes_table=tables+ctypes_offset;
ctypes_table[0]=bit;	ctypes_table[0]=bit;
for(; *cstr; cstr++) {	for(; *cstr; cstr++) {
Line 63 inline unsigned int to_wchar_code(const	Line 65 inline unsigned int to_wchar_code(const
inline bool to_bool(const char* cstr) {	inline bool to_bool(const char* cstr) {
return cstr && *cstr!=0;	return cstr && *cstr!=0;
}	}
static void element2ctypes(unsigned char c, bool belongs,	static void element2ctypes(unsigned char c, bool belongs, unsigned char *tables, unsigned char bit, int group_offset=-1) {
unsigned char *tables, unsigned char bit, int group_offset=-1) {
if(!belongs)	if(!belongs)
return;	return;

Line 74 static void element2ctypes(unsigned char	Line 75 static void element2ctypes(unsigned char
if(group_offset>=0)	if(group_offset>=0)
tables[cbits_offset+group_offset+c/8] \|= 1<<(c%8);	tables[cbits_offset+group_offset+c/8] \|= 1<<(c%8);
}	}
static void element2case(unsigned char from, unsigned char to,	static void element2case(unsigned char from, unsigned char to, unsigned char *tables) {
unsigned char *tables) {
if(!to)	if(!to)
return;	return;

Line 85 static void element2case(unsigned char f	Line 85 static void element2case(unsigned char f
fcc_table[from]=to; fcc_table[to]=from;	fcc_table[from]=to; fcc_table[to]=from;
}	}

	inline XMLByte append_hex_8(XMLByte dest, unsigned char c, const char* prefix=0) {
	if(prefix) {
	strcpy((char *)dest, prefix);
	dest+=strlen(prefix);
	}
	*dest++=hex_digits[c >> 4];
	*dest++=hex_digits[c & 0x0F];
	return dest;
	}

	inline XMLByte append_hex_16(XMLByte dest, unsigned int c, const char* prefix=0) {
	if(prefix) {
	strcpy((char *)dest, prefix);
	dest+=strlen(prefix);
	}
	*dest++=hex_digits[(c >> 12) & 0x0F];
	*dest++=hex_digits[(c >> 8) & 0x0F];
	*dest++=hex_digits[(c >> 4) & 0x0F];
	*dest++=hex_digits[(c) & 0x0F];
	return dest;
	}

// methods	// methods

Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):	Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec):
Line 100 Charset::Charset(Request_charsets* chars	Line 122 Charset::Charset(Request_charsets* chars
} else {	} else {
fisUTF8=true;	fisUTF8=true;
// grab default onces [for UTF-8 so to be able to make a-z =>A-Z	// grab default onces [for UTF-8 so to be able to make a-z =>A-Z
memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));	memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
}	}

#ifdef XML	#ifdef XML
Line 151 void Charset::load_definition(Request_ch	Line 173 void Charset::load_definition(Request_ch
case 8:	case 8:
// charset	// charset
if(tables.toTableSize>MAX_CHARSET_UNI_CODES)	if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, &afile_spec, "charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
&afile_spec,
"charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);

XMLCh unicode=(XMLCh)to_wchar_code(cell);	XMLCh unicode=(XMLCh)to_wchar_code(cell);
if(!unicode && column==7/unicode1 column/)	if(!unicode && column==7/unicode1 column/)
Line 197 void Charset::sort_ToTable() {	Line 217 void Charset::sort_ToTable() {
}	}

// @todo: precache for spedup searching	// @todo: precache for spedup searching
static XMLByte xlatOneTo(const XMLCh toXlat,	static XMLByte xlatOneTo(const XMLCh toXlat, const Charset::Tables& tables, XMLByte not_found) {
const Charset::Tables& tables,
XMLByte not_found) {
int lo = 0;	int lo = 0;
int hi = tables.toTableSize - 1;	int hi = tables.toTableSize - 1;
while(lo<=hi) {	while(lo<=hi) {
Line 218 static XMLByte xlatOneTo(const XMLCh toX	Line 236 static XMLByte xlatOneTo(const XMLCh toX
return not_found;	return not_found;
}	}

String::C Charset::transcode(const String::C src,	String::C Charset::transcode(const String::C src, const Charset& source_charset, const Charset& dest_charset) {
const Charset& source_charset,
const Charset& dest_charset) {
if(!src.length)	if(!src.length)
return String::C("", 0);	return String::C("", 0);

Line 277 static const XMLByte gFirstByteMark[7] =	Line 293 static const XMLByte gFirstByteMark[7] =
0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC	0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
};	};

static int transcodeToUTF8(const XMLByte* srcData, int& srcLen,	static int transcodeToUTF8(const XMLByte* srcData, int& srcLen, XMLByte *toFill, int& toFillLen, const Charset::Tables& tables) {
XMLByte *toFill, int& toFillLen,
const Charset::Tables& tables) {
const XMLByte* srcPtr=srcData;	const XMLByte* srcPtr=srcData;
const XMLByte* srcEnd=srcData+srcLen;	const XMLByte* srcEnd=srcData+srcLen;
XMLByte* outPtr=toFill;	XMLByte* outPtr=toFill;
Line 360 of ocetes consumed.	Line 374 of ocetes consumed.
return 0;	return 0;
}	}
/// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]	/// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen,	static int transcodeFromUTF8(const XMLByte* srcData, int& srcLen, XMLByte* toFill, int& toFillLen, const Charset::Tables& tables) {
XMLByte* toFill, int& toFillLen,
const Charset::Tables& tables) {
const XMLByte* srcPtr=srcData;	const XMLByte* srcPtr=srcData;
const XMLByte* srcEnd=srcData+srcLen;	const XMLByte* srcEnd=srcData+srcLen;
XMLByte* outPtr=toFill;	XMLByte* outPtr=toFill;
Line 401 static int transcodeFromUTF8(const XMLBy	Line 413 static int transcodeFromUTF8(const XMLBy
break;	break;

default:	default:
throw Exception(0,	throw Exception(0, 0, "transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
0,
"transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes); // never
}	}
tmpVal-=gUTFOffsets[trailingBytes];	tmpVal-=gUTFOffsets[trailingBytes];

Line 570 size_t Charset::calc_escaped_length(cons	Line 580 size_t Charset::calc_escaped_length(cons
if(char_size==1) \	if(char_size==1) \
if(first_byte){ \	if(first_byte){ \
if(need_escape(first_byte)) \	if(need_escape(first_byte)) \
dest_ptr+=sprintf((char)dest_ptr, "%%%02X", first_byte); / %XX */ \	dest_ptr=append_hex_8(dest_ptr, first_byte, "%"); /* %XX */ \
else \	else \
dest_ptr++=first_byte; /as is*/ \	dest_ptr++=first_byte; /as is*/ \
} else \	} else \
dest_ptr++='?'; / replacement char '?' */ \	dest_ptr++='?'; / replacement char '?' */ \
else \	else \
dest_ptr+=sprintf((char*)dest_ptr, "%%u%04X", UTF8_char); // %uXXXX	dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */


size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {	size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
Line 643 size_t Charset::calc_JSON_escaped_length	Line 653 size_t Charset::calc_JSON_escaped_length
size_t dest_length=0;	size_t dest_length=0;

for(UTF8_string_iterator i(src, src_length); i.has_next(); ){	for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
if(i.getCharSize()==1)	if(i.getCharSize()==1){
dest_length+=need_json_escape(i.getFirstByte()) ? 2 : 1;	XMLByte first_byte=i.getFirstByte();
else	dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
	} else
dest_length+=6; // \uXXXX	dest_length+=6; // \uXXXX
}	}

Line 660 size_t Charset::calc_JSON_escaped_length	Line 671 size_t Charset::calc_JSON_escaped_length

while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){	while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
if(char_size==1)	if(char_size==1)
dest_length+=(!first_byte/replacement char '?'/ \|\| !need_json_escape(first_byte))? 1 : 2;	dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
else	else
dest_length+=6; // \uXXXX	dest_length+=6; // \uXXXX
}	}
Line 694 size_t Charset::calc_JSON_escaped_length	Line 705 size_t Charset::calc_JSON_escaped_length
case '\b': dest_ptr++='\\'; dest_ptr++='b'; break; \	case '\b': dest_ptr++='\\'; dest_ptr++='b'; break; \
case '\f': dest_ptr++='\\'; dest_ptr++='f'; break; \	case '\f': dest_ptr++='\\'; dest_ptr++='f'; break; \
case 0 : dest_ptr++='?'; break; /replacement char*/ \	case 0 : dest_ptr++='?'; break; /replacement char*/ \
default : *dest_ptr++=first_byte; \	default : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
	else *dest_ptr++=first_byte; \
} \	} \
else \	else \
dest_ptr+=sprintf((char*)dest_ptr, "\\u%04X", UTF8_char); // \uXXXX	dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX


size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {	size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
Line 780 const String::C Charset::transcodeToUTF8	Line 792 const String::C Charset::transcodeToUTF8
(XMLByte *)src.str, src_length,	(XMLByte *)src.str, src_length,
dest_body, dest_length,	dest_body, dest_length,
tables)<0)	tables)<0)
throw Exception(0,	throw Exception(0, 0, "Charset::transcodeToUTF8 buffer overflow");
0,
"Charset::transcodeToUTF8 buffer overflow");

assert(dest_length<=saved_dest_length);	assert(dest_length<=saved_dest_length);
dest_body[dest_length]=0; // terminator	dest_body[dest_length]=0; // terminator
Line 857 static void store_UTF8(XMLCh src, XMLByt	Line 867 static void store_UTF8(XMLCh src, XMLByt
outPtr+= encodedBytes;	outPtr+= encodedBytes;
}	}

static void change_case_UTF8(XMLCh src, XMLByte*& outPtr,	static void change_case_UTF8(XMLCh src, XMLByte*& outPtr, const Charset::UTF8CaseTable& table) {
const Charset::UTF8CaseTable& table) {
store_UTF8(change_case_UTF8(src, table), outPtr);	store_UTF8(change_case_UTF8(src, table), outPtr);
};	}
void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
XMLByte* toFill, size_t toFillLen,	void change_case_UTF8(const XMLByte* srcData, size_t srcLen, XMLByte* toFill, size_t toFillLen, const Charset::UTF8CaseTable& table) {
const Charset::UTF8CaseTable& table) {
const XMLByte* srcPtr=srcData;	const XMLByte* srcPtr=srcData;
const XMLByte* srcEnd=srcData+srcLen;	const XMLByte* srcEnd=srcData+srcLen;
XMLByte* outPtr=toFill;	XMLByte* outPtr=toFill;
Line 895 void change_case_UTF8(const XMLByte* src	Line 903 void change_case_UTF8(const XMLByte* src
break;	break;

default:	default:
throw Exception(0,	throw Exception(0, 0, "change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
0,
"change_case_UTF8 error: wrong trailingBytes value(%d)", trailingBytes);
}	}
tmpVal-=gUTFOffsets[trailingBytes];	tmpVal-=gUTFOffsets[trailingBytes];

Line 907 void change_case_UTF8(const XMLByte* src	Line 913 void change_case_UTF8(const XMLByte* src
if(!(tmpVal & 0xFFFF0000))	if(!(tmpVal & 0xFFFF0000))
change_case_UTF8(tmpVal, outPtr, table);	change_case_UTF8(tmpVal, outPtr, table);
else	else
throw Exception(0,	throw Exception(0, 0, "change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
0,
"change_case_UTF8 error: too big tmpVal(0x%08X)", tmpVal);
}	}

if(srcPtr!=outPtr)	if(srcPtr!=outPtr)
throw Exception(0,	throw Exception(0, 0, "change_case_UTF8 error: end pointers do not match");
0,
"change_case_UTF8 error: end pointers do not match");
}	}

static size_t getDecNumLength(XMLCh UTF8Char){	static size_t getDecNumLength(XMLCh UTF8Char){
Line 954 const String::C Charset::transcodeFromUT	Line 956 const String::C Charset::transcodeFromUT
(XMLByte *)src.str, src_length,	(XMLByte *)src.str, src_length,
dest_body, dest_length,	dest_body, dest_length,
tables)<0)	tables)<0)
throw Exception(0,	throw Exception(0, 0, "Charset::transcodeFromUTF8 buffer overflow");
0,
"Charset::transcodeFromUTF8 buffer overflow");

assert(dest_length<=saved_dest_length);	assert(dest_length<=saved_dest_length);
dest_body[dest_length]=0; // terminator	dest_body[dest_length]=0; // terminator
Line 964 const String::C Charset::transcodeFromUT	Line 964 const String::C Charset::transcodeFromUT
}	}

/// transcode using both charsets	/// transcode using both charsets
const String::C Charset::transcodeToCharset(const String::C src,	const String::C Charset::transcodeToCharset(const String::C src, const Charset& dest_charset) const {
const Charset& dest_charset) const {
if(&dest_charset==this)	if(&dest_charset==this)
return src;	return src;
else {	else {
Line 1064 static size_t handlers_count=0;	Line 1063 static size_t handlers_count=0;

void Charset::addEncoding(char *name_cstr) {	void Charset::addEncoding(char *name_cstr) {
if(handlers_count==MAX_CHARSETS)	if(handlers_count==MAX_CHARSETS)
throw Exception(0,	throw Exception(0, 0, "already allocated %d handlers, no space for new encoding '%s'", MAX_CHARSETS, name_cstr);
0,
"already allocated %d handlers, no space for new encoding '%s'",
MAX_CHARSETS, name_cstr);

xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;	xmlCharEncodingHandler* handler=new xmlCharEncodingHandler;
{	{
handler->name=name_cstr;	handler->name=name_cstr;
handler->input=inputFuncs[handlers_count];	handler->input=inputFuncs[handlers_count];
Line 1089 void Charset::initTranscoder(const Strin	Line 1085 void Charset::initTranscoder(const Strin

xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {	xmlCharEncodingHandler& Charset::transcoder(const String::Body NAME) {
if(!ftranscoder)	if(!ftranscoder)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, new String(NAME, String::L_TAINTED), "unsupported encoding");
new String(NAME, String::L_TAINTED),
"unsupported encoding");
return *ftranscoder;	return *ftranscoder;
}	}

Line 1120 String::C Charset::transcode_cstr(const	Line 1114 String::C Charset::transcode_cstr(const
error=0;	error=0;
}	}
if(error<0)	if(error<0)
throw Exception(0,	throw Exception(0, 0, "transcode_cstr failed (%d)", error);
0,
"transcode_cstr failed (%d)", error);

assert(outlen<=saved_outlen); out[outlen]=0;	assert(outlen<=saved_outlen); out[outlen]=0;
return String::C(out, outlen);	return String::C(out, outlen);
Line 1164 xmlChar* Charset::transcode_buf2xchar(co	Line 1156 xmlChar* Charset::transcode_buf2xchar(co
}	}

if(error<0)	if(error<0)
throw Exception(0,	throw Exception(0, 0, "transcode_buf failed (%d)", error);
0,
"transcode_buf failed (%d)", error);

assert(outlen<=saved_outlen); out[outlen]=0;	assert(outlen<=saved_outlen); out[outlen]=0;
return out;	return out;
Line 1182 xmlChar* Charset::transcode(const String	Line 1172 xmlChar* Charset::transcode(const String
}	}
#endif	#endif

String::Body Charset::transcode(const String::Body src,	String::Body Charset::transcode(const String::Body src, const Charset& source_transcoder, const Charset& dest_transcoder) {
const Charset& source_transcoder,	return String::Body(Charset::transcode(String::C(src.cstr(), src.length()), source_transcoder, dest_transcoder));
const Charset& dest_transcoder) {

const char *src_ptr=src.cstr();
size_t src_size=src.length();

String::C dest=Charset::transcode(String::C(src_ptr, src_size), source_transcoder, dest_transcoder);

return String::Body(dest.length ? dest.str:0);
}	}

String& Charset::transcode(const String& src,	String& Charset::transcode(const String& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
const Charset& source_transcoder,
const Charset& dest_transcoder) {
if(src.is_empty())	if(src.is_empty())
return *new String();	return *new String();

return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);	return *new String(transcode((String::Body)src, source_transcoder, dest_transcoder), String::L_CLEAN);
}	}

void Charset::transcode(ArrayString& src,	void Charset::transcode(ArrayString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
const Charset& source_transcoder,
const Charset& dest_transcoder) {
for(size_t i=0; i<src.count(); i++)	for(size_t i=0; i<src.count(); i++)
src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));	src.put(i, &transcode(*src[i], source_transcoder, dest_transcoder));
}	}
Line 1216 struct Transcode_pair_info {	Line 1194 struct Transcode_pair_info {
const Charset* dest_transcoder;	const Charset* dest_transcoder;
};	};
#endif	#endif
static void transcode_pair(HashStringValue::key_type /akey/,	static void transcode_pair(HashStringValue::key_type /akey/, String::Body& avalue, Transcode_pair_info* info) {
String::Body& avalue,	avalue=Charset::transcode(avalue, info->source_transcoder, info->dest_transcoder);
Transcode_pair_info* info) {
avalue=Charset::transcode(avalue,
*info->source_transcoder,
*info->dest_transcoder);
}	}

void Charset::transcode(HashStringString& src,	void Charset::transcode(HashStringString& src, const Charset& source_transcoder, const Charset& dest_transcoder) {
const Charset& source_transcoder,
const Charset& dest_transcoder) {
Transcode_pair_info info={&source_transcoder, &dest_transcoder};	Transcode_pair_info info={&source_transcoder, &dest_transcoder};
src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);	src.for_each_ref<Transcode_pair_info*>(transcode_pair, &info);
}	}
Line 1249 size_t getUTF8CharPos(const XMLByte* src	Line 1221 size_t getUTF8CharPos(const XMLByte* src
}	}

// scan till end but position in bytes still too low	// scan till end but position in bytes still too low
throw Exception(0,	throw Exception(0, 0, "Error convertion byte pos to char pos");
0,
"Error convertion byte pos to char pos");
}	}

size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){	size_t lengthUTF8(const XMLByte* srcBegin, const XMLByte* srcEnd){
Line 1266 unsigned int lengthUTF8Char(const XMLByt	Line 1236 unsigned int lengthUTF8Char(const XMLByt
return gUTFBytes[c]+1;	return gUTFBytes[c]+1;
}	}

	const char fixUTF8(const char src){
	if(src && *src){
	size_t length=strlen(src);

	int error_offset;
	if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){

	char result=(char )pa_malloc_atomic(length+1);
	char *dst=result;

	do {

	if(error_offset){
	strncpy(dst, src, error_offset);
	dst+=error_offset;

	src+=error_offset;
	length-=error_offset;

	}

	*dst++='?';
	src++;
	length--;

	} while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));

	if(length){
	strcpy(dst, src);
	} else {
	*dst='\0';
	}

	return result;
	}
	}
	return src;
	}

bool UTF8_string_iterator::has_next(){	bool UTF8_string_iterator::has_next(){
fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);	fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
return fcharSize!=0;	return fcharSize!=0;

E-mail:

Removed from v.1.92
changed lines
	Added in v.1.101