parser3/src/main/pa_charset.C - diff

Return to pa_charset.C CVS log

Up to [parser3project] / parser3 / src / main

Diff for /parser3/src/main/pa_charset.C between versions 1.29 and 1.33.2.19.2.12

version 1.29, 2002/09/10 08:41:00	version 1.33.2.19.2.12, 2003/03/26 18:52:50
Line 1	Line 1
/** @file	/** @file
Parser: Charset connection implementation.	Parser: Charset connection implementation.

Copyright(c) 2001, 2002 ArtLebedev Group (http://www.artlebedev.com)	Copyright(c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)
Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)	Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
*/	*/

static const char* IDENT_CHARSET_C="$Date$";	static const char* IDENT_CHARSET_C="$Date$";

#include "pa_charset.h"	#include "pa_charset.h"
	#include "pa_charsets.h"

#ifdef XML	#ifdef XML
#include "libxml/encoding.h"	#include "libxml/encoding.h"
#endif	#endif

// globals


// consts

#define MAX_CHARSET_UNI_CODES 500

// helpers	// helpers

inline void prepare_case_tables(unsigned char *tables) {	inline void prepare_case_tables(unsigned char *tables) {
Line 37 inline void cstr2ctypes(unsigned char *t	Line 31 inline void cstr2ctypes(unsigned char *t
ctypes_table[c]\|=bit;	ctypes_table[c]\|=bit;
}	}
}	}
inline unsigned int to_wchar_code(const char *cstr) {	inline unsigned int to_wchar_code(const char* cstr) {
if(!cstr \|\| !*cstr)	if(!cstr \|\| !*cstr)
return 0;	return 0;
if(cstr[1]==0)	if(cstr[1]==0)
Line 46 inline unsigned int to_wchar_code(const	Line 40 inline unsigned int to_wchar_code(const
char *error_pos;	char *error_pos;
return(unsigned int)strtol(cstr, &error_pos, 0);	return(unsigned int)strtol(cstr, &error_pos, 0);
}	}
inline bool to_bool(const char *cstr) {	inline bool to_bool(const char* cstr) {
return cstr && *cstr!=0;	return cstr && *cstr!=0;
}	}
static void element2ctypes(unsigned char c, bool belongs,	static void element2ctypes(unsigned char c, bool belongs,
Line 74 static void element2case(unsigned char f	Line 68 static void element2case(unsigned char f
// methods	// methods

extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c	extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c
Charset::Charset(Pool& apool, const String& aname, const String *request_file_spec): Pooled(apool),	Charset::Charset(const String& aname, const String* afile_spec):
fname(aname) {	fname(aname),
	fname_cstr(aname.cstrm()) {

char *name_cstr=fname.cstr();	for(char c=fname_cstr; c; c++)
for(char c=name_cstr; c; c++)	c = toupper(c);
c = toupper(c);

if(request_file_spec) {	if(afile_spec) {
fisUTF8=false;	fisUTF8=false;
loadDefinition(*request_file_spec);	load_definition(*afile_spec);
#ifdef XML	#ifdef XML
addEncoding(name_cstr);	addEncoding(fname_cstr);
#endif	#endif
} else {	} else {
fisUTF8=true;	fisUTF8=true;
Line 94 Charset::Charset(Pool& apool, const Stri	Line 88 Charset::Charset(Pool& apool, const Stri
}	}

#ifdef XML	#ifdef XML
initTranscoder(&aname, name_cstr);	initTranscoder(fname, fname_cstr);
#endif
}

Charset::~Charset() {
#ifdef XML
// not deleting transcoder, that's not our business
#endif	#endif
}	}

void Charset::loadDefinition(const String& request_file_spec) {	void Charset::load_definition(const String& afile_spec) {
// pcre_tables	// pcre_tables
// lowcase, flipcase, bits digit+word+whitespace, masks	// lowcase, flipcase, bits digit+word+whitespace, masks

Line 115 void Charset::loadDefinition(const Strin	Line 103 void Charset::loadDefinition(const Strin
cstr2ctypes(pcre_tables,(const unsigned char )"+?{^.$\|()[", ctype_meta);	cstr2ctypes(pcre_tables,(const unsigned char )"+?{^.$\|()[", ctype_meta);

// charset	// charset
memset(tables.fromTable, 0, sizeof(tables.fromTable));	memset(&tables, 0, sizeof(tables));
tables.toTable=(Charset_TransRec )calloc(sizeof(Charset_TransRec)MAX_CHARSET_UNI_CODES);
tables.toTableSize=0;
// strangly vital	// strangly vital
tables.toTable[tables.toTableSize].intCh=0;	tables.toTable[tables.toTableSize].intCh=0;
tables.toTable[tables.toTableSize].extCh=(XMLByte)0;	tables.toTable[tables.toTableSize].extCh=(XMLByte)0;
tables.toTableSize++;	tables.toTableSize++;

// loading text	// loading text
char *data=file_read_text(pool(), request_file_spec);	char *data=file_read_text(UTF8_charset, afile_spec);

// ignore header	// ignore header
getrow(&data);	getrow(&data);
Line 154 void Charset::loadDefinition(const Strin	Line 140 void Charset::loadDefinition(const Strin
// charset	// charset
if(tables.toTableSize>MAX_CHARSET_UNI_CODES)	if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
throw Exception("parser.runtime",	throw Exception("parser.runtime",
&request_file_spec,	&afile_spec,
"charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);	"charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);

XMLCh unicode=(XMLCh)to_wchar_code(cell);	XMLCh unicode=(XMLCh)to_wchar_code(cell);
Line 214 static XMLByte xlatOneTo(const XMLCh toX	Line 200 static XMLByte xlatOneTo(const XMLCh toX
return not_found;	return not_found;
}	}

void Charset::transcode(Pool& pool,	void Charset::transcode(
const Charset& source_charset, const void *source_body, size_t source_content_length,	const Charset& source_charset, const void* source_body, size_t source_content_length,
const Charset& dest_charset, const void *& dest_body, size_t& dest_content_length	const Charset& dest_charset, const void *& dest_body, size_t& dest_content_length
) {	) {
if(!source_content_length) {	if(!source_content_length) {
Line 226 void Charset::transcode(Pool& pool,	Line 212 void Charset::transcode(Pool& pool,

switch((source_charset.isUTF8()?0x10:0x00)\|(dest_charset.isUTF8()?0x01:0x00)) {	switch((source_charset.isUTF8()?0x10:0x00)\|(dest_charset.isUTF8()?0x01:0x00)) {
default: // 0x00	default: // 0x00
source_charset.transcodeToCharset(pool, dest_charset,	source_charset.transcodeToCharset(dest_charset,
source_body, source_content_length,	source_body, source_content_length,
dest_body, dest_content_length);	dest_body, dest_content_length);
break;	break;
case 0x01:	case 0x01:
source_charset.transcodeToUTF8(pool,	source_charset.transcodeToUTF8(
source_body, source_content_length,	source_body, source_content_length,
dest_body, dest_content_length);	dest_body, dest_content_length);
break;	break;
case 0x10:	case 0x10:
dest_charset.transcodeFromUTF8(pool,	dest_charset.transcodeFromUTF8(
source_body, source_content_length,	source_body, source_content_length,
dest_body, dest_content_length);	dest_body, dest_content_length);
break;	break;
Line 372 of ocetes consumed.	Line 358 of ocetes consumed.
return 0;	return 0;
}	}
/// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]	/// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
static size_t transcodeFromUTF8(	static int transcodeFromUTF8(
const XMLByte *srcData, size_t& srcLen,	const XMLByte* srcData, size_t& srcLen,
XMLByte* toFill, size_t& toFillLen,	XMLByte* toFill, size_t& toFillLen,
const Charset::Tables& tables) {	const Charset::Tables& tables) {
const XMLByte* srcPtr=srcData;	const XMLByte* srcPtr=srcData;
Line 384 static size_t transcodeFromUTF8(	Line 370 static size_t transcodeFromUTF8(
// We now loop until we either run out of input data, or room to store	// We now loop until we either run out of input data, or room to store
while ((srcPtr < srcEnd) && (outPtr < outEnd)) {	while ((srcPtr < srcEnd) && (outPtr < outEnd)) {
// Get the next leading byte out	// Get the next leading byte out
const XMLByte firstByte = *srcPtr;	const XMLByte firstByte =* srcPtr;

// Special-case ASCII, which is a leading byte value of<= 127	// Special-case ASCII, which is a leading byte value of<= 127
if(firstByte<= 127) {	if(firstByte<= 127) {
Line 452 of ocetes consumed.	Line 438 of ocetes consumed.
}	}

/// @todo not so memory-hungry with prescan	/// @todo not so memory-hungry with prescan
void Charset::transcodeToUTF8(Pool& pool,	void Charset::transcodeToUTF8(
const void *source_body, size_t source_content_length,	const void* source_body, size_t source_content_length,
const void *& adest_body, size_t& dest_content_length) const {	const void *& adest_body, size_t& dest_content_length) const {
dest_content_length=source_content_length6/so that surly enough, max utf8 seq len=6*/;	dest_content_length=source_content_length6/so that surly enough, max utf8 seq len=6*/;
XMLByte dest_body=(XMLByte)pool.malloc(dest_content_length);	XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_content_length];

if(::transcodeToUTF8(	if(::transcodeToUTF8(
(XMLByte *)source_body, source_content_length,	(XMLByte *)source_body, source_content_length,
Line 469 void Charset::transcodeToUTF8(Pool& pool	Line 455 void Charset::transcodeToUTF8(Pool& pool
// return	// return
adest_body=dest_body;	adest_body=dest_body;
}	}
void Charset::transcodeFromUTF8(Pool& pool,	void Charset::transcodeFromUTF8(
const void *source_body, size_t source_content_length,	const void* source_body, size_t source_content_length,
const void *& adest_body, size_t& dest_content_length) const {	const void *& adest_body, size_t& dest_content_length) const {
dest_content_length=source_content_length6/so that surly enough, "ÿ" has max ratio */;	dest_content_length=source_content_length6/so that surly enough, "ÿ" has max ratio */;
XMLByte dest_body=(XMLByte)pool.malloc(dest_content_length);	XMLByte *dest_body=new(PointerFreeGC) XMLByte[dest_content_length];

if(::transcodeFromUTF8(	if(::transcodeFromUTF8(
(XMLByte *)source_body, source_content_length,	(XMLByte *)source_body, source_content_length,
Line 488 void Charset::transcodeFromUTF8(Pool& po	Line 474 void Charset::transcodeFromUTF8(Pool& po
}	}

/// transcode using both charsets	/// transcode using both charsets
void Charset::transcodeToCharset(Pool& pool,	void Charset::transcodeToCharset(
const Charset& dest_charset,	const Charset& dest_charset,
const void *source_body, size_t source_content_length,	const void* source_body, size_t source_content_length,
const void *& adest_body, size_t& adest_content_length) const {	const void *& adest_body, size_t& adest_content_length) const {
if(&dest_charset==this) {	if(&dest_charset==this) {
adest_body=source_body;	adest_body=source_body;
adest_content_length=source_content_length;	adest_content_length=source_content_length;
} else {	} else {
size_t dest_content_length=source_content_length;	size_t dest_content_length=source_content_length;
unsigned char dest_body=(unsigned char )pool.malloc(dest_content_length);	unsigned char *dest_body=new(PointerFreeGC) unsigned char[dest_content_length];

const XMLByte* srcPtr=(XMLByte *)source_body;	const XMLByte* srcPtr=(XMLByte *)source_body;
const XMLByte* srcEnd=(XMLByte *)source_body+source_content_length;	const XMLByte* srcEnd=(XMLByte *)source_body+source_content_length;
Line 525 static int xml256CharEncodingInp	Line 511 static int xml256CharEncodingInp
int *inlen,	int *inlen,
void *info) {	void *info) {
return transcodeToUTF8(	return transcodeToUTF8(
in, (size_t)inlen,	in, (size_t)inlen,
out, (size_t)outlen,	out, (size_t)outlen,
(const Charset::Tables )info);	(const Charset::Tables )info);
}	}

static int xml256CharEncodingOutputFunc (	static int xml256CharEncodingOutputFunc (
Line 537 static int xml256CharEncodingOut	Line 523 static int xml256CharEncodingOut
int *inlen,	int *inlen,
void *info) {	void *info) {
return transcodeFromUTF8(	return transcodeFromUTF8(
in, (size_t)inlen,	in, (size_t)inlen,
out, (size_t)outlen,	out, (size_t)outlen,
(const Charset::Tables )info);	(const Charset::Tables )info);
}	}


void Charset::addEncoding(char *name_cstr) {	void Charset::addEncoding(char *name_cstr) {
xmlCharEncodingHandler *handler=	xmlCharEncodingHandler* handler=new(PointerFreeGC) xmlCharEncodingHandler;
(xmlCharEncodingHandler *)malloc(sizeof(xmlCharEncodingHandler));
handler->name=name_cstr;	handler->name=name_cstr;
handler->input=xml256CharEncodingInputFunc; handler->inputInfo=&tables;	handler->input=xml256CharEncodingInputFunc; handler->inputInfo=&tables;
handler->output=xml256CharEncodingOutputFunc; handler->outputInfo=&tables;	handler->output=xml256CharEncodingOutputFunc; handler->outputInfo=&tables;
Line 553 void Charset::addEncoding(char *name_cst	Line 538 void Charset::addEncoding(char *name_cst
xmlRegisterCharEncodingHandler(handler);	xmlRegisterCharEncodingHandler(handler);
}	}

void Charset::initTranscoder(const String source, const char name_cstr) {	void Charset::initTranscoder(const String& name, const char* name_cstr) {
ftranscoder=xmlFindCharEncodingHandler(name_cstr);	ftranscoder=xmlFindCharEncodingHandler(name_cstr);
transcoder(source); // check right way	transcoder(name); // check right way
}	}

xmlCharEncodingHandler Charset::transcoder(const String source) {	xmlCharEncodingHandler& Charset::transcoder(const String& name) {
if(!ftranscoder)	if(!ftranscoder)
throw Exception("parser.runtime",	throw Exception("parser.runtime",
source,	&name,
"unsupported encoding");	"unsupported encoding");
return ftranscoder;	return *ftranscoder;
}	}

const char Charset::transcode_cstr(xmlChar s) {	const char* Charset::transcode_cstr(xmlChar* s) {
if(!s)	if(!s)
return "";	return "";

int inlen=strlen((const char *)s);	int inlen=strlen((const char* )s);
int outlen=inlen+1; // max	int outlen=inlen+1; // max
char out=(char )malloc(outlen*sizeof(char));	char *out=new(PointerFreeGC) char[outlen];

int size;	int error;
if(xmlCharEncodingOutputFunc output=transcoder(0)->output) {	if(xmlCharEncodingOutputFunc output=transcoder(fname).output) {
size=output(	error=output(
(unsigned char*)out, &outlen,	(unsigned char*)out, &outlen,
(const unsigned char*)s, &inlen,	(const unsigned char*)s, &inlen,
transcoder(0)->outputInfo);	transcoder(fname).outputInfo);
} else	} else {
memcpy(out, s, size=inlen);	memcpy(out, s, outlen=inlen);
if(size<0)	error=0;
	}
	if(error<0)
throw Exception(0,	throw Exception(0,
0,	0,
"transcode_cstr failed (%d)", size);	"transcode_cstr failed (%d)", error);

out[size]=0;	out[outlen/surely would be less then on input/]=0;
return out;	return out;
}	}
String& Charset::transcode(xmlChar *s) {	const String& Charset::transcode(xmlChar* s) {
return *NEW String(pool(), transcode_cstr(s));	return new String(transcode_cstr(s), 0/auto-size*/, true);
}	}
const char Charset::transcode_cstr(GdomeDOMString s) {	const char* Charset::transcode_cstr(GdomeDOMString* s) {
return s?transcode_cstr(BAD_CAST s->str):"";	return s?transcode_cstr(BAD_CAST s->str):"";
}	}
String& Charset::transcode(GdomeDOMString *s) {	const String& Charset::transcode(GdomeDOMString* s) {
return *NEW String(pool(), transcode_cstr(s));	return new String(transcode_cstr(s), 0/auto-size*/, true);
}	}

/// @test less memory using -maybe- xmlParserInputBufferCreateMem	/// @test less memory using -maybe- xmlParserInputBufferCreateMem
xmlChar Charset::transcode_buf2xchar(const char buf, size_t buf_size) {	void* Charset::transcode_buf2mchar(transcode_buf_malloc_func malloc_func,
int outlen=buf_size6/max*/+1;	const char* buf, size_t buf_size) {
unsigned char out=(unsigned char)malloc(outlen*sizeof(unsigned char));	unsigned char* out;
	int outlen;
int size;	int error;
if(xmlCharEncodingInputFunc input=transcoder(0)->input) {	if(xmlCharEncodingInputFunc input=transcoder(fname).input) {
size=input(	outlen=buf_size6/max*/;
	out=(unsigned char*)malloc_func(outlen+1);
	error=input(
out, &outlen,	out, &outlen,
(const unsigned char )buf, (int )&buf_size,	(const unsigned char )buf, (int )&buf_size,
transcoder(0)->inputInfo);	transcoder(fname).inputInfo);
} else	} else {
memcpy(out, buf, size=buf_size);	outlen=buf_size;
	out=(unsigned char*)malloc_func(outlen+1);
	memcpy(out, buf, outlen);
	error=0;
	}

if(size<0)	if(error<0)
throw Exception(0,	throw Exception(0,
0,	0,
"transcode_buf failed (%d)", size);	"transcode_buf failed (%d)", error);

	out[outlen/surely would be less then on input/]=0;
	return out;
	}

out[size]=0;	xmlChar* Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
return (xmlChar *)out;	return static_cast<xmlChar*>(transcode_buf2mchar(xmlMalloc, buf, buf_size));
}	}
GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char *buf, size_t buf_size) {	static void* g_malloc_wrapper(size_t size) {
return GdomeDOMString_auto_ptr((gchar*)transcode_buf2xchar(buf, buf_size));	return g_malloc(size);
	}
	gchar* Charset::transcode_buf2gchar(const char* buf, size_t buf_size) {
	return static_cast<gchar*>(transcode_buf2mchar(g_malloc_wrapper, buf, buf_size));
	}
	GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char* buf, size_t buf_size) {
	return GdomeDOMString_auto_ptr(transcode_buf2gchar(buf, buf_size));
}	}
GdomeDOMString_auto_ptr Charset::transcode(const String& s) {	GdomeDOMString_auto_ptr Charset::transcode(const String& s) {
const char *cstr=s.cstr(String::UL_UNSPECIFIED);	const char* cstr=s.cstr(String::L_UNSPECIFIED);

return transcode_buf2dom(cstr, strlen(cstr));	return transcode_buf2dom(cstr, strlen(cstr));
}	}
	GdomeDOMString_auto_ptr Charset::transcode(const StringBody s) {
	const char* cstr=s.cstr();

	return transcode_buf2dom(cstr, s.length());
	}
#endif	#endif

E-mail:

Removed from v.1.29
changed lines
	Added in v.1.33.2.19.2.12