--- parser3/src/main/pa_charset.C	2002/03/18 08:16:15	1.22
+++ parser3/src/main/pa_charset.C	2003/02/04 12:08:56	1.33.2.7
@@ -1,12 +1,12 @@
 /** @file
 	Parser: Charset connection implementation.
 
-	Copyright(c) 2001, 2002 ArtLebedev Group (http://www.artlebedev.com)
+	Copyright(c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)
 	Author: Alexander Petrosyan<paf@design.ru>(http://paf.design.ru)
-
-	$Id: pa_charset.C,v 1.22 2002/03/18 08:16:15 paf Exp $
 */
 
+static const char* IDENT_CHARSET_C="$Date: 2003/02/04 12:08:56 $";
+
 #include "pa_charset.h"
 
 #ifdef XML
@@ -15,6 +15,10 @@
 
 // globals
 
+#define CHARSET_UTF8_NAME "UTF-8"
+
+CharsetPtr UTF8_charset(new Charset(StringPtr(new String(CHARSET_UTF8_NAME)), 
+					 StringPtr(0)/*no file=system*/));
 
 // consts
 
@@ -37,7 +41,7 @@ inline void cstr2ctypes(unsigned char *t
 		ctypes_table[c]|=bit;
 	}
 }
-inline unsigned int to_wchar_code(const char *cstr) {
+inline unsigned int to_wchar_code(const char* cstr) {
 	if(!cstr || !*cstr)
 		return 0;
 	if(cstr[1]==0)
@@ -46,7 +50,7 @@ inline unsigned int to_wchar_code(const
 	char *error_pos;
 	return(unsigned int)strtol(cstr, &error_pos, 0);
 }
-inline bool to_bool(const char *cstr) {
+inline bool to_bool(const char* cstr) {
 	return cstr && *cstr!=0;
 }
 static void element2ctypes(unsigned char c, bool belongs,
@@ -74,18 +78,18 @@ static void element2case(unsigned char f
 // methods
 
 extern "C" unsigned char pcre_default_tables[]; // pcre/chartables.c
-Charset::Charset(Pool& apool, const String& aname, const String *request_file_spec): Pooled(apool),
-	fname(aname) {
+Charset::Charset(StringPtr aname, StringPtr afile_spec): 
+	fname(aname),
+	fname_cstr(aname->cstr()) {
 
-	char *name_cstr=fname.cstr();
-    for(char *c=name_cstr; *c; c++)
+	for(char *c=fname_cstr; *c; c++)
         *c = toupper(*c);
 
-	if(request_file_spec) {
+	if(afile_spec) {
 		fisUTF8=false;
-		loadDefinition(*request_file_spec);
+		load_definition(fname);
 #ifdef XML
-		addEncoding(name_cstr);
+		addEncoding(fname_cstr);
 #endif
 	} else {
 		fisUTF8=true;
@@ -94,17 +98,18 @@ Charset::Charset(Pool& apool, const Stri
 	}
 
 #ifdef XML
-	initTranscoder(&aname, name_cstr);
+	initTranscoder(fname, fname_cstr);
 #endif
 }
 
 Charset::~Charset() {
+	// @todonow unregister encodings
 #ifdef XML
 	// not deleting transcoder, that's not our business
 #endif
 }
 
-void Charset::loadDefinition(const String& request_file_spec) {
+void Charset::load_definition(StringPtr afile_spec) {
 	// pcre_tables
 	// lowcase, flipcase, bits digit+word+whitespace, masks
 
@@ -116,7 +121,7 @@ void Charset::loadDefinition(const Strin
 
 	// charset
 	memset(tables.fromTable, 0, sizeof(tables.fromTable));
-	tables.toTable=(Charset_TransRec *)calloc(sizeof(Charset_TransRec)*MAX_CHARSET_UNI_CODES);
+	tables.toTable=(Charset_TransRec *)pool_for_load.calloc(sizeof(Charset_TransRec)*MAX_CHARSET_UNI_CODES);
 	tables.toTableSize=0;
 	// strangly vital
 	tables.toTable[tables.toTableSize].intCh=0;
@@ -124,7 +129,7 @@ void Charset::loadDefinition(const Strin
 	tables.toTableSize++;
 
 	// loading text
-	char *data=file_read_text(pool(), request_file_spec);
+	char *data=file_read_text(pool_for_load, *UTF8_charset, afile_spec);
 
 	// ignore header
 	getrow(&data);
@@ -153,8 +158,8 @@ void Charset::loadDefinition(const Strin
 			case 8:
 				// charset
 				if(tables.toTableSize>MAX_CHARSET_UNI_CODES)
-					throw Exception(0, 0,
-						&request_file_spec,
+					throw Exception("parser.runtime",
+						afile_spec,
 						"charset must contain not more then %d unicode values", MAX_CHARSET_UNI_CODES);
 
 				XMLCh unicode=(XMLCh)to_wchar_code(cell);
@@ -191,7 +196,8 @@ void Charset::sort_ToTable() {
 }
 
 static XMLByte xlatOneTo(const XMLCh toXlat,
-						 const Charset::Tables& tables) {
+						 const Charset::Tables& tables,
+						 XMLByte not_found) {
     unsigned int    lowOfs = 0;
     unsigned int    hiOfs = tables.toTableSize - 1;
     XMLByte         curByte = 0;
@@ -210,7 +216,7 @@ static XMLByte xlatOneTo(const XMLCh toX
 			return tables.toTable[midOfs].extCh;
 	} while(lowOfs+1<hiOfs);
 
-    return '?';
+    return not_found;
 }
 
 void Charset::transcode(Pool& pool,
@@ -360,9 +366,18 @@ static int transcodeToUTF8(
     // Return the characters read
     toFillLen = outPtr - toFill;
 
-	return srcPtr==srcEnd?(int)toFillLen:-1;
+	//return srcPtr==srcEnd?(int)toFillLen:-1;
+/*
+xmlCharEncodingInputFunc
+Returns :
+the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
+number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
+of ocetes consumed.
+*/
+	return 0;
 }
-static size_t transcodeFromUTF8(
+/// @todo digital entites only when xml/html output [at output in html/xml mode, in html part of a letter]
+static int transcodeFromUTF8(
 								const XMLByte *srcData, size_t& srcLen,
 								XMLByte* toFill, size_t& toFillLen,
 								const Charset::Tables& tables) {
@@ -404,8 +419,8 @@ static size_t transcodeFromUTF8(
                      break;
 
             default:
-                throw Exception(0, 0,
-					0,
+                throw Exception(0,
+					Exception::undefined_source,
 					"transcodeFromUTF8 error: wrong trailingBytes value(%d)", trailingBytes);
         }
         tmpVal-=gUTFOffsets[trailingBytes];
@@ -413,11 +428,14 @@ static size_t transcodeFromUTF8(
         //  If it will fit into a single char, then put it in. Otherwise
         //  fail [*encode it as a surrogate pair. If its not valid, use the
         //  replacement char.*]
-        if(!(tmpVal & 0xFFFF0000))
-            *outPtr++= xlatOneTo(tmpVal, tables);
-		else
-			throw Exception(0, 0,
-				0,
+        if(!(tmpVal & 0xFFFF0000)) {
+			if(XMLByte xlat=xlatOneTo(tmpVal, tables, 0))
+				*outPtr++=xlat;
+			else 
+				outPtr+=sprintf((char *)outPtr, "&#%d;", tmpVal); // &#decimal;
+		} else
+			throw Exception(0,
+				Exception::undefined_source,
 				"transcodeFromUTF8 error: too big tmpVal(0x%08X)", tmpVal);
 	}
 
@@ -427,14 +445,22 @@ static size_t transcodeFromUTF8(
     // Return the characters read
     toFillLen = outPtr - toFill;
 
-	return srcPtr==srcEnd?(int)toFillLen:-1;
+	//return srcPtr==srcEnd?(int)toFillLen:-1;
+/*
+xmlCharEncodingOutputFunc
+Returns :
+the number of byte written, or -1 by lack of space, or -2 if the transcoding failed. The value of inlen after return is the
+number of octets consumed as the return value is positive, else unpredictiable. The value of outlen after return is the number
+of ocetes consumed.
+*/
+	return 0;
 }
 
 /// @todo not so memory-hungry with prescan
 void Charset::transcodeToUTF8(Pool& pool,
 								 const void *source_body, size_t source_content_length,
 								 const void *& adest_body, size_t& dest_content_length) const {
-	dest_content_length=source_content_length*6/*so that surly enough*/;
+	dest_content_length=source_content_length*6/*so that surly enough, max utf8 seq len=6*/;
 	XMLByte *dest_body=(XMLByte*)pool.malloc(dest_content_length);
 
 	if(::transcodeToUTF8(
@@ -451,7 +477,7 @@ void Charset::transcodeToUTF8(Pool& pool
 void Charset::transcodeFromUTF8(Pool& pool,
 								   const void *source_body, size_t source_content_length,
 								   const void *& adest_body, size_t& dest_content_length) const {
-	dest_content_length=source_content_length/*surly enough*/;
+	dest_content_length=source_content_length*6/*so that surly enough, "&#255;" has max ratio */;
 	XMLByte *dest_body=(XMLByte*)pool.malloc(dest_content_length);
 
 	if(::transcodeFromUTF8(
@@ -484,7 +510,7 @@ void Charset::transcodeToCharset(Pool& p
 		for(XMLByte* outPtr=dest_body; srcPtr<srcEnd; srcPtr++) {
 			XMLCh curVal = tables.fromTable[*srcPtr];
 			if(curVal) 
-				*outPtr++=xlatOneTo(curVal, dest_charset.tables);
+				*outPtr++=xlatOneTo(curVal, dest_charset.tables, '?');
 			else {
 				// use the replacement character
 				*outPtr++= '?';
@@ -532,78 +558,100 @@ void Charset::addEncoding(char *name_cst
 	xmlRegisterCharEncodingHandler(handler);
 }
 
-void Charset::initTranscoder(const String *source, const char *name_cstr) {
+void Charset::initTranscoder(const String *source, const char* name_cstr) {
 	ftranscoder=xmlFindCharEncodingHandler(name_cstr);
 	transcoder(source); // check right way
 }
 
 xmlCharEncodingHandler *Charset::transcoder(const String *source) {
 	if(!ftranscoder)
-		throw Exception(0, 0,
+		throw Exception("parser.runtime",
 			source,
 			"unsupported encoding");
 	return ftranscoder;
 }
 
-const char *Charset::transcode_cstr(xmlChar *s) {
+const char* Charset::transcode_cstr(xmlChar *s) {
 	if(!s)
 		return "";
 
-	int inlen=strlen((const char *)s);
+	int inlen=strlen((const char* )s);
 	int outlen=inlen+1; // max
 	char *out=(char *)malloc(outlen*sizeof(char));
 	
-	int size;
+	int error;
 	if(xmlCharEncodingOutputFunc output=transcoder(0)->output) {
-		size=output(
+		error=output(
 			(unsigned char*)out, &outlen,
 			(const unsigned char*)s, &inlen,
 			transcoder(0)->outputInfo);
-	} else
-		memcpy(out, s, size=inlen);
-	if(size<0)
-		throw Exception(0, 0,
+	} else {
+		memcpy(out, s, outlen=inlen);
+		error=0;
+	}
+	if(error<0)
+		throw Exception(0,
 			0,
-			"transcode_cstr failed (%d)", size);
+			"transcode_cstr failed (%d)", error);
 
-	out[size]=0;
+	out[outlen/*surely would be less then on input*/]=0;
 	return out;
 }
-String& Charset::transcode(xmlChar *s) { 
-	return *NEW String(pool(), transcode_cstr(s)); 
+String& Charset::transcode(xmlChar *s
+#ifndef NO_STRING_ORIGIN
+		, const String *origin
+#endif
+						   ) { 
+	String& result=*NEW String(pool());
+	result.APPEND_CLEAN(transcode_cstr(s), 0/*auto-size*/, origin->origin().file, origin->origin().line);
+	return result; 
 }
-const char *Charset::transcode_cstr(GdomeDOMString *s) { 
+const char* Charset::transcode_cstr(GdomeDOMString *s) { 
 	return s?transcode_cstr(BAD_CAST s->str):"";
 }
-String& Charset::transcode(GdomeDOMString *s) { 
-	return *NEW String(pool(), transcode_cstr(s)); 
+String& Charset::transcode(GdomeDOMString *s
+#ifndef NO_STRING_ORIGIN
+		, const String *origin
+#endif
+						   ) { 
+	String& result=*NEW String(pool());
+	result.APPEND_CLEAN(transcode_cstr(s), 0/*auto-size*/, origin->origin().file, origin->origin().line);
+	return result; 
 }
 
 /// @test less memory using -maybe- xmlParserInputBufferCreateMem
-GdomeDOMString_auto_ptr Charset::transcode_buf(const char *buf, size_t buf_size) { 
-	int outlen=buf_size*6/*max*/+1;
-	unsigned char *out=(unsigned char*)malloc(outlen*sizeof(unsigned char));
-
-	int size;
+xmlChar *Charset::transcode_buf2xchar(const char* buf, size_t buf_size) {
+	unsigned char *out;
+	int outlen;
+	int error;
 	if(xmlCharEncodingInputFunc input=transcoder(0)->input) {
-		size=input(
+		outlen=buf_size*6/*max*/;
+		out=(unsigned char*)malloc((outlen+1)*sizeof(unsigned char));
+		error=input(
 			out, &outlen,
 			(const unsigned char *)buf,  (int *)&buf_size,
 			transcoder(0)->inputInfo);
-	} else
-		memcpy(out, buf, size=buf_size);
+	} else {
+		outlen=buf_size;
+		out=(unsigned char*)malloc((outlen+1)*sizeof(unsigned char));
+		memcpy(out, buf, outlen);
+		error=0;
+	}
 	
-	if(size<0)
-		throw Exception(0, 0,
+	if(error<0)
+		throw Exception(0,
 			0,
-			"transcode_buf failed (%d)", size);
+			"transcode_buf failed (%d)", error);
 
-	out[size]=0;
-	return GdomeDOMString_auto_ptr((gchar*)out);
+	out[outlen/*surely would be less then on input*/]=0;
+	return (xmlChar *)out;
+}
+GdomeDOMString_auto_ptr Charset::transcode_buf2dom(const char* buf, size_t buf_size) { 
+	return GdomeDOMString_auto_ptr((gchar*)transcode_buf2xchar(buf, buf_size));
 }
 GdomeDOMString_auto_ptr Charset::transcode(const String& s) { 
-	const char *cstr=s.cstr(String::UL_UNSPECIFIED);
+	const char* cstr=s.cstr(String::UL_UNSPECIFIED);
 
-	return transcode_buf(cstr, strlen(cstr)); 
+	return transcode_buf2dom(cstr, strlen(cstr)); 
 }
 #endif