--- parser3/src/main/pa_charset.C	2012/05/28 10:33:18	1.91
+++ parser3/src/main/pa_charset.C	2015/05/30 22:55:29	1.98
@@ -8,7 +8,10 @@
 #include "pa_charset.h"
 #include "pa_charsets.h"
 
-volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.91 2012/05/28 10:33:18 moko Exp $" IDENT_PA_CHARSET_H;
+// we are using some pcre_internal.h stuff as well
+#include "../lib/pcre/pa_pcre_internal.h"
+
+volatile const char * IDENT_PA_CHARSET_C="$Id: pa_charset.C,v 1.98 2015/05/30 22:55:29 moko Exp $" IDENT_PA_CHARSET_H;
 
 #ifdef XML
 #include "libxml/encoding.h"
@@ -85,6 +88,28 @@ static void element2case(unsigned char f
 	fcc_table[from]=to; fcc_table[to]=from;
 }
 
+inline XMLByte *append_hex_8(XMLByte *dest, unsigned char c, const char* prefix=0) {
+    if(prefix) {
+        strcpy((char *)dest, prefix);
+        dest+=strlen(prefix);
+    }
+    *dest++=hex_digits[c >> 4];
+    *dest++=hex_digits[c & 0x0F];
+    return dest;
+}
+
+inline XMLByte *append_hex_16(XMLByte *dest, unsigned int c, const char* prefix=0) {
+    if(prefix) {
+        strcpy((char *)dest, prefix);
+        dest+=strlen(prefix);
+    }
+    *dest++=hex_digits[(c >> 12) & 0x0F];
+    *dest++=hex_digits[(c >> 8) & 0x0F];
+    *dest++=hex_digits[(c >> 4) & 0x0F];
+    *dest++=hex_digits[(c) & 0x0F];
+    return dest;
+}
+
 // methods
 
 Charset::Charset(Request_charsets* charsets, const String::Body ANAME, const String* afile_spec): 
@@ -100,7 +125,7 @@ Charset::Charset(Request_charsets* chars
 	} else {
 		fisUTF8=true;
 		// grab default onces [for UTF-8 so to be able to make a-z =>A-Z
-		memcpy(pcre_tables, _pcre_default_tables, sizeof(pcre_tables));
+		memcpy(pcre_tables, pa_pcre_default_tables, sizeof(pcre_tables));
 	}
 
 #ifdef XML
@@ -190,8 +215,7 @@ static int sort_cmp_Trans_rec_intCh(cons
 }
 
 void Charset::sort_ToTable() {
-	_qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), 
-		sort_cmp_Trans_rec_intCh);
+	qsort(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), sort_cmp_Trans_rec_intCh);
 	//FILE *f=fopen("c:\\temp\\a", "wb");
 	//fwrite(tables.toTable, tables.toTableSize, sizeof(*tables.toTable), f);
 	//fclose(f);
@@ -571,13 +595,13 @@ size_t Charset::calc_escaped_length(cons
 	if(char_size==1) \
 		if(first_byte){ \
 			if(need_escape(first_byte)) \
-				dest_ptr+=sprintf((char*)dest_ptr, "%%%02X", first_byte); /* %XX */ \
+				dest_ptr=append_hex_8(dest_ptr, first_byte, "%");  /* %XX */ \
 			else \
 				*dest_ptr++=first_byte; /*as is*/ \
 		} else \
 			*dest_ptr++='?'; /* replacement char '?' */ \
 	else \
-		dest_ptr+=sprintf((char*)dest_ptr, "%%u%04X", UTF8_char); // %uXXXX
+		dest_ptr=append_hex_16(dest_ptr, UTF8_char, "%u"); /* %uXXXX */
 
 
 size_t Charset::escape_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
@@ -644,9 +668,10 @@ size_t Charset::calc_JSON_escaped_length
 	size_t dest_length=0;
 
 	for(UTF8_string_iterator i(src, src_length); i.has_next(); ){
-		if(i.getCharSize()==1)
-			dest_length+=need_json_escape(i.getFirstByte()) ? 2 : 1;
-		else
+		if(i.getCharSize()==1){
+			XMLByte first_byte=i.getFirstByte();
+			dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
+		} else
 			dest_length+=6; // \uXXXX
 	}
 
@@ -661,7 +686,7 @@ size_t Charset::calc_JSON_escaped_length
 
 	while(uint char_size=readChar(src, src_end, first_byte, UTF8_char, tables)){
 		if(char_size==1)
-			dest_length+=(!first_byte/*replacement char '?'*/ || !need_json_escape(first_byte))? 1 : 2;
+			dest_length+=need_json_escape(first_byte) ? 2 : (first_byte < 0x20 && first_byte /* 0 replacement char is '?' */) ? 6 : 1;
 		else
 			dest_length+=6; // \uXXXX
 	}
@@ -695,10 +720,11 @@ size_t Charset::calc_JSON_escaped_length
 			case '\b': *dest_ptr++='\\'; *dest_ptr++='b';  break; \
 			case '\f': *dest_ptr++='\\'; *dest_ptr++='f';  break; \
 			case   0 : *dest_ptr++='?'; break; /*replacement char*/ \
-			default  : *dest_ptr++=first_byte; \
+			default  : if(first_byte < 0x20) dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); \
+						else *dest_ptr++=first_byte; \
 		} \
 	else \
-		dest_ptr+=sprintf((char*)dest_ptr, "\\u%04X", UTF8_char); // \uXXXX
+		dest_ptr=append_hex_16(dest_ptr, UTF8_char, "\\u"); // \uXXXX
 
 
 size_t Charset::escape_JSON_UTF8(const XMLByte* src, size_t src_length, XMLByte* dest) {
@@ -861,7 +887,8 @@ static void store_UTF8(XMLCh src, XMLByt
 static void change_case_UTF8(XMLCh src, XMLByte*& outPtr, 
 						const Charset::UTF8CaseTable& table) {
 	store_UTF8(change_case_UTF8(src, table), outPtr);
-};
+}
+
 void change_case_UTF8(const XMLByte* srcData, size_t srcLen,
 					  XMLByte* toFill, size_t toFillLen,
 					  const Charset::UTF8CaseTable& table) {
@@ -1070,7 +1097,7 @@ void Charset::addEncoding(char *name_cst
 			"already allocated %d handlers, no space for new encoding '%s'",
 				MAX_CHARSETS, name_cstr);
 
-	xmlCharEncodingHandler* handler=new(UseGC) xmlCharEncodingHandler;
+	xmlCharEncodingHandler* handler=new xmlCharEncodingHandler;
 	{
 		handler->name=name_cstr;
 		handler->input=inputFuncs[handlers_count]; 
@@ -1267,6 +1294,45 @@ unsigned int lengthUTF8Char(const XMLByt
 	return gUTFBytes[c]+1;
 }
 
+const char *fixUTF8(const char *src){
+	if(src && *src){
+		size_t length=strlen(src);
+
+		int error_offset;
+		if(pa_pcre_valid_utf((unsigned char *)src, length, &error_offset)){
+
+			char *result=(char *)pa_malloc_atomic(length+1);
+			char *dst=result;
+
+			do {
+
+				if(error_offset){
+					strncpy(dst, src, error_offset);
+					dst+=error_offset;
+
+					src+=error_offset;
+					length-=error_offset;
+
+				}
+
+				*dst++='?';
+				src++;
+				length--;
+
+			} while (length && pa_pcre_valid_utf((unsigned char *)src, length, &error_offset));
+
+			if(length){
+				strcpy(dst, src);
+			} else {
+				*dst='\0';
+			}
+
+			return result;
+		}
+	}
+	return src;
+}
+
 bool UTF8_string_iterator::has_next(){
 	fcharSize=readUTF8Char(fsrcPtr, fsrcEnd, ffirstByte, fUTF8Char);
 	return fcharSize!=0;