--- parser3/src/main/pa_common.C	2007/05/23 08:31:47	1.225
+++ parser3/src/main/pa_common.C	2008/06/11 11:38:35	1.234
@@ -26,7 +26,7 @@
  *
  */
 
-static const char * const IDENT_COMMON_C="$Date: 2007/05/23 08:31:47 $"; 
+static const char * const IDENT_COMMON_C="$Date: 2008/06/11 11:38:35 $"; 
 
 #include "pa_common.h"
 #include "pa_exception.h"
@@ -92,11 +92,12 @@ void fix_line_breaks(char *str, size_t&
 }
 
 char* file_read_text(Request_charsets& charsets, 
-		     const String& file_spec, 
-		     bool fail_on_read_problem,
-		     HashStringValue* params/*, HashStringValue* * out_fields*/) {
+			const String& file_spec, 
+			bool fail_on_read_problem,
+			HashStringValue* params,
+			bool transcode_result) {
 	File_read_result file=
-		file_read(charsets, file_spec, true, params, fail_on_read_problem);
+		file_read(charsets, file_spec, true, params, fail_on_read_problem, 0, 0, 0, transcode_result);
 	return file.success?file.str:0;
 }
 
@@ -124,10 +125,10 @@ struct File_read_action_info {
 }; 
 #endif
 static void file_read_action(
-			     struct stat& finfo, 
-			     int f, 
-			     const String& file_spec, const char* /*fname*/, bool as_text, 
-			     void *context) {
+				struct stat& finfo, 
+				int f, 
+				const String& file_spec, const char* /*fname*/, bool as_text, 
+				void *context) {
 	File_read_action_info& info=*static_cast<File_read_action_info *>(context); 
 	size_t to_read_size=info.count;
 	if(!to_read_size)
@@ -155,9 +156,9 @@ static void file_read_action(
 	}
 }
 File_read_result file_read(Request_charsets& charsets, const String& file_spec, 
-			   bool as_text, HashStringValue *params,
-			   bool fail_on_read_problem,
-			   char* buf, size_t offset, size_t count) {
+			bool as_text, HashStringValue *params,
+			bool fail_on_read_problem,
+			char* buf, size_t offset, size_t count, bool transcode_text_result) {
 	File_read_result result={false, 0, 0, 0};
 	if(file_spec.starts_with("http://")) {
 		if(offset || count)
@@ -166,7 +167,7 @@ File_read_result file_read(Request_chars
 				"offset and load options are not supported for HTTP:// file load");
 
 		// fail on read problem
-		File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params);
+		File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params, transcode_text_result);
 		result.success=true;
 		result.str=http.str;
 		result.length=http.length;
@@ -186,7 +187,7 @@ File_read_result file_read(Request_chars
 			"read", file_read_action, &info, 
 			as_text, fail_on_read_problem); 
 
-		if(result.length && as_text && params) {
+		if(result.length && as_text && params && transcode_text_result) {
 			if( Value* vcharset_name=params->get(PA_CHARSET_NAME) ) {
 				Charset asked_charset=::charsets.get(vcharset_name->as_string().
 					change_case(charsets.source(), String::CC_UPPER));
@@ -248,7 +249,7 @@ bool file_read_action_under_lock(const S
 	//   they delay update till open, so we would receive "!^test[" string
 	//   if would do stat, next open.
 	// later: it seems, even this does not help sometimes
-    if((f=open(fname, O_RDONLY|(as_text?_O_TEXT:_O_BINARY)))>=0) {
+	if((f=open(fname, O_RDONLY|(as_text?_O_TEXT:_O_BINARY)))>=0) {
 		try {
 			if(pa_lock_shared_blocking(f)!=0)
 				throw Exception("file.lock", 
@@ -277,7 +278,7 @@ bool file_read_action_under_lock(const S
 
 		pa_unlock(f);close(f); 
 		return true;
-    } else {
+	} else {
 		if(fail_on_read_problem)
 			throw Exception(errno==EACCES?"file.access":errno==ENOENT?"file.missing":0, 
 				&file_spec, 
@@ -468,11 +469,11 @@ bool file_executable(const String& file_
 }
 
 bool file_stat(const String& file_spec, 
-			   size_t& rsize, 
-			   time_t& ratime, 
-			   time_t& rmtime, 
-			   time_t& rctime, 
-			   bool fail_on_read_problem) {
+			size_t& rsize,
+			time_t& ratime,
+			time_t& rmtime,
+			time_t& rctime,
+			bool fail_on_read_problem) {
 	const char* fname=file_spec.cstr(String::L_FILE_SPEC); 
 	struct stat finfo;
 	if(stat(fname, &finfo)!=0)
@@ -491,62 +492,140 @@ bool file_stat(const String& file_spec,
 }
 
 char* getrow(char* *row_ref, char delim) {
-    char* result=*row_ref;
-    if(result) {
+	char* result=*row_ref;
+	if(result) {
 		*row_ref=strchr(result, delim); 
 		if(*row_ref) 
 			*((*row_ref)++)=0; 
 		else if(!*result) 
 			return 0;
-    }
-    return result;
+	}
+	return result;
 }
 
 char* lsplit(char* string, char delim) {
-    if(string) {
+	if(string) {
 		char* v=strchr(string, delim); 
 		if(v) {
 			*v=0;
 			return v+1;
 		}
-    }
-    return 0;
+	}
+	return 0;
 }
 
 char* lsplit(char* *string_ref, char delim) {
-    char* result=*string_ref;
+	char* result=*string_ref;
 	char* next=lsplit(*string_ref, delim); 
-    *string_ref=next;
-    return result;
+	*string_ref=next;
+	return result;
 }
 
 char* rsplit(char* string, char delim) {
-    if(string) {
+	if(string) {
 		char* v=strrchr(string, delim); 
 		if(v) {
 			*v=0;
 			return v+1;
 		}
-    }
-    return NULL;	
+	}
+	return NULL;	
 }
 
-/// @todo less stupid type detection
+
+// format: %[flags][width][.precision]type	http://msdn.microsoft.com/ru-ru/library/56e442dc(en-us,VS.80).aspx
+//		flags: '-', '+', ' ', '#', '0'		http://msdn.microsoft.com/ru-ru/library/8aky45ct(en-us,VS.80).aspx
+//		width, precision: non negative decimal number
+enum FormatType {
+	FormatInvalid,
+	FormatInt,
+	FormatUInt,
+	FormatDouble
+};
+FormatType format_type(char* fmt){
+	enum FormatState {
+		Percent, 
+		Flags, 
+		Width,
+		Precision,
+		Done
+	} state=Percent;
+
+	FormatType result=FormatInvalid;
+
+	char* pos=fmt;
+	while(char c=*(pos++)){
+		switch(state){
+			case Percent:
+				if(c=='%'){
+					state=Flags;
+				} else {
+					return FormatInvalid; // 1st char must be '%' only
+				}
+				break;
+			case Flags:
+				if(strchr("-+ #0", c)!=0){
+					break;
+				}
+				// go to the next step
+			case Width:
+				if(c=='.'){
+					state=Precision;
+					break;
+				}
+				// go to the next step
+			case Precision:
+				if(c>='0' && c<='9'){
+					if(state == Flags) state=Width; // no more flags
+					break;
+				} else if(c=='d' || c=='i'){
+					result=FormatInt;
+				} else if(strchr("feEgG", c)!=0){
+					result=FormatDouble;
+				} else if(strchr("uoxX", c)!=0){
+					result=FormatUInt;
+				} else {
+					return FormatInvalid; // invalid char
+				}
+				state=Done;
+				break;
+			case Done:
+				return FormatInvalid; // no chars allowed after 'type'
+		}
+	}
+	return result;
+}
+
+
 const char* format(double value, char* fmt) {
-	char local_buf[MAX_NUMBER]; 
+	char local_buf[MAX_NUMBER];
 	size_t size;
-	
-	if(fmt)
-		if(strpbrk(fmt, "diouxX"))
-			if(strpbrk(fmt, "ouxX"))
-				size=snprintf(local_buf, sizeof(local_buf), fmt, (uint)value); 
-			else
+
+	if(fmt && strlen(fmt)){
+		switch(format_type(fmt)){
+			case FormatDouble:
+				size=snprintf(local_buf, sizeof(local_buf), fmt, value); 
+				break;
+			case FormatInt:
 				size=snprintf(local_buf, sizeof(local_buf), fmt, (int)value); 
-		else
-			size=snprintf(local_buf, sizeof(local_buf), fmt, value); 
-	else
-		size=snprintf(local_buf, sizeof(local_buf), "%d", (int)value); 
-	
+				break;
+			case FormatUInt:
+				size=snprintf(local_buf, sizeof(local_buf), fmt, (uint)value); 
+				break;
+			case FormatInvalid:
+				throw Exception(PARSER_RUNTIME, 
+					0, 
+					"Incorrect format string '%s' was specified.", fmt);
+		}
+	} else
+		size=snprintf(local_buf, sizeof(local_buf), "%d", (int)value);
+
+	if(size < 0 || size >= MAX_NUMBER-1){ // on win32 we manually reduce max size while printing
+		throw Exception(PARSER_RUNTIME, 
+			0, 
+			"Error occure white executing snprintf with format string '%s'.", fmt);
+	}
+
 	return pa_strdup(local_buf, size);
 }
 
@@ -567,13 +646,16 @@ size_t stdout_write(const void *buf, siz
 #endif
 }
 
+enum EscapeState {
+	EscapeRest, 
+	EscapeFirst, 
+	EscapeSecond,
+	EscapeUnicode
+};
+/*
 char* unescape_chars(const char* cp, int len) {
 	char* s=new(PointerFreeGC) char[len + 1]; 
-	enum EscapeState {
-		EscapeRest, 
-		EscapeFirst, 
-		EscapeSecond
-	} escapeState=EscapeRest;
+	EscapeState escapeState=EscapeRest;
 	uint escapedValue=0;
 	int srcPos=0;
 	int dstPos=0;
@@ -581,29 +663,96 @@ char* unescape_chars(const char* cp, int
 		uchar ch=(uchar)cp[srcPos]; 
 		switch(escapeState) {
 			case EscapeRest:
-			if(ch=='%') {
-				escapeState=EscapeFirst;
-			} else if(ch=='+') {
-				s[dstPos++]=' '; 
-			} else {
-				s[dstPos++]=ch;	
-			}
-			break;
+				if(ch=='%') {
+					escapeState=EscapeFirst;
+				} else if(ch=='+'){
+					s[dstPos++]=' '; 
+				} else {
+					s[dstPos++]=ch;	
+				}
+				break;
 			case EscapeFirst:
-			escapedValue=hex_value[ch] << 4;
-			escapeState=EscapeSecond;
-			break;
+				escapedValue=hex_value[ch] << 4;
+				escapeState=EscapeSecond;
+				break;
 			case EscapeSecond:
-			escapedValue +=hex_value[ch]; 
-			s[dstPos++]=(char)escapedValue;
-			escapeState=EscapeRest;
-			break;
+				escapedValue +=hex_value[ch]; 
+				s[dstPos++]=(char)escapedValue;
+				escapeState=EscapeRest;
+				break;
 		}
 		srcPos++; 
 	}
 	s[dstPos]=0;
 	return s;
 }
+*/
+
+char* unescape_chars(const char* cp, int len, Charset* charset){
+	char* s=new(PointerFreeGC) char[len + 1]; // enough (%uXXXX==6 bytes, max utf-8 char length==6 bytes)
+	char* dst=s;
+	EscapeState escapeState=EscapeRest;
+	uint escapedValue=0;
+	int srcPos=0;
+	short int jsCnt=0;
+	while(srcPos < len) {
+		uchar c=(uchar)cp[srcPos]; 
+		if(c=='%'){
+			escapeState=EscapeFirst;
+		} else {
+			switch(escapeState) {
+				case EscapeRest:
+					if(c=='+'){
+						*dst++=' ';
+					} else {
+						*dst++=c;
+					}
+					break;
+				case EscapeFirst:
+					if(charset && c=='u'){
+						// escaped unicode value: %u0430
+						jsCnt=0;
+						escapedValue=0;
+						escapeState=EscapeUnicode;
+					} else {
+						if(isxdigit(c)){
+							escapedValue=hex_value[c] << 4;
+							escapeState=EscapeSecond;
+						} else {
+							*dst++=c;
+							escapeState=EscapeRest;
+						}
+					}
+					break;
+				case EscapeSecond:
+					if(isxdigit(c)){
+						escapedValue+=hex_value[c]; 
+						*dst++=(char)escapedValue;
+					}
+					escapeState=EscapeRest;
+					break;
+				case EscapeUnicode:
+					if(isxdigit(c)){
+						escapedValue=(escapedValue << 4) + hex_value[c];
+						if(++jsCnt==4){
+							// transcode utf8 char to client charset (we can lost some chars here)
+							charset->store_Char((XMLByte*&)dst, (XMLCh)escapedValue, '?');
+							escapeState=EscapeRest;
+						}
+					} else {
+						// not full unicode value
+						escapeState=EscapeRest;
+					}
+					break;
+			}
+		}
+
+		srcPos++;
+	}
+
+	*dst=0; // zero-termination
+	return s;
+}
 
 #ifdef WIN32
 void back_slashes_to_slashes(char* s) {
@@ -622,55 +771,145 @@ void slashes_to_back_slashes(char* s) {
 */
 #endif
 
-bool StrEqNc(const char* s1, const char* s2, bool strict) {
+bool StrStartFromNC(const char* str, const char* substr, bool equal){
 	while(true) {
-		if(!(*s1)) {
-			if(!(*s2))
+		if(!(*substr)){
+			if(!(*str))
 				return true;
 			else
-				return !strict;
-		} else if(!(*s2))
-			return !strict;
-		if(isalpha((unsigned char)*s1)) {
-			if(tolower((unsigned char)*s1) !=tolower((unsigned char)*s2))
+				return !equal;
+		}
+		if(!(*str))
+			return false;
+		if(isalpha((unsigned char)*str)) {
+			if(tolower((unsigned char)*str)!=tolower((unsigned char)*substr))
 				return false;
-		} else if((*s1) !=(*s2))
+		} else if((*str) != (*substr))
 			return false;
-		s1++; 
-		s2++; 
+		str++; 
+		substr++; 
 	}
 }
 
+size_t strpos(const char *str, const char *substr) {
+	const char *p = strstr(str, substr);
+	return (p==0)?STRING_NOT_FOUND:p-str;
+}
+
+// content-type: xxx; charset=WE-NEED-THIS
+// content-type: xxx; charset="WE-NEED-THIS"
+// content-type: xxx; charset="WE-NEED-THIS";
+Charset* detect_charset(const char* content_type){
+	if(content_type){
+		size_t len=strlen(content_type);
+		char* CONTENT_TYPE=new(PointerFreeGC) char[len+1];
+		memcpy(CONTENT_TYPE, content_type, len);
+		for(char *p=CONTENT_TYPE; *p; p++)
+			*p=(char)toupper((unsigned char)*p);
+
+		if(const char* begin=strstr(CONTENT_TYPE, "CHARSET=")){
+			begin+=8; // skip "CHARSET="
+			char* end=0;
+			if(*begin && (*begin=='"' || *begin =='\'')){
+				char quote=*begin;
+				begin++;
+				end=(char*)strchr(begin, quote);
+			}
+			if(!end)
+				end=(char*)strchr(begin, ';');
+
+			size_t len;
+			if(end){
+				*end=0; // terminator
+				len=end-begin;
+			} else {
+				len=strlen(begin);
+			}
+
+			String::Body NAME=String::Body(begin, len);
+			return &charsets.get(NAME);
+		}
+	}
+	return 0;
+}
+
+Charset* detect_charset(Charset& source_charset, const String& content_type){
+	const String& CONTENT_TYPE=content_type.change_case(source_charset, String::CC_UPPER);
+	size_t begin=CONTENT_TYPE.pos("CHARSET=");
+	if(begin!=STRING_NOT_FOUND) {
+		begin+=8; // skip "CHARSET="
+		size_t end=STRING_NOT_FOUND;
+
+		if(CONTENT_TYPE.pos('"', begin)==begin){
+			begin++;
+			end=CONTENT_TYPE.pos('"', begin);
+		} else if(CONTENT_TYPE.pos('\'', begin)==begin){
+			begin++;
+			end=CONTENT_TYPE.pos('\'', begin);
+		}
+
+		if(end==STRING_NOT_FOUND)
+			end=CONTENT_TYPE.pos(';', begin);
+
+		if(end==STRING_NOT_FOUND)
+			end=CONTENT_TYPE.length();
+
+		const String::Body NAME=CONTENT_TYPE.mid(begin, end);
+		return &charsets.get(NAME);
+	}
+	return 0;
+}
+
+
 static bool isLeap(int year) {
-    return !(
-             (year % 4) || ((year % 400) && !(year % 100))
-            ); 
+	return !(
+				(year % 4) || ((year % 400) && !(year % 100))
+			); 
 }
 
 int getMonthDays(int year, int month) {
 	static int monthDays[]={
-        31, 
-        28, 
-        31, 
-        30, 
-        31, 
-        30, 
-        31, 
-        31, 
-        30, 
-        31, 
-        30, 
-        31
-    }; 
-	return (month == 2 && isLeap(year)) ? 29 : monthDays[month]; 
-}
-
-void remove_crlf(char* start, char* end) {
-	for(char* p=start; p<end; p++)
-		switch(*p) {
-			case '\n': *p='|';  break;
-			case '\r': *p=' ';  break;
+		31, 
+		28, 
+		31, 
+		30, 
+		31, 
+		30, 
+		31, 
+		31, 
+		30, 
+		31, 
+		30, 
+		31
+	}; 
+	return (month == 1 /* january -- 0 */ && isLeap(year)) ? 29 : monthDays[month]; 
+}
+
+int remove_crlf(char* start, char* end) {
+	char* from=start;
+	char* to=start;
+	bool skip=false;
+	while(from < end){
+		switch(*from){
+			case '\n':
+			case '\r':
+			case '\t':
+			case ' ':
+				if(!skip){
+					*to=' ';
+					to++;
+					skip=true;
+				}
+				break;
+			default:
+				if(from != to)
+					*to=*from;
+				to++;
+				skip=false;
 		}
+		from++;
+	}
+	return to-start;
 }
 
 
@@ -1001,10 +1240,10 @@ char* pa_base64_encode(const String& fil
 
 
 static void file_base64_file_action(
-			     struct stat& finfo, 
-			     int f, 
-			     const String&, const char* /*fname*/, bool, 
-			     void *context) {
+				struct stat& finfo, 
+				int f, 
+				const String&, const char* /*fname*/, bool, 
+				void *context) {
 
 	if(finfo.st_size) { 
 		File_base64_action_info& info=*static_cast<File_base64_action_info *>(context);
@@ -1066,10 +1305,10 @@ const unsigned long pa_crc32(const Strin
 }
 
 static void file_crc32_file_action(
-			     struct stat& finfo, 
-			     int f, 
-			     const String&, const char* /*fname*/, bool, 
-			     void *context)
+				struct stat& finfo, 
+				int f, 
+				const String&, const char* /*fname*/, bool, 
+				void *context)
 {
 	unsigned long& crc32=*static_cast<unsigned long *>(context);
 	if(finfo.st_size) {