--- parser3/src/main/untaint.C	2001/03/12 21:54:20	1.2
+++ parser3/src/main/untaint.C	2003/01/29 11:55:22	1.115.2.4
@@ -1,120 +1,541 @@
-/*
-	Parser
-	Copyright (c) 2001 ArtLebedev Group (http://www.artlebedev.com)
-	Author: Alexander Petrosyan <paf@design.ru> (http://design.ru/paf)
+/** @file
+	Parser: String class part: untaint mechanizm.
 
-	$Id: untaint.C,v 1.2 2001/03/12 21:54:20 paf Exp $
+	Copyright(c) 2001, 2003 ArtLebedev Group (http://www.artlebedev.com)
+	Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)
 */
 
-#include <string.h>
+static const char* IDENT_UNTAINT_C="$Date: 2003/01/29 11:55:22 $";
 
 #include "pa_pool.h"
+#include "pa_value_includes.h"
 #include "pa_string.h"
 #include "pa_hash.h"
 #include "pa_exception.h"
+#include "pa_table.h"
+#include "pa_globals.h"
+#include "pa_sql_connection.h"
+#include "pa_dictionary.h"
+#include "pa_common.h"
+#include "pa_charset.h"
+#include "pa_request_charsets.h"
 
-#define escape(cases) \
+#define escape(action) \
 	{ \
-		const char *ptr=row->item.ptr; \
-		int size=row->item.size; \
-		for (;*ptr;ptr++) \
-			switch(*ptr) { \
-				cases \
-				default: *copy_here++=*ptr; break; \
-			} \
+		const char *src=fragment->ptr; \
+		for(int size=fragment->size; size--; src++) \
+			action \
 	}
-#define escape_value(a, c)  case a: *copy_here++=c; break;
-#define escape_subst(a, b, bsize)  \
-	case a: \
-		{ \
-			strncpy(copy_here, b, bsize); \
-			copy_here+=bsize; \
-		} \
-		break;
+#define _default  default: *dest++=*src; break
+#define encode(need_encode_func, prefix)  \
+			if(need_encode_func(*src)) { \
+				static const char *hex="0123456789ABCDEF"; \
+				char chunk[3]={prefix}; \
+				chunk[1]=hex[((unsigned char)*src)/0x10]; \
+				chunk[2]=hex[((unsigned char)*src)%0x10]; \
+				memcpy(dest, chunk, 3);  dest+=3; \
+			} else \
+				*dest++=*src; \
+			break
+#define to_char(c)  *dest++=c
+#define to_string(b, bsize)  \
+		memcpy(dest, b, bsize); \
+		dest+=bsize; \
+
+inline bool need_file_encode(unsigned char c){
+	// russian letters and space ENABLED
+	// encoding only these...
+	return strchr(
+			  "*?'\"<>|"
+#ifndef WIN32
+			  ":\\~"
+#endif
+			  , c)!=0;
+}
+inline bool need_uri_encode(unsigned char c){
+    if((c>='0') &&(c<='9') ||(c>='A') &&(c<='Z') ||(c>='a') &&(c<='z')) 
+		return false;
+
+    return !strchr("_-./", c);
+}
+inline bool need_http_header_encode(unsigned char c){
+    if(strchr(" , :", c))
+		return false;
+
+	return need_uri_encode(c);
+}
+
+//
+
+static const char * String_Untaint_lang_name[]={
+	"U", ///< zero value handy for hash lookup @see untaint_lang_name2enum
+	"C", ///< clean
+	"T",  ///< tainted, untaint language as assigned later 
+	// untaint languages. assigned by ^untaint[lang]{...}
+	"P",
+		/**<
+			leave language built into string being appended.
+			just a flag, that value not stored
+		*/
+	"A",     ///< leave all characters intact
+	"F", ///< file specification
+	"H",    ///< ext in HTTP response header
+	"M",    ///< text in mail header
+	"URI",       ///< text in uri
+	"T",     ///< ^table:set body
+	"SQL",       ///< ^table:sql body
+	"JS",        ///< JavaScript code
+	"XML",		///< ^dom:set xml
+	"HTML"      ///< HTML code (for editing)
+};
+
 
 // String
 
-char *String::cstr() const {
-	char *result=(char *)malloc(size()*UNTAINT_TIMES_BIGGER+1);
+/*
+
+HTTP-header    = field-name ":" [ field-value ] CRLF
+
+       field-name     = token
+       field-value    = *( field-content | LWS )
+
+       field-content  = <the OCTETs making up the field-value
+                        and consisting of either *TEXT or combinations
+                        of token, tspecials, and quoted-string>
+
+
+
+word           = token | quoted-string
+
+token          = 1*<any CHAR except CTLs or tspecials>
+
+
+
+tspecials      = "(" | ")" | "<" | ">" | "@"
+                      | "," | ";" | ":" | "\" | <">
+                      | "/" | "[" | "]" | "?" | "="
+                      | "{" | "}" | SP | HT
+
+SP             = <US-ASCII SP, space (32)>
+HT             = <US-ASCII HT, horizontal-tab (9)>
+
+LWS            = [CRLF] 1*( SP | HT )
+TEXT           = <any OCTET except CTLs,
+                        but including LWS>
+
+quoted-pair    = "\" CHAR
+
+  if(strchr("()<>@,;:\\\"/[]?={} \t", *ptr))
+*/
+inline bool need_quote_http_header(const char *ptr, size_t size) {
+	for(; size--; ptr++)
+		if(strchr(";\\\"= \t" /* excluded ()<>@, :/ ? []{} */, *ptr))
+			return true;
+	return false;
+}
+
+//#include "pa_sapi.h"
+/** 
+	appends to other String,
+	marking all tainted pieces of it with @a lang.
+	or marking ALL pieces of it with a @a lang when @a forced to,
+	and propagating OPTIMIZE language bit.
+*/
+String& String::append_to(String& dest, uchar lang, bool forced) const {
+	if(is_empty())
+		return dest;
+
+	// without language-chage, small[fitting dest tail] string?
+	if(lang==UL_PASS_APPENDED && fused<=(dest.fallocated-dest.fused)) {
+		// felements[fused...]=src.felements[0..fused], fused+=src.fused
+		memcpy(&dest.felements[dest.fused], felements, sizeof(felements[0])*fused);
+		dest.fused+=fused;
+		return dest;
+	}
+
+	// manually unrolled code to avoid do{if(const)} constructs
+	if(forced) 
+		STRING_FOREACH_FRAGMENT(
+			dest.APPEND(fragment->ptr, fragment->size, 
+				lang, //forcing passed lang
+				fragment->origin.file, fragment->origin.line);
+		)
+	else if(lang==UL_PASS_APPENDED) 
+		STRING_FOREACH_FRAGMENT(
+			dest.APPEND(fragment->ptr, fragment->size, 
+				fragment->lang, // passing item's lang
+				fragment->origin.file, fragment->origin.line);
+		)
+	else if(lang&UL_OPTIMIZE_BIT) // main idea here
+		// tainted piece would get OPTIMIZED bit from 'lang'
+		// clean piece would be marked OPTIMIZED manually
+		// pieces with determined languages [not tainted|clean] would retain theirs langs
+		STRING_FOREACH_FRAGMENT(
+			dest.APPEND(fragment->ptr, fragment->size, 
+				fragment->lang==UL_TAINTED?lang:(
+					fragment->lang==UL_CLEAN?UL_CLEAN|UL_OPTIMIZE_BIT: // ORing with OPTIMIZED flag
+						fragment->lang
+				), 
+				fragment->origin.file, fragment->origin.line);
+		)
+	else
+		STRING_FOREACH_FRAGMENT(
+			dest.APPEND(fragment->ptr, fragment->size, 
+				fragment->lang==UL_TAINTED?lang:fragment->lang,
+				fragment->origin.file, fragment->origin.line);
+		);
 
-	char *copy_here=result;
-	const Chunk *chunk=&head; 
-	// TODO: оптимизировать whitespaces для всех, кроме 'html'
-	do {
-		const Chunk::Row *row=chunk->rows;
-		for(int i=0; i<chunk->count; i++) {
-			if(row==append_here)
-				goto break2;
-
-			// WARNING:
-			//	string can grow only UNTAINT_TIMES_BIGGER
-			switch(row->item.lang) {
-			case NO:
-				// clean piece
-			case YES:
-				// tainted piece, but undefined untaint language
-				// for VString.get_double of tainted values
-				// for ^process{body} evaluation
-			case AS_IS:
-				// tainted, untaint language: as-is
-				memcpy(copy_here, row->item.ptr, row->item.size); 
-				copy_here+=row->item.size;
-				break;
-			case TABLE: 
-				escape(
-					escape_value('\t', ' ')
-					escape_value('\n', ' ')
-				);
-				break;
-			case SQL:
-				// tainted, untaint language: sql
-				// TODO: зависимость от sql сервера
-				memset(copy_here, '?', row->item.size); 
-				copy_here+=row->item.size;
-				break;
-			case JS:
-				escape(
-					escape_subst('"', "\\\"", 2)
-					escape_subst('\'', "\\'", 2)
-					escape_subst('\n', "\\n", 2)
-					escape_subst('\r', "\\r", 2)
-					escape_subst('\\', "\\\\", 2)
-					escape_subst('я', "\\я", 2)
-				);
-				break;
-			case HTML:
-				escape(
-					escape_subst('&', "&amp;", 5) // BEFORE consequent relpaces yelding '&'
-					escape_subst('>', "&gt;", 4)
-					escape_subst('<', "&lt;",4)
-					escape_subst('"', "&quot;",6)
-					escape_value('\t', ' ')
-					//TODO: XSLT escape_subst('\'', "&apos;", 6)
-				);
-				break;
-			case HTML_TYPO: 
-				// tainted, untaint language: html-typo
-				escape(
-					escape_subst('&', "&amp;", 5) // BEFORE consequent relpaces yelding '&'
-					escape_subst('>', "&gt;", 4)
-					escape_subst('<', "&lt;",4)
-					escape_subst('"', "&quot;",6)
-					escape_value('\t', ' ')
-					//TODO: $MAIN:html-type table replace, max length(b)==UNTAINT_TIMES_BIGGER*length(a)
-				);
-				break;
-			default:
-				THROW(0,0,
-					this,
-					"unknown untaint language #%d of %d piece", 
-						static_cast<int>(row->item.lang),
-						i);
+	return dest;
+}
+
+size_t String::cstr_bufsize(Untaint_lang lang,
+							SQL_Connection *connection,
+							const Request_charsets *charsets) const {
+	size_t dest=1; // for terminating 0
+	STRING_FOREACH_FRAGMENT(
+		uchar to_lang=lang==UL_UNSPECIFIED?fragment->lang:lang;
+
+		switch(to_lang & ~UL_OPTIMIZE_BIT) {
+		case UL_CLEAN:
+		case UL_TAINTED:
+		case UL_AS_IS:
+			// clean piece
+
+			// tainted piece, but undefined untaint language
+			// for VString.as_double of tainted values
+			// for ^process{body} evaluation
+
+			// tainted, untaint language: as-is
+			dest+=fragment->size;
+			break;
+		case UL_FILE_SPEC:
+			// tainted, untaint language: file [name]
+			dest+=fragment->size*3/* worst: Z->%XX */;
+			break;
+		case UL_URI:
+			// tainted, untaint language: uri
+			dest+=fragment->size*6*3/* worst utf8 x worst Z->%XX */;
+			break;
+		case UL_HTTP_HEADER:
+			// tainted, untaint language: http-field-content-text
+			dest+=fragment->size*3/* worst: Z->%XX */;
+			break;
+		case UL_MAIL_HEADER:
+			// tainted, untaint language: mail-header
+			if(charsets) {
+				int parts_count=1;
+				const char *prev=fragment->ptr;
+				size_t size=fragment->size;
+				while(const char *comma_at=(const char*)memchr(prev, ',', size)) {
+					parts_count++;
+					size-=comma_at-prev;
+					prev=comma_at+1;
+				}
+				// Subject: Re: parser3: =?koi8-r?Q?=D3=C5=CD=C9=CE=C1=D2?=
+				dest+=
+					fragment->size*3+
+					parts_count*(charsets->mail().name()->size()+MAX_STRING/* worst: =?charset?Q?=%XX?= */);
+			} else
+				dest+=fragment->size;
+			break;
+		case UL_TABLE: 
+			// tainted, untaint language: table
+			dest+=fragment->size;
+			break;
+		case UL_SQL:
+			// tainted, untaint language: sql
+			if(connection)
+				dest+=connection->quote(0, fragment->ptr, fragment->size);
+			break;
+		case UL_JS:
+			escape(switch(*src) {
+				case '"': case '\'': case '\n': case '\\': case '\xFF':
+					dest+=2;  break;
+				default: 
+					dest++;  break;
+			});
+			break;
+		case UL_XML:
+			escape(switch(*src) {
+				case '&': case '>': case '<': case '"': case '\'': 
+					dest+= 6;  break;
+				default: 
+					dest++;  break;
+			});
+			break;
+		case UL_HTML:
+			escape(switch(*src) {
+				case '&': 
+				case '>': 
+				case '<': 
+				case '"': 
+					dest+=6;  break;
+				default: 
+					dest++;  break;
+			});
+			break;
+		}
+	);
+	return dest;
+}
+
+/** http://www.ietf.org/rfc/rfc2047.txt
+RFC
+(3) As a replacement for a 'word' entity within a 'phrase', for example,
+    one that precedes an address in a From, To, or Cc header.  The ABNF
+    definition for 'phrase' from RFC 822 thus becomes:
+
+    phrase = 1*( encoded-word / word )
+
+    In this case the set of characters that may be used in a "Q"-encoded
+    'encoded-word' is restricted to: <upper and lower case ASCII
+    letters, decimal digits, "!", "*", "+", "-", "/", "=", and "_"
+    (underscore, ASCII 95.)>.  An 'encoded-word' that appears within a
+    'phrase' MUST be separated from any adjacent 'word', 'text' or
+    'special' by 'linear-white-space'.
+...
+   (2) The 8-bit hexadecimal value 20 (e.g., ISO-8859-1 SPACE) may be
+       represented as "_" (underscore, ASCII 95.).  (This character may
+       not pass through some internetwork mail gateways, but its use
+       will greatly enhance readability of "Q" encoded data with mail
+       readers that do not support this encoding.)  Note that the "_"
+       always represents hexadecimal 20, even if the SPACE character
+       occupies a different code position in the character set in use.
+
+	paf: obviously, 
+		without "=", or one could not differ "=E0" and "russian letter a"
+		and without "_", or in would mean 0x20
+*/
+static bool mail_header_char_valid_within_Qencoded(char c) {
+	return c>='A' && c<='Z'
+		|| c>='a' && c<='Z'
+		|| c>='0' && c<='9'
+		|| strchr("!*+-/", c);
+}
+char *String::store_to(char *dest, Untaint_lang lang, 
+					   SQL_Connection *connection,
+					   const Request_charsets *charsets) const {
+	// WARNING:
+	//	 before any changes check cstr_bufsize first!!!
+	bool whitespace=true;
+//	STRING_FOREACH_FRAGMENT(
+	element_type *last = felements+fused; element_type *fragment = felements; for(; fragment<last; fragment++) { 
+
+		uchar to_lang=lang==UL_UNSPECIFIED?fragment->lang:lang;
+
+		char *start=dest;
+
+		switch(to_lang & ~UL_OPTIMIZE_BIT) {
+		case UL_CLEAN:
+		case UL_TAINTED:
+		case UL_AS_IS:
+			// clean piece
+
+			// tainted piece, but undefined untaint language
+			// for VString.as_double of tainted values
+			// for ^process{body} evaluation
+
+			// tainted, untaint language: as-is
+			memcpy(dest, fragment->ptr, fragment->size); 
+			dest+=fragment->size;
+			break;
+		case UL_FILE_SPEC:
+			// tainted, untaint language: file [name]
+			escape(
+				// Macintosh has problems with small Russian letter 'r'
+				if( *src=='\xF0' && charsets && *charsets->source().name()=="windows-1251" ) {
+					// fixing that letter for most common charset
+					to_char('p');
+				} else // fallback to default
+					encode(need_file_encode, '_'); 
+			);
+			break;
+		case UL_URI:
+			// tainted, untaint language: uri
+			const void *client_ptr;
+			size_t client_size;
+			if(charsets) 
+				Charset::transcode(charsets->pool(), 
+					charsets->source(), fragment->ptr, fragment->size,
+					charsets->client(), client_ptr, client_size);
+			else {
+				client_ptr=fragment->ptr; 
+				client_size=fragment->size;
+			}
+
+			{
+				const char *src=(const char *)client_ptr;
+				for(int size=client_size; size--; src++) 
+					encode(need_uri_encode, '%');
 			}
-			row++;
+			break;
+		case UL_HTTP_HEADER:
+			// tainted, untaint language: http-field-content-text
+			escape(
+				encode(need_uri_encode, '%');
+			);
+			break;
+		case UL_MAIL_HEADER:
+			// tainted, untaint language: mail-header
+			// http://www.ietf.org/rfc/rfc2047.txt
+			if(charsets) {
+				const void *mail_ptr;
+				size_t mail_size;
+				Charset::transcode(charsets->pool(), 
+					charsets->source(), fragment->ptr, fragment->size,
+					charsets->mail(), mail_ptr, mail_size);
+				CharPtr charset_name=charsets->mail().name()->cstr();
+
+				// Subject: Re: parser3: =?koi8-r?Q?=D3=C5=CD=C9=CE=C1=D2?=
+				const char *src=(const char *)mail_ptr;
+				bool to_quoted_printable=false;
+
+				bool email=false;
+				for(const char *end=src+mail_size; src<end; src++) {
+					//RFC   + An 'encoded-word' MUST NOT appear in any portion of an 'addr-spec'.
+					if(to_quoted_printable && (*src==',' || *src=='<')) {
+						email=*src=='<';
+						dest+=sprintf(dest, "?=");
+						to_quoted_printable=false;
+					}
+					if(!email && (
+						!to_quoted_printable && (*src & 0x80)  // starting quote-printable-encoding on first 8bit char
+						|| to_quoted_printable && !mail_header_char_valid_within_Qencoded(*src)
+						)) {
+						if(!to_quoted_printable) {
+							dest+=sprintf(dest, "=?%s?Q?", charset_name);
+							to_quoted_printable=true;
+						}
+						//RFC Upper case should be used for hexadecimal digits "A" through "F"
+						if(*src == 0x20) // RFC The 8-bit hexadecimal value 20 (e.g., ISO-8859-1 SPACE) 
+							*dest++='_'; // RFC may be represented as "_" 
+						else
+							dest+=sprintf(dest, "=%02X", *src & 0xFF);
+					} else
+						*dest++=*src;
+					if(*src=='>')
+						email=false;
+				}
+				if(to_quoted_printable) // close
+					dest+=sprintf(dest, "?=");
+			
+			} else {
+				memcpy(dest, fragment->ptr, fragment->size); 
+				dest+=fragment->size;
+			}
+			break;
+		case UL_TABLE: 
+			// tainted, untaint language: table
+			escape(switch(*src) {
+				case '\t': to_char(' ');  break;
+				case '\n': to_char(' ');  break;
+				_default;
+			});
+			break;
+		case UL_SQL:
+			// tainted, untaint language: sql
+			if(connection)
+				dest+=connection->quote(dest, fragment->ptr, fragment->size);
+			else
+				throw Exception(0,
+					ConstStringPtr(this),
+					"untaint in SQL language failed - no connection specified");
+			break;
+		case UL_JS:
+			escape(switch(*src) {
+				case '"': to_string("\\\"", 2);  break;
+				case '\'': to_string("\\'", 2);  break;
+				case '\n': to_string("\\n", 2);  break;
+				case '\\': to_string("\\\\", 2);  break;
+				case '\xFF': to_string("\\\xFF", 2);  break;
+				_default;
+			});
+			break;
+		case UL_XML:
+			escape(switch(*src) {
+				case '&': to_string("&amp;", 5);  break;
+				case '>': to_string("&gt;", 4);  break;
+				case '<': to_string("&lt;", 4);  break;
+				case '"': to_string("&quot;", 6);  break;
+				case '\'': to_string("&apos;", 6);  break;
+				_default;
+			});
+			break;
+		case UL_HTML:
+			escape(switch(*src) {
+				case '&': to_string("&amp;", 5);  break;
+				case '>': to_string("&gt;", 4);  break;
+				case '<': to_string("&lt;", 4);  break;
+				case '"': to_string("&quot;", 6);  break;
+				_default;
+			});
+			break;
+		default:
+			throw Exception(0, 
+				ConstStringPtr(this), 
+				"unknown untaint language #%d", 
+					static_cast<int>(to_lang)); // should never
+			break; // never
 		}
-		chunk=row->link;
-	} while(chunk);
-break2:
-	*copy_here=0;
+
+		if(to_lang & UL_OPTIMIZE_BIT) { 
+			// optimizing whitespace
+			char *stop=dest;  dest=start;
+			for(char *src=start; src<stop; src++)
+				switch(*src) {
+				// of all consequent white space chars leaving only first one
+				case ' ': case '\r': case '\n': case '\t':
+					if(!whitespace) {
+						*dest++=*src;
+						whitespace=true;
+					}
+					break;
+				default:
+					whitespace=false;
+					*dest++=*src;
+					break;
+				};
+		} else // piece without optimization
+			whitespace=false;
+//	);
+	}
+
+	return dest;
+}
+
+char *String::cstr_debug_origins(Pool& pool) const {
+	//_asm int 3;
+	char *result=new(pool) char[size()+fused*MAX_STRING*2];
+	char *dest=result;
+	
+	STRING_FOREACH_FRAGMENT(
+IFNDEF_NO_STRING_ORIGIN(
+		if(fragment->origin.file)
+			dest+=sprintf(dest, ORIGIN_FILE_LINE_FORMAT,
+				fragment->origin.file,
+				1+fragment->origin.line);
+		else
+			dest+=sprintf(dest, "<unknown>");
+);
+		uchar show_lang=fragment->lang & ~UL_OPTIMIZE_BIT;
+		if(show_lang>=sizeof(String_Untaint_lang_name)/sizeof(String_Untaint_lang_name[0]))
+			throw Exception(0, 
+				ConstStringPtr(this), 
+				"unknown untaint language #%d", 
+					static_cast<int>(show_lang)); // sould never
+
+		dest+=sprintf(dest, "#%s%s: ",
+			String_Untaint_lang_name[show_lang],
+			fragment->lang & UL_OPTIMIZE_BIT?".O":"");
+		char *dest_after_origins=dest;
+
+		memcpy(dest, fragment->ptr, fragment->size); 
+		dest+=fragment->size;
+
+		remove_crlf(dest_after_origins, dest);
+		to_char('\n');
+	);
+
+	*dest=0;
 	return result;
 }