--- parser3/src/main/pa_common.C	2005/11/24 13:50:29	1.204.6.9
+++ parser3/src/main/pa_common.C	2005/11/22 11:31:58	1.211
@@ -24,16 +24,48 @@
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *
-*/
+ */
 
-static const char * const IDENT_COMMON_C="$Date: 2005/11/24 13:50:29 $"; 
+static const char * const IDENT_COMMON_C="$Date: 2005/11/22 11:31:58 $"; 
 
 #include "pa_common.h"
 #include "pa_exception.h"
 #include "pa_hash.h"
 #include "pa_globals.h"
+#include "pa_request_charsets.h"
 #include "pa_charsets.h"
-#include "pa_http.h"
+
+#define PA_HTTP
+
+#ifdef PA_HTTP
+#include "pa_vstring.h"
+#include "pa_vint.h"
+#include "pa_vhash.h"
+#include "pa_vtable.h"
+#include "pa_socks.h"
+
+#ifdef CYGWIN
+#define _GNU_H_WINDOWS32_SOCKETS
+// for PASCAL
+#include <windows.h>
+// SOCKET
+typedef u_int	SOCKET;
+int PASCAL closesocket(SOCKET);
+#else
+#	if defined(WIN32)
+#		include <windows.h>
+#	else
+#		define closesocket close
+#	endif
+#endif
+
+#else
+
+#	if defined(WIN32)
+#		include <windows.h>
+#	endif
+
+#endif
 
 // some maybe-undefined constants
 
@@ -54,6 +86,10 @@ static const char * const IDENT_COMMON_C
 #	endif
 #endif
 
+#	ifndef INADDR_NONE
+#		define INADDR_NONE ((ulong) -1)
+#	endif
+
 // defines for globals
 
 #define FILE_STATUS_NAME  "status"
@@ -62,6 +98,23 @@ static const char * const IDENT_COMMON_C
 
 const String file_status_name(FILE_STATUS_NAME);
 
+// defines
+
+#define HTTP_METHOD_NAME  "method"
+#define HTTP_FORM_NAME  "form"
+#define HTTP_BODY_NAME  "body"
+#define HTTP_TIMEOUT_NAME    "timeout"
+#define HTTP_HEADERS_NAME "headers"
+#define HTTP_ANY_STATUS_NAME "any-status"
+#define HTTP_CHARSET_NAME "charset"
+#define HTTP_TABLES_NAME "tables"
+#define HTTP_USER "user"
+#define HTTP_PASSWORD "password"
+
+// defines
+
+#define DEFAULT_USER_AGENT "parser3"
+
 // functions
 
 void fix_line_breaks(char *str, size_t& length) {
@@ -100,7 +153,7 @@ char* file_read_text(Request_charsets& c
 }
 
 /// these options were handled but not checked elsewhere, now check them
-int pa_get_valid_file_options_count(HashStringValue& options)
+static int get_valid_file_options_count(HashStringValue& options)
 {
 	int result=0;
 	if(options.get(PA_SQL_LIMIT_NAME))
@@ -114,6 +167,640 @@ int pa_get_valid_file_options_count(Hash
 	return result;
 }
 
+//http request stuff
+#ifdef PA_HTTP
+
+#undef CRLF
+#define CRLF "\r\n"
+
+static bool set_addr(struct sockaddr_in *addr, const char* host, const short port){
+    memset(addr, 0, sizeof(*addr)); 
+    addr->sin_family=AF_INET;
+    addr->sin_port=htons(port); 
+    if(host) {
+		ulong packed_ip=inet_addr(host);
+		if(packed_ip!=INADDR_NONE)
+			memcpy(&addr->sin_addr, &packed_ip, sizeof(packed_ip)); 
+		else {
+			struct hostent *hostIP=gethostbyname(host);
+			if(hostIP) 
+				memcpy(&addr->sin_addr, hostIP->h_addr, hostIP->h_length); 
+			else
+				return false;
+		} 
+    } else 
+		addr->sin_addr.s_addr=INADDR_ANY;
+    return true;
+}
+
+size_t guess_content_length(char* buf) {
+	char* ptr;
+	if((ptr=strstr(buf, "Content-Length:"))) // Apache
+		goto found;
+	if((ptr=strstr(buf, "content-length:"))) // Parser 3
+		goto found;
+	if((ptr=strstr(buf, "Content-length:"))) // maybe 1
+		goto found;
+	if((ptr=strstr(buf, "CONTENT-LENGTH:"))) // maybe 2
+		goto found;
+	return 0;
+found:
+	char *error_pos;
+	size_t result=(size_t)strtol(ptr+15/*strlen("CONTENT-LENGTH:")*/, &error_pos, 0);
+	
+	const size_t reasonable_initial_max=0x400*0x400*10 /*10M*/;
+	if(result>reasonable_initial_max) // sanity check
+		return reasonable_initial_max;
+	return 0;//result;
+}
+
+static int http_read_response(char*& response, size_t& response_size, int sock, bool fail_on_status_ne_200) {
+	int result=0;
+	// fetching some to local buffer, guessing on possible content-length	
+	response_size=0x400*20; // initial size if content-length could not be determined	
+	const size_t preview_size=0x400*20;
+	char preview_buf[preview_size+1/*terminator*/];  // 20K buffer to preview headers
+	ssize_t received_size=recv(sock, preview_buf, preview_size, 0); 
+	if(received_size==0)
+		goto done;
+	if(received_size<0) {
+		if(int no=pa_socks_errno())
+			throw Exception("http.timeout", 
+				0, 
+				"error receiving response header: %s (%d)", pa_socks_strerr(no), no); 
+		goto done;
+	}
+	// detecting response_size
+	{
+		preview_buf[received_size]=0; // terminator
+		if(size_t content_length=guess_content_length(preview_buf))
+			response_size=preview_size+content_length; // a little more than needed, will adjust response_size by actual received size later
+	}
+
+	// [gcc is happier this way, see goto above]
+	{
+		// allocating initial buf
+		response=(char*)pa_malloc_atomic(response_size+1/*terminator*/); // just setting memory block type
+		char* ptr=response;
+		size_t todo_size=response_size;
+		// coping part of already received body
+		memcpy(ptr, preview_buf, received_size);
+		ptr+=received_size;
+		todo_size-=received_size;		
+
+		// we use terminator byte for two purposes here:
+		// 1. we return there zero always, not knowing: maybe they would want to create String form $file.body?
+		//     invariant: all Strings should have zero-terminated buffers
+		// 2. we use that out-of-size byte to detect if our content-length guess was wrong
+		//    when recv gets more than we expected
+		//    a) we know that the content-length guess was wrong
+		//    b) we have space to put the first byte of extra data
+		//    c) we use less code to detect normal situation: on last while-cycle recv expected to just return 0
+		while(true) {
+			received_size=recv(sock, ptr, todo_size+1/*there is always a place for terminator*/, 0); 
+			if(received_size==0) {
+				response_size-=todo_size; // in case we received less than expected, cut down the reported size
+				break;
+			}
+			if(received_size<0) {
+				if(int no=pa_socks_errno())
+					throw Exception("http.timeout", 
+						0, 
+						"error receiving response body: %s (%d)", pa_socks_strerr(no), no); 
+				break;
+			}
+			// they've touched the terminator?
+			if((size_t)received_size>todo_size)
+			{
+				// that means that our guessed response_size was not big enough
+				const size_t grow_chunk_size=0x400*0x400; // 1M
+				response_size+=grow_chunk_size;
+				size_t ptr_offset=ptr-response;
+				response=(char*)pa_realloc(response, response_size+1/*terminator*/);
+				ptr=response+ptr_offset;
+				todo_size+=grow_chunk_size;
+			}
+			// can't do this before realloc: we need <todo_size check
+			ptr+=received_size;
+			todo_size-=received_size;
+
+			char* EOLat=0;
+			if(!result && (EOLat=strstr(response, "\n"))) { // checking status in first response
+				const String status_line(pa_strdup(response, EOLat-response));
+				ArrayString astatus; 
+				size_t pos_after=0;
+				status_line.split(astatus, pos_after, " "); 
+				const String& status_code=*astatus.get(astatus.count()>1?1:0);
+				result=status_code.as_int(); 
+
+				if(fail_on_status_ne_200 && result!=200)
+					throw Exception("http.status",
+						&status_code,
+						"invalid HTTP response status");
+			}
+		}
+	}
+done:
+	if(result)
+	{
+		response[response_size]=0;
+		return result;
+	}
+	else
+		throw Exception("http.response",
+			0,
+			"bad response from host - no status found (size=%u)", response_size); 
+}
+
+/* ********************** request *************************** */
+
+#if defined(SIGALRM) && defined(HAVE_SIGSETJMP) && defined(HAVE_SIGLONGJMP)
+#	define PA_USE_ALARM
+#endif
+
+#ifdef PA_USE_ALARM
+static sigjmp_buf timeout_env;
+static void timeout_handler(int /*sig*/){
+    siglongjmp(timeout_env, 1); 
+}
+#endif
+
+static int http_request(char*& response, size_t& response_size,
+			const char* host, short port, 
+			const char* request, 
+			int timeout_secs,
+			bool fail_on_status_ne_200) {
+	if(!host)
+		throw Exception("http.host", 
+			0, 
+			"zero hostname");  //never
+
+	volatile // to prevent makeing it register variable, because it will be clobbered by longjmp [thanks gcc warning]
+		int sock=-1;
+#ifdef PA_USE_ALARM
+	signal(SIGALRM, timeout_handler); 
+#endif
+#ifdef PA_USE_ALARM
+	if(sigsetjmp(timeout_env, 1)) {
+		// stupid gcc [2.95.4] generated bad code
+		// which failed to handle sigsetjmp+throw: crashed inside of pre-throw code.
+		// rewritten simplier [athough duplicating closesocket code]
+		if(sock>=0) 
+			closesocket(sock); 
+		throw Exception("http.timeout", 
+			0, 
+			"timeout occured while retrieving document"); 
+		return 0; // never
+	} else {
+		alarm(timeout_secs); 
+#endif
+		try {
+			int result;
+			struct sockaddr_in dest;
+		
+			if(!set_addr(&dest, host, port))
+				throw Exception("http.host", 
+					0, 
+					"can not resolve hostname \"%s\"", host); 
+			
+			if((sock=socket(AF_INET, SOCK_STREAM, IPPROTO_TCP/*0*/))<0) {
+				int no=pa_socks_errno();
+				throw Exception("http.connect", 
+					0, 
+					"can not make socket: %s (%d)", pa_socks_strerr(no), no); 
+			}
+
+			// To enable SO_DONTLINGER (that is, disable SO_LINGER) 
+			// l_onoff should be set to zero and setsockopt should be called
+			linger dont_linger={0,0};
+			setsockopt(sock, SOL_SOCKET, SO_LINGER, (const char *)&dont_linger, sizeof(dont_linger));
+
+#ifdef WIN32
+// SO_*TIMEO can be defined in .h but not implemlemented in protocol,
+// failing subsequently with Option not supported by protocol (99) message
+// could not suppress that, so leaving this only for win32
+			int timeout_ms=timeout_secs*1000;
+			setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout_ms, sizeof(timeout_ms));
+			setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout_ms, sizeof(timeout_ms));
+#endif
+
+			if(connect(sock, (struct sockaddr *)&dest, sizeof(dest))) {
+				int no=pa_socks_errno();
+				throw Exception("http.connect", 
+					0, 
+					"can not connect to host \"%s\": %s (%d)", host, pa_socks_strerr(no), no); 
+			}
+			size_t request_size=strlen(request);
+			if(send(sock, request, request_size, 0)!=(ssize_t)request_size) {
+				int no=pa_socks_errno();
+				throw Exception("http.timeout", 
+					0, 
+					"error sending request: %s (%d)", pa_socks_strerr(no), no); 
+			}
+
+			result=http_read_response(response, response_size, sock, fail_on_status_ne_200); 
+			closesocket(sock); 
+#ifdef PA_USE_ALARM
+			alarm(0); 
+#endif
+			return result;
+		} catch(...) {
+#ifdef PA_USE_ALARM
+			alarm(0); 
+#endif
+			if(sock>=0) 
+				closesocket(sock); 
+			rethrow;
+		}
+#ifdef PA_USE_ALARM
+	}
+#endif
+}
+
+#ifndef DOXYGEN
+struct Http_pass_header_info {
+	Request_charsets* charsets;
+	String* request;
+	bool user_agent_specified;
+};
+#endif
+static void http_pass_header(HashStringValue::key_type key, 
+			     HashStringValue::value_type value, 
+			     Http_pass_header_info *info) {
+    *info->request <<key<<": "
+	<< attributed_meaning_to_string(*value, String::L_HTTP_HEADER, false)
+	<< CRLF; 
+	
+    if(String(key, String::L_TAINTED).change_case(info->charsets->source(), String::CC_UPPER)=="USER-AGENT")
+	info->user_agent_specified=true;
+}
+
+
+static Charset* detect_charset(Charset& source_charset, const String& content_type_value) {
+	const String::Body CONTENT_TYPE_VALUE=
+		content_type_value.change_case(source_charset, String::CC_UPPER);
+	// content-type: xxx/xxx; source_charset=WE-NEED-THIS
+	// content-type: xxx/xxx; source_charset="WE-NEED-THIS"
+	// content-type: xxx/xxx; source_charset="WE-NEED-THIS";
+	size_t before_charseteq_pos=CONTENT_TYPE_VALUE.pos("CHARSET=");
+	if(before_charseteq_pos!=STRING_NOT_FOUND) {
+		size_t charset_begin=before_charseteq_pos+8/*CHARSET="*/;
+		size_t open_quote_pos=CONTENT_TYPE_VALUE.pos('"', charset_begin);
+		bool quoted=open_quote_pos==charset_begin;
+		if(quoted)
+			charset_begin++; // skip opening '"'
+		size_t charset_end=CONTENT_TYPE_VALUE.length();
+		if(quoted) {
+			size_t close_quote_pos=CONTENT_TYPE_VALUE.pos('"', charset_begin);
+			if(close_quote_pos!=STRING_NOT_FOUND)
+				charset_end=close_quote_pos;
+		} else {
+			size_t delim_pos=CONTENT_TYPE_VALUE.pos(';', charset_begin);
+			if(delim_pos!=STRING_NOT_FOUND)
+				charset_end=delim_pos;
+		}
+		const String::Body CHARSET_NAME_BODY=
+			CONTENT_TYPE_VALUE.mid(charset_begin, charset_end);
+
+		return &charsets.get(CHARSET_NAME_BODY);
+	}
+
+	return 0;
+}
+
+static const String* basic_authorization_field(const char* user, const char* pass) {
+	if(!user&& !pass)
+		return 0;
+
+	String combined;  
+	if(user)
+		combined<<user;
+	combined<<":";
+	if(pass)
+		combined<<pass;
+	
+	String* result=new String("Basic "); *result<<pa_base64_encode(combined.cstr(), combined.length());
+	return result;
+}
+
+static void form_string_value2string(
+								  HashStringValue::key_type key, 
+								  const String& value, 
+								  String& result) 
+{
+	result << String(key, String::L_TAINTED) << "=";
+	result.append(value, String::L_URI, true);
+	result<< "&";
+}
+#ifndef DOXYGEN
+struct Form_table_value2string_info {
+	HashStringValue::key_type key;
+	String& result;
+
+	Form_table_value2string_info(HashStringValue::key_type akey, String& aresult): 
+		key(akey), result(aresult) {}
+};
+#endif
+static void form_table_value2string(Table::element_type row, Form_table_value2string_info* info) {
+	form_string_value2string(info->key, *row->get(0), info->result);
+}
+static void form_value2string(
+								  HashStringValue::key_type key, 
+								  HashStringValue::value_type value, 
+								  String* result) 
+{
+	if(const String* svalue=value->get_string())
+		form_string_value2string(key, *svalue, *result);
+	else if(Table* tvalue=value->get_table()) {
+		Form_table_value2string_info info(key, *result);
+		tvalue->for_each(form_table_value2string, &info);
+	} else
+		throw Exception(0,
+			new String(key, String::L_TAINTED),
+			"is %s, "HTTP_FORM_NAME" option value must either string or table", value->type());
+}
+static const char* form2string(HashStringValue& form) {
+	String string;
+	form.for_each(form_value2string, &string);
+	return string.cstr(String::L_UNSPECIFIED);
+}
+#ifndef DOXYGEN
+struct File_read_http_result {
+	char *str; size_t length;
+	HashStringValue* headers;
+}; 
+#endif
+static void find_headers_end(char* p,
+		char*& headers_end_at,
+		char*& raw_body)
+{
+	raw_body=p;
+	// \n\n
+	// \r\n\r\n
+	while((p=strchr(p, '\n'))) {
+		headers_end_at=++p; // \n>.<
+		if(*p=='\r')  // \r\n>\r?<\n
+			p++;
+		if(*p=='\n') { // \r\n\r>\n?<
+			raw_body=p+1;
+			return;			
+		}
+	}
+	headers_end_at=0;
+}
+
+/// @todo build .cookies field. use ^file.tables.SET-COOKIES.menu{ for now
+static File_read_http_result file_read_http(Request_charsets& charsets, 
+					    const String& file_spec, 
+					    bool as_text,
+						HashStringValue *options=0) {
+	File_read_http_result result;
+	char host[MAX_STRING]; 
+	const char* uri; 
+	short port;
+	const char* method="GET"; bool method_is_get;
+	HashStringValue* form=0;
+	const char* body_cstr=0;
+	int timeout_secs=2;
+	bool fail_on_status_ne_200=true;
+	Value* vheaders=0;
+	Charset *asked_remote_charset=0;
+	const char* user_cstr=0;
+	const char* password_cstr=0;
+
+	if(options) {
+		int valid_options=get_valid_file_options_count(*options);
+
+		if(Value* vmethod=options->get(HTTP_METHOD_NAME)) {
+			valid_options++;
+			method=vmethod->as_string().cstr();
+		}
+		if(Value* vform=options->get(HTTP_FORM_NAME)) {
+			valid_options++;
+			form=vform->get_hash(); 
+		} 
+		if(Value* vbody=options->get(HTTP_BODY_NAME)) {
+			valid_options++;
+			body_cstr=vbody->as_string().cstr(String::L_UNSPECIFIED);
+		} 
+		if(Value* vtimeout=options->get(HTTP_TIMEOUT_NAME)) {
+			valid_options++;
+			timeout_secs=vtimeout->as_int(); 
+		} 
+		if((vheaders=options->get(HTTP_HEADERS_NAME))) {
+			valid_options++;
+		} 
+		if(Value* vany_status=options->get(HTTP_ANY_STATUS_NAME)) {
+			valid_options++;
+			fail_on_status_ne_200=!vany_status->as_bool(); 
+		} 
+		if(Value* vcharset_name=options->get(HTTP_CHARSET_NAME)) {
+			valid_options++;
+			asked_remote_charset=&::charsets.get(vcharset_name->as_string().
+				change_case(charsets.source(), String::CC_UPPER));
+		} 
+		if(Value* vuser=options->get(HTTP_USER)) {
+			valid_options++;
+			user_cstr=vuser->as_string().cstr();
+		} 
+		if(Value* vpassword=options->get(HTTP_PASSWORD)) {
+			valid_options++;
+			password_cstr=vpassword->as_string().cstr();
+		}
+
+		if(valid_options!=options->count())
+			throw Exception("parser.runtime",
+				0,
+				"invalid option passed");
+	}
+	if(!asked_remote_charset) // defaulting to $request:charset
+		asked_remote_charset=&charsets.source();
+
+	method_is_get=strcmp(method, "GET")==0;
+	if(method_is_get && body_cstr)
+		throw Exception("parser.runtime",
+			0,
+			"you can not use $."HTTP_BODY_NAME" option with method GET");
+
+	//preparing request
+	String& connect_string=*new String;
+	// not in ^sql{... L_SQL ...} spirit, but closer to ^file::load one
+	connect_string.append(file_spec, String::L_URI); // tainted pieces -> URI pieces
+
+	String request_head_and_body;
+	{
+		// influence URLencoding of tainted pieces to String::L_URI lang
+		Temp_client_charset temp(charsets, *asked_remote_charset);
+
+		const char* connect_string_cstr=connect_string.cstr(String::L_UNSPECIFIED); 
+
+		const char* current=connect_string_cstr;
+		if(strncmp(current, "http://", 7)!=0)
+			throw Exception(0, 
+				&connect_string, 
+				"does not start with http://"); //never
+		current+=7;
+
+		strncpy(host, current, sizeof(host)-1);  host[sizeof(host)-1]=0;
+		char* host_uri=lsplit(host, '/'); 
+		uri=host_uri?current+(host_uri-1-host):"/"; 
+		char* port_cstr=lsplit(host, ':'); 
+		char* error_pos=0;
+		port=port_cstr?(short)strtol(port_cstr, &error_pos, 0):80;
+
+		if(strchr(uri, '?') && form)
+			throw Exception("parser.runtime",
+				0,
+				"use either uri with ?params or $."HTTP_FORM_NAME" option");
+
+		//making request head
+		String head;
+		head << method;
+		head << " " << uri;
+		if(form)
+			if(method_is_get)
+				head << "?" << form2string(*form);
+		head <<" HTTP/1.0" CRLF
+			"host: "<< host << CRLF; 
+		if(form && !method_is_get) {
+			head << "content-type: application/x-www-form-urlencoded" CRLF;
+			body_cstr = form2string(*form);
+		}
+
+		// http://www.ietf.org/rfc/rfc2617.txt
+		if(const String* authorization_field_value=basic_authorization_field(user_cstr, password_cstr))
+			head<<"authorization: "<<*authorization_field_value<<CRLF;
+
+		bool user_agent_specified=false;
+		if(vheaders && !vheaders->is_string()) { // allow empty
+			if(HashStringValue *headers=vheaders->get_hash()) {
+				Http_pass_header_info info={&charsets, &head, false};
+				headers->for_each(http_pass_header, &info); 
+				user_agent_specified=info.user_agent_specified;
+			} else
+				throw Exception("parser.runtime", 
+					&connect_string,
+					"headers param must be hash"); 
+		};
+		if(!user_agent_specified) // defaulting
+			head << "user-agent: " DEFAULT_USER_AGENT CRLF;
+
+		if(body_cstr) {
+			// recode those pieces which are not in String::L_URI lang 
+			// [those violating HTTP standard, but widly used]
+			body_cstr=Charset::transcode(
+				String::C(body_cstr, strlen(body_cstr)),
+				charsets.source(),
+				*asked_remote_charset);
+
+			head << "content-length: " << format(strlen(body_cstr), "%u") << CRLF;
+		}
+
+		const char* head_cstr=head.cstr(String::L_UNSPECIFIED);
+
+		// recode those pieces which are not in String::L_URI lang 
+		// [those violating HTTP standard, but widly used]
+		head_cstr=Charset::transcode(
+			String::C(head_cstr, strlen(head_cstr)),
+			charsets.source(),
+			*asked_remote_charset);
+
+		// head + end of header
+		request_head_and_body << head_cstr << CRLF;
+		// body
+		if(body_cstr)
+			request_head_and_body << body_cstr;
+	}
+	
+	//sending request
+	char* response;
+	size_t response_size;
+	int status_code=http_request(response, response_size,
+		host, port, request_head_and_body.cstr(), 
+		timeout_secs, fail_on_status_ne_200); 
+	
+	//processing results	
+	char* raw_body; size_t raw_body_size;
+	char* headers_end_at;
+	find_headers_end(response, 
+		headers_end_at,
+		raw_body);
+	raw_body_size=response_size-(raw_body-response);
+	
+	result.headers=new HashStringValue;
+	VHash* vtables=new VHash;
+	result.headers->put(HTTP_TABLES_NAME, vtables);
+	Charset* real_remote_charset=0; // undetected, yet
+
+	if(headers_end_at) {
+		*headers_end_at=0;
+		const String header_block(String::C(response, headers_end_at-response), true);
+		
+		ArrayString aheaders;
+		HashStringValue& tables=vtables->hash();
+
+		size_t pos_after=0;
+		header_block.split(aheaders, pos_after, "\n"); 
+		
+		//processing headers
+		size_t aheaders_count=aheaders.count();
+		for(size_t i=1; i<aheaders_count; i++) {
+			const String& line=*aheaders.get(i);
+			size_t pos=line.pos(':'); 
+			if(pos==STRING_NOT_FOUND || pos<1)
+				throw Exception("http.response", 
+					&connect_string,
+					"bad response from host - bad header \"%s\"", line.cstr());
+			const String::Body HEADER_NAME=
+				line.mid(0, pos).change_case(charsets.source(), String::CC_UPPER);
+			const String& header_value=line.mid(pos+1, line.length()).trim(String::TRIM_BOTH, " \t\r");
+			if(as_text && HEADER_NAME=="CONTENT-TYPE")
+				real_remote_charset=detect_charset(charsets.source(), header_value);
+
+			// tables
+			{
+				Value *valready=(Value *)tables.get(HEADER_NAME);
+				bool existed=valready!=0;
+				Table *table;
+				if(existed) {
+					// second+ appearence
+					table=valready->get_table();
+				} else {
+					// first appearence
+					Table::columns_type columns =new ArrayString(1);
+					*columns+=new String("value");
+					table=new Table(columns);
+				}
+				// this string becomes next row
+				ArrayString& row=*new ArrayString(1);
+				row+=&header_value;
+				*table+=&row;
+				// not existed before? add it
+				if(!existed)
+					tables.put(HEADER_NAME, new VTable(table));
+			}
+
+			result.headers->put(HEADER_NAME, new VString(header_value));
+		}
+	}
+
+	// output response
+	String::C real_body=String::C(raw_body, raw_body_size);
+	if(as_text && raw_body_size) { // must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
+		// defaulting to used-asked charset [it's never empty!]
+		if(!real_remote_charset)
+			real_remote_charset=asked_remote_charset;
+		real_body=Charset::transcode(real_body, *real_remote_charset, charsets.source());
+	}
+
+	result.str=const_cast<char *>(real_body.str); // hacking a little
+	result.length=real_body.length;
+	result.headers->put(file_status_name, new VInt(status_code));
+	return result;
+}
+
+#endif
+
 #ifndef DOXYGEN
 struct File_read_action_info {
 	char **data; size_t *data_size;
@@ -156,6 +843,7 @@ File_read_result file_read(Request_chars
 			   bool fail_on_read_problem,
 			   char* buf, size_t offset, size_t count) {
 	File_read_result result={false, 0, 0, 0};
+#ifdef PA_HTTP
 	if(file_spec.starts_with("http://")) {
 		if(offset || count)
 			throw Exception("parser.runtime",
@@ -163,14 +851,15 @@ File_read_result file_read(Request_chars
 				"offset and load options are not supported for HTTP:// file load");
 
 		// fail on read problem
-		File_read_http_result http=pa_internal_file_read_http(charsets, file_spec, as_text, params);
+		File_read_http_result http=file_read_http(charsets, file_spec, as_text, params);
 		result.success=true;
 		result.str=http.str;
 		result.length=http.length;
 		result.headers=http.headers; 
 	} else {
+#endif
 		if(params) {
-			int valid_options=pa_get_valid_file_options_count(*params);
+			int valid_options=get_valid_file_options_count(*params);
 			if(valid_options!=params->count())
 				throw Exception("parser.runtime",
 					0,
@@ -182,7 +871,9 @@ File_read_result file_read(Request_chars
 		result.success=file_read_action_under_lock(file_spec, 
 			"read", file_read_action, &info, 
 			as_text, fail_on_read_problem); 
+#ifdef PA_HTTP
 	}
+#endif
 
 	if(result.success && as_text) {
 		// UTF-8 signature: EF BB BF
@@ -902,10 +1593,10 @@ static unsigned char gmime_base64_rank[2
 size_t
 g_mime_utils_base64_decode_step (const unsigned char *in, size_t inlen, unsigned char *out, int *state, int *save)
 {
-	const unsigned char *inptr;
-	unsigned char *outptr;
+	const register unsigned char *inptr;
+	register unsigned char *outptr;
 	const unsigned char *inend;
-	int saved;
+	register guint32 saved;
 	unsigned char c;
 	int i;