parser3/src/main/pa_http.C - diff

Return to pa_http.C CVS log

Up to [parser3project] / parser3 / src / main

Diff for /parser3/src/main/pa_http.C between versions 1.65 and 1.72

version 1.65, 2015/04/30 17:37:43	version 1.72, 2016/07/26 13:20:23
Line 1	Line 1
/** @file	/** @file
Parser: http support functions.	Parser: http support functions.

Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com)	Copyright (c) 2001-2015 Art. Lebedev Studio (http://www.artlebedev.com)
Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)	Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)
*/	*/

Line 32 volatile const char * IDENT_PA_HTTP_C="$	Line 32 volatile const char * IDENT_PA_HTTP_C="$
#define HTTP_ANY_STATUS_NAME "any-status"	#define HTTP_ANY_STATUS_NAME "any-status"
#define HTTP_OMIT_POST_CHARSET_NAME "omit-post-charset" // ^file::load[...;http://...;$.method[post]] by default adds charset to content-type	#define HTTP_OMIT_POST_CHARSET_NAME "omit-post-charset" // ^file::load[...;http://...;$.method[post]] by default adds charset to content-type

#define HTTP_TABLES_NAME "tables"

#define HTTP_USER "user"	#define HTTP_USER "user"
#define HTTP_PASSWORD "password"	#define HTTP_PASSWORD "password"

	#define HTTP_USER_AGENT "user-agent"
#define DEFAULT_USER_AGENT "parser3"	#define DEFAULT_USER_AGENT "parser3"

#ifndef INADDR_NONE	#ifndef INADDR_NONE
Line 616 Table* parse_cookies(Request& r, Table *	Line 615 Table* parse_cookies(Request& r, Table *
return &result;	return &result;
}	}

	void *tables_update(HashStringValue& tables, const String::Body name, const String& value){
	Table *table;
	if(Value *valready=tables.get(name)) {
	// second+ appearence
	table=valready->get_table();
	} else {
	// first appearence
	Table::columns_type columns=new ArrayString(1);
	*columns+=new String("value");
	table=new Table(columns);
	tables.put(name, new VTable(table));
	}
	// this string becomes next row
	ArrayString& row=*new ArrayString(1);
	row+=&value;
	*table+=&row;
	}

/// @todo build .cookies field. use ^file.tables.SET-COOKIES.menu{ for now	/// @todo build .cookies field. use ^file.tables.SET-COOKIES.menu{ for now
File_read_http_result pa_internal_file_read_http(Request& r,	File_read_http_result pa_internal_file_read_http(Request& r, const String& file_spec, bool as_text, HashStringValue *options, bool transcode_text_result) {
const String& file_spec,
bool as_text,
HashStringValue *options,
bool transcode_text_result) {
File_read_http_result result;	File_read_http_result result;
char host[MAX_STRING];	char host[MAX_STRING];
	const char *idna_host;
const char* uri;	const char* uri;
short port=80;	short port=80;
const char* method="GET";	const char* method="GET";
Line 635 File_read_http_result pa_internal_file_r	Line 649 File_read_http_result pa_internal_file_r
Value* vheaders=0;	Value* vheaders=0;
Value* vcookies=0;	Value* vcookies=0;
Value* vbody=0;	Value* vbody=0;
Charset *asked_remote_charset=0;	Charset* asked_remote_charset=0;
Charset* real_remote_charset=0;	Charset* real_remote_charset=0;
const char* user_cstr=0;	const char* user_cstr=0;
const char* password_cstr=0;	const char* password_cstr=0;
Line 703 File_read_http_result pa_internal_file_r	Line 717 File_read_http_result pa_internal_file_r

if(encode){	if(encode){
if(method_is_get)	if(method_is_get)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "you can not use $." HTTP_FORM_ENCTYPE_NAME " option with method GET");
0,
"you can not use $." HTTP_FORM_ENCTYPE_NAME " option with method GET");

multipart=strcasecmp(encode, HTTP_CONTENT_TYPE_MULTIPART_FORMDATA)==0;	multipart=strcasecmp(encode, HTTP_CONTENT_TYPE_MULTIPART_FORMDATA)==0;

if(!multipart && strcasecmp(encode, HTTP_CONTENT_TYPE_FORM_URLENCODED)!=0)	if(!multipart && strcasecmp(encode, HTTP_CONTENT_TYPE_FORM_URLENCODED)!=0)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "$." HTTP_FORM_ENCTYPE_NAME " option value can be " HTTP_CONTENT_TYPE_FORM_URLENCODED " or " HTTP_CONTENT_TYPE_MULTIPART_FORMDATA " only");
0,
"$." HTTP_FORM_ENCTYPE_NAME " option value can be " HTTP_CONTENT_TYPE_FORM_URLENCODED " or " HTTP_CONTENT_TYPE_MULTIPART_FORMDATA " only");
}	}

if(vbody){	if(vbody){
if(method_is_get)	if(method_is_get)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "you can not use $." HTTP_BODY_NAME " option with method GET");
0,
"you can not use $." HTTP_BODY_NAME " option with method GET");

if(form)	if(form)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "you can not use options $." HTTP_BODY_NAME " and $." HTTP_FORM_NAME " together");
0,
"you can not use options $." HTTP_BODY_NAME " and $." HTTP_FORM_NAME " together");
}	}

//preparing request	//preparing request
Line 740 File_read_http_result pa_internal_file_r	Line 746 File_read_http_result pa_internal_file_r

const char* current=connect_string_cstr;	const char* current=connect_string_cstr;
if(strncmp(current, "http://", 7)!=0)	if(strncmp(current, "http://", 7)!=0)
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, &connect_string, "does not start with http://"); //never
&connect_string,
"does not start with http://"); //never
current+=7;	current+=7;

strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0;	strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0;
Line 757 File_read_http_result pa_internal_file_r	Line 761 File_read_http_result pa_internal_file_r
throw Exception(PARSER_RUNTIME, &connect_string, "invalid port number '%s'", port_cstr);	throw Exception(PARSER_RUNTIME, &connect_string, "invalid port number '%s'", port_cstr);
}	}

	idna_host=pa_idna_encode(host, r.charsets.source());

// making request head	// making request head
String head;	String head;
head << method << " " << uri;	head << method << " " << uri;
if(method_is_get && form)	if(method_is_get && form)
head << (strchr(uri, '?')!=0?"&":"?") << pa_form2string(*form, r.charsets);	head << (strchr(uri, '?')!=0?"&":"?") << pa_form2string(*form, r.charsets);

head <<" HTTP/1.0" CRLF "Host: "<< host;	head <<" HTTP/1.0" CRLF "Host: "<< idna_host;
if (port != 80)	if (port != 80)
head << ":" << port_cstr;	head << ":" << port_cstr;
head << CRLF;	head << CRLF;

char* boundary=0;	char* boundary= multipart ? get_uuid_boundary() : 0;

if(multipart){
uuid uuid=get_uuid();
const int boundary_bufsize=10+32+1/for zero-teminator/+1/for faulty snprintfs/;
boundary=new(PointerFreeGC) char[boundary_bufsize];
snprintf(boundary, boundary_bufsize,
"----------%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X",
uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,
uuid.clock_seq >> 8, uuid.clock_seq & 0xFF,
uuid.node[0], uuid.node[1], uuid.node[2],
uuid.node[3], uuid.node[4], uuid.node[5]);
}

String user_headers;	String user_headers;
bool user_agent_specified=false;	bool user_agent_specified=false;
Line 796 File_read_http_result pa_internal_file_r	Line 790 File_read_http_result pa_internal_file_r
&content_type_url_encoded};	&content_type_url_encoded};
headers->for_each<Http_pass_header_info*>(http_pass_header, &info);	headers->for_each<Http_pass_header_info*>(http_pass_header, &info);
} else	} else
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "headers param must be hash");
0,
"headers param must be hash");
};	};

const char* request_body=0;	const char* request_body=0;
Line 824 File_read_http_result pa_internal_file_r	Line 816 File_read_http_result pa_internal_file_r
request_body=vbody->as_string().untaint_and_transcode_cstr(String::L_URI, &(r.charsets));	request_body=vbody->as_string().untaint_and_transcode_cstr(String::L_URI, &(r.charsets));
} else {	} else {
// content-type != application/x-www-form-urlencoded -> transcode only, don't url-encode!	// content-type != application/x-www-form-urlencoded -> transcode only, don't url-encode!
request_body=Charset::transcode(	const String &sbody=vbody->as_string();
String::C(vbody->as_string().cstr(), vbody->as_string().length()),	request_body=Charset::transcode(String::C(sbody.cstr(), sbody.length()), r.charsets.source(), *asked_remote_charset).str;
r.charsets.source(),
*asked_remote_charset
);
}	}
post_size=strlen(request_body);	post_size=strlen(request_body);
}	}
Line 843 File_read_http_result pa_internal_file_r	Line 832 File_read_http_result pa_internal_file_r
head << "User-Agent: " DEFAULT_USER_AGENT CRLF;	head << "User-Agent: " DEFAULT_USER_AGENT CRLF;

if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified	if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "$.content-type can't be specified with method POST");
0,
"$.content-type can't be specified with method POST");

if(vcookies && !vcookies->is_string()){ // allow empty	if(vcookies && !vcookies->is_string()){ // allow empty
if(HashStringValue* cookies=vcookies->get_hash()) {	if(HashStringValue* cookies=vcookies->get_hash()) {
Line 854 File_read_http_result pa_internal_file_r	Line 841 File_read_http_result pa_internal_file_r
cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info);	cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info);
head << CRLF;	head << CRLF;
} else	} else
throw Exception(PARSER_RUNTIME,	throw Exception(PARSER_RUNTIME, 0, "cookies param must be hash");
0,
"cookies param must be hash");
}	}

if(request_body)	if(request_body)
Line 879 File_read_http_result pa_internal_file_r	Line 864 File_read_http_result pa_internal_file_r
}	}
}	}

char* response;	char* response_str;
size_t response_size;	size_t response_size;

// sending request	// sending request
int status_code=http_request(response, response_size,	int status_code=http_request(response_str, response_size, idna_host, port, request, request_size, timeout_secs, fail_on_status_ne_200);
pa_idna_encode(host, r.charsets.source()), port, request, request_size,
timeout_secs, fail_on_status_ne_200);

// processing results	// processing results
char* raw_body; size_t raw_body_size;	char* raw_body; size_t raw_body_size;
char* headers_end_at;	char* headers_end_at;
find_headers_end(response,	find_headers_end(response_str, headers_end_at, raw_body);
headers_end_at,	raw_body_size=response_size-(raw_body-response_str);
raw_body);
raw_body_size=response_size-(raw_body-response);

result.headers=new HashStringValue;	result.headers=new HashStringValue;
VHash* vtables=new VHash;	VHash* vtables=new VHash;
result.headers->put(HTTP_TABLES_NAME, vtables);	result.headers->put("tables", vtables);

	ResponseHeaders response;

if(headers_end_at) {	if(headers_end_at) {
*headers_end_at=0;	*headers_end_at=0;
const String header_block(String::C(response, headers_end_at-response), String::L_TAINTED);	const String header_block(String::C(response_str, headers_end_at-response_str), String::L_TAINTED);

ArrayString aheaders;	ArrayString aheaders;
HashStringValue& tables=vtables->hash();

size_t pos_after=0;	size_t pos_after=0;
header_block.split(aheaders, pos_after, "\n");	header_block.split(aheaders, pos_after, "\n");

// processing headers
size_t aheaders_count=aheaders.count();
for(size_t i=1; i<aheaders_count; i++) {
const String& line=*aheaders.get(i);
size_t pos=line.pos(':');
if(pos==STRING_NOT_FOUND \|\| pos<1)
throw Exception("http.response",
&connect_string,
"bad response from host - bad header \"%s\"", line.cstr());
const String::Body HEADER_NAME=line.mid(0, pos).change_case(r.charsets.source(), String::CC_UPPER);
const String& HEADER_VALUE=line.mid(pos+1, line.length()).trim(String::TRIM_BOTH, " \t\r");
if(as_text && HEADER_NAME==HTTP_CONTENT_TYPE_UPPER && !real_remote_charset)
real_remote_charset=detect_charset(HEADER_VALUE.cstr());

// tables
{
Value valready=(Value )tables.get(HEADER_NAME);
bool existed=valready!=0;
Table *table;
if(existed) {
// second+ appearence
table=valready->get_table();
} else {
// first appearence
Table::columns_type columns=new ArrayString(1);
*columns+=new String("value");
table=new Table(columns);
}
// this string becomes next row
ArrayString& row=*new ArrayString(1);
row+=&HEADER_VALUE;
*table+=&row;
// not existed before? add it
if(!existed)
tables.put(HEADER_NAME, new VTable(table));
}

result.headers->put(HEADER_NAME, new VString(HEADER_VALUE));	Array_iterator<const String*> i(aheaders);
	i.next(); // skipping status
	for(;i.has_next();){
	const char *line=i.next()->cstr();
	if(!response.add_header(line))
	throw Exception("http.response", &connect_string, "bad response from host - bad header \"%s\"", line);
}	}

// filling $.cookies
if(Value vcookies=(Value )tables.get("SET-COOKIE"))
result.headers->put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table())));
}	}

if(as_text && raw_body_size>=3 && strncmp(raw_body, "\xEF\xBB\xBF", 3)==0){	if (!real_remote_charset && !response.content_type.is_empty())
// skip UTF-8 signature (BOM code)	real_remote_charset= detect_charset(response.content_type.cstr());
raw_body+=3;
raw_body_size-=3;	if(as_text)
if(!real_remote_charset)	real_remote_charset=charsets.checkBOM(raw_body, raw_body_size, real_remote_charset);
real_remote_charset=&UTF8_charset;
	if (!real_remote_charset)
	real_remote_charset=asked_remote_charset; // never null

	for(Array_iterator<ResponseHeaders::Header> i(response.headers); i.has_next(); ){
	ResponseHeaders::Header header=i.next();

	header.transcode(*real_remote_charset, r.charsets.source());

	String &header_value=*new String(header.value, String::L_TAINTED);

	tables_update(vtables->hash(), header.name, header_value);
	result.headers->put(header.name, new VString(header_value));
}	}

	// filling $.cookies
	if(Value *vcookies=vtables->hash().get("SET-COOKIE"))
	result.headers->put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table())));

// output response	// output response
String::C real_body=String::C(raw_body, raw_body_size);	String::C real_body=String::C(raw_body, raw_body_size);

if(as_text && transcode_text_result && raw_body_size) { // raw_body_size must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below	if(as_text && transcode_text_result && raw_body_size) { // raw_body_size must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below
// defaulting to used-asked charset [it's never empty!]
if(!real_remote_charset)
real_remote_charset=asked_remote_charset;

real_body=Charset::transcode(real_body, *real_remote_charset, r.charsets.source());	real_body=Charset::transcode(real_body, *real_remote_charset, r.charsets.source());

}	}

result.str=const_cast<char *>(real_body.str); // hacking a little	result.str=const_cast<char *>(real_body.str); // hacking a little

E-mail:

Removed from v.1.65
changed lines
	Added in v.1.72