parser3/src/main/pa_common.C - diff

Return to pa_common.C CVS log

Up to [parser3project] / parser3 / src / main

Diff for /parser3/src/main/pa_common.C between versions 1.143.2.12 and 1.143.2.21

version 1.143.2.12, 2003/02/03 15:57:44	version 1.143.2.21, 2003/03/04 14:55:37
Line 7	Line 7

static const char* IDENT_COMMON_C="$Date$";	static const char* IDENT_COMMON_C="$Date$";

//#include "pa_value_includes.h"
#include "pa_common.h"	#include "pa_common.h"
#include "pa_exception.h"	#include "pa_exception.h"
#include "pa_value_includes.h"
#include "pa_hash.h"	#include "pa_hash.h"
#include "pa_vstring.h"	#include "pa_vstring.h"
#include "pa_vint.h"	#include "pa_vint.h"
	#include "pa_globals.h"

#ifdef CYGWIN	#ifdef CYGWIN
#define _GNU_H_WINDOWS32_SOCKETS	#define _GNU_H_WINDOWS32_SOCKETS
Line 94 static int unlock(int fd) { FLOCK(F_TLOC	Line 93 static int unlock(int fd) { FLOCK(F_TLOC
#endif	#endif
#endif	#endif

	// defines for globals

	#define FILE_STATUS_NAME "status"

	// globals

	StringPtr file_status_name(new String(FILE_STATUS_NAME));

	// defines for statics

	#define HTTP_METHOD_NAME "method"
	#define HTTP_TIMEOUT_NAME "timeout"
	#define HTTP_HEADERS_NAME "headers"
	#define HTTP_ANY_STATUS_NAME "any-status"

	// statics

	static StringPtr http_method_name(new String(HTTP_METHOD_NAME));
	static StringPtr http_timeout_name(new String(HTTP_TIMEOUT_NAME));
	static StringPtr http_headers_name(new String(HTTP_HEADERS_NAME));
	static StringPtr http_any_status_name(new String(HTTP_ANY_STATUS_NAME));

	// defines

#define DEFAULT_USER_AGENT "parser3"	#define DEFAULT_USER_AGENT "parser3"

	// functions

void fix_line_breaks(char* buf, size_t& size) {	void fix_line_breaks(char* buf, size_t& size) {
if(size==0)	if(size==0)
Line 128 void fix_line_breaks(char* buf, size_t&	Line 152 void fix_line_breaks(char* buf, size_t&
char* file_read_text(Pool& pool, Charset& charset,	char* file_read_text(Pool& pool, Charset& charset,
StringPtr file_spec,	StringPtr file_spec,
bool fail_on_read_problem,	bool fail_on_read_problem,
HashStringValue params, HashStringValue* out_fields) {	HashStringValue* params/, HashStringValuePtr out_fields*/) {
char *result; size_t size;	File_read_result file=
File_read_result file_read_result=file_read(pool, charset, file_spec, result, size, true, params, out_fields, fail_on_read_problem);	file_read(pool, charset, file_spec, true, params, fail_on_read_problem);
return file_read_result.success?result:0;	return file.success?file.data:0;
}	}

//http request stuff	//http request stuff
Line 155 static int http_read_response(Pool& pool	Line 179 static int http_read_response(Pool& pool
int result=0;	int result=0;
ssize_t EOLat=0;	ssize_t EOLat=0;
while(true) {	while(true) {
char buf=(char )pool.malloc(MAX_STRING);	char *buf=new(pool) char[MAX_STRING];
ssize_t size=recv(sock, buf, MAX_STRING, 0);	ssize_t size=recv(sock, buf, MAX_STRING, 0);
if(size<=0)	if(size<=0)
break;	break;
response.APPEND_TAINTED(buf, size, "remote HTTP server response", 0);	response.APPEND_TAINTED(buf, size, "remote HTTP server response", 0);
if(!result && (EOLat=response.pos("\r\n", 2))>=0) { // checking status in first response	if(!result && (EOLat=response.pos("\r\n", 2))>=0) { // checking status in first response
StringPtr status_line=response.mid(0, (size_t)EOLat);	StringPtr status_line=response.mid(0, (size_t)EOLat);
Array<StringPtr> astatus;	ArrayString astatus;
status_line->split(astatus, 0, " ", 1);	status_line->split(astatus, 0, " ", 1);
StringPtr status_code=astatus.get(1);	StringPtr status_code=astatus.get(1);
result=status_code->as_int();	result=status_code->as_int();
Line 275 static void http_pass_header(HashStringV	Line 299 static void http_pass_header(HashStringV
if(key->change_case(info->pool, *info->charset, String::CC_UPPER)=="USER-AGENT")	if(key->change_case(info->pool, *info->charset, String::CC_UPPER)=="USER-AGENT")
info->user_agent_specified=true;	info->user_agent_specified=true;
}	}
static HashStringValuePtr file_read_http(Pool& pool, Charset& charset, StringPtr file_spec,	#ifndef DOXYGEN
char*& data, size_t& data_size,	struct File_read_http_result {
HashStringValue *options=0, bool return_outfields=false) {	char *data; size_t size;
HashStringValuePtr result(0);	HashStringValuePtr headers;
	};
	#endif
	static File_read_http_result file_read_http(Pool& pool, Charset& charset, StringPtr file_spec,
	HashStringValue *options=0) {
	File_read_http_result result;
char host[MAX_STRING];	char host[MAX_STRING];
const char* uri;	const char* uri;
int port;	int port;
Line 369 static HashStringValuePtr file_read_http	Line 398 static HashStringValuePtr file_read_http
StringPtr header_block=response.mid(0, pos);	StringPtr header_block=response.mid(0, pos);
StringPtr body=response.mid(pos+4, response.size());	StringPtr body=response.mid(pos+4, response.size());

Array<StringPtr> aheaders;	ArrayString aheaders;
HashStringValuePtr headers(new HashStringValue);	result.headers=HashStringValuePtr(new HashStringValue);
header_block->split(aheaders, 0, "\r\n", 2);	header_block->split(aheaders, 0, "\r\n", 2);

//processing headers	//processing headers
Line 382 static HashStringValuePtr file_read_http	Line 411 static HashStringValuePtr file_read_http
connect_string,	connect_string,
"bad response from host - bad header \"%s\"", line->cstr().get());	"bad response from host - bad header \"%s\"", line->cstr().get());

headers->put(	result.headers->put(
line->mid(0, pos)->change_case(pool, charset, String::CC_UPPER),	line->mid(0, pos)->change_case(pool, charset, String::CC_UPPER),
ValuePtr(new VString(line->mid(pos+2, line->size()))));	ValuePtr(new VString(line->mid(pos+2, line->size()))));
}	}

// output response	// output response
data=new(pool) char[data_size=body->size()];	result.data=new(pool) char[1/as text puts terminating zero later/+(result.size=body->size())];
body->store_to(data, String::UL_AS_IS);	body->store_to(result.data, String::UL_AS_IS);
if(return_outfields) {	result.headers->put(file_status_name, ValuePtr(new VInt(status_code)));
headers->put(file_status_name, ValuePtr(new VInt(status_code)));	return result;
return headers;
} else
return HashStringValuePtr(0);
}	}

#ifndef DOXYGEN	#ifndef DOXYGEN
Line 409 static void file_read_action(Pool& pool,	Line 435 static void file_read_action(Pool& pool,
void *context) {	void *context) {
File_read_action_info& info=static_cast<File_read_action_info >(context);	File_read_action_info& info=static_cast<File_read_action_info >(context);
if(size_t to_read_size=(size_t)finfo.st_size) {	if(size_t to_read_size=(size_t)finfo.st_size) {
*info.data=pool.malloc(to_read_size+(as_text?1:0));	*info.data=new(pool) char[to_read_size+(as_text?1:0)];
info.data_size=(size_t)read(f, info.data, to_read_size);	info.data_size=(size_t)read(f, info.data, to_read_size);

if(ssize_t(info.data_size)<0 \|\| info.data_size>to_read_size)	if(ssize_t(info.data_size)<0 \|\| info.data_size>to_read_size)
Line 419 static void file_read_action(Pool& pool,	Line 445 static void file_read_action(Pool& pool,
*info.data_size, to_read_size);	*info.data_size, to_read_size);
} else { // empty file	} else { // empty file
if(as_text) {	if(as_text) {
*info.data=pool.malloc(1);	*info.data=new(pool) char[1];
(char)(*info.data)=0;	(char)(*info.data)=0;
} else	} else
*info.data=0;	*info.data=0;
Line 428 static void file_read_action(Pool& pool,	Line 454 static void file_read_action(Pool& pool,
}	}
}	}
File_read_result file_read(Pool& pool, Charset& charset, StringPtr file_spec,	File_read_result file_read(Pool& pool, Charset& charset, StringPtr file_spec,
char*& data, size_t& data_size,	bool as_text, HashStringValue *params,
bool as_text, HashStringValue params, HashStringValue* out_fields,
bool fail_on_read_problem) {	bool fail_on_read_problem) {
File_read_result result;	File_read_result result;
if(file_spec->starts_with("http://", 7)) {	if(file_spec->starts_with("http://", 7)) {
// fail on read problem	// fail on read problem
result.fields=file_read_http(pool, charset, file_spec, data, data_size, params);	File_read_http_result http=file_read_http(pool, charset, file_spec, params);
result.success=true;	result.success=true;
	result.data=http.data;
	result.size=http.size;
	result.headers=http.headers;
} else {	} else {
File_read_action_info info={&data, &data_size};	File_read_action_info info={&result.data, &result.size};
result.success=file_read_action_under_lock(pool, file_spec,	result.success=file_read_action_under_lock(pool, file_spec,
"read", file_read_action, &info,	"read", file_read_action, &info,
as_text, fail_on_read_problem);	as_text, fail_on_read_problem);
Line 445 File_read_result file_read(Pool& pool, C	Line 473 File_read_result file_read(Pool& pool, C

if(result.success && as_text) {	if(result.success && as_text) {
// UTF-8 signature: EF BB BF	// UTF-8 signature: EF BB BF
if(data_size>=3) {	if(result.size>=3) {
char in=(char )data;	char in=(char )result.data;
if((in[0] == '\xEF') && (in[1] == '\xBB') &&	if((in[0] == '\xEF') && (in[1] == '\xBB') &&
(in[2] == '\xBF')) {	(in[2] == '\xBF')) {
data=in+3; data_size-=3;// skip prefix	result.data=in+3; result.size-=3;// skip prefix
}	}
}	}

fix_line_breaks((char *)(data), data_size);	fix_line_breaks((char *)(result.data), result.size);
// note: after fixing	// note: after fixing
((char*&)(data))[data_size]=0;	((char*&)(result.data))[result.size]=0;
}	}

return result;	return result;
Line 584 bool file_write_action_under_lock(	Line 612 bool file_write_action_under_lock(

#ifndef DOXYGEN	#ifndef DOXYGEN
struct File_write_action_info {	struct File_write_action_info {
const void *data; size_t size;	const char *data; size_t size;
};	};
#endif	#endif
static void file_write_action(int f, void *context) {	static void file_write_action(int f, void *context) {
Line 599 static void file_write_action(int f, voi	Line 627 static void file_write_action(int f, voi
}	}
void file_write(	void file_write(
StringPtr file_spec,	StringPtr file_spec,
const void *data, size_t size,	const char *data, size_t size,
bool as_text,	bool as_text,
bool do_append) {	bool do_append) {
File_write_action_info info={data, size};	File_write_action_info info={data, size};
Line 686 bool file_readable(StringPtr file_spec)	Line 714 bool file_readable(StringPtr file_spec)
bool dir_readable(StringPtr file_spec) {	bool dir_readable(StringPtr file_spec) {
return entry_readable(file_spec, true);	return entry_readable(file_spec, true);
}	}
StringPtr file_readable(StringPtr path, const String& name) {	StringPtr file_readable(String& path, String& name) {
StringPtr result(new String(*path));	StringPtr result(new String(path));
*result << "/";	*result << "/";
*result << name;	*result << name;
return file_readable(result)?result:StringPtr(0);	return file_readable(result)?result:StringPtr(0);
Line 796 size_t stdout_write(const void *buf, siz	Line 824 size_t stdout_write(const void *buf, siz
}	}

char* unescape_chars(Pool& pool, const char* cp, int len) {	char* unescape_chars(Pool& pool, const char* cp, int len) {
char* s=pool.malloc(len + 1);	char* s=new(pool) char[len + 1];
enum EscapeState {	enum EscapeState {
EscapeRest,	EscapeRest,
EscapeFirst,	EscapeFirst,

E-mail:

Removed from v.1.143.2.12
changed lines
	Added in v.1.143.2.21