parser3/src/classes/curl.C - diff

Return to curl.C CVS log

Up to [parser3project] / parser3 / src / classes

Diff for /parser3/src/classes/curl.C between versions 1.68 and 1.80

version 1.68, 2021/10/19 16:16:35	version 1.80, 2026/07/19 19:06:14
Line 1	Line 1
/** @file	/** @file
Parser: @b curl parser class.	Parser: @b curl parser class.

Copyright (c) 2001-2020 Art. Lebedev Studio (http://www.artlebedev.com)	Copyright (c) 2001-2026 Art. Lebedev Studio (https://www.artlebedev.com)
	Authors: Konstantin Morshnev <moko@design.ru>
*/	*/

#include "pa_config_includes.h"	#include "pa_config_includes.h"
Line 14	Line 15
#include "pa_vdate.h"	#include "pa_vdate.h"
#include "pa_vtable.h"	#include "pa_vtable.h"
#include "pa_common.h"	#include "pa_common.h"
#include "pa_http.h"	#include "pa_http.h"
#include "ltdl.h"	#include "ltdl.h"

volatile const char * IDENT_CURL_C="$Id$";	volatile const char * IDENT_CURL_C="$Id$";
Line 43 typedef void (*t_curl_formfree)(struct c	Line 44 typedef void (*t_curl_formfree)(struct c

#define GLINK(name) f_##name=(t_##name)lt_dlsym(handle, #name);	#define GLINK(name) f_##name=(t_##name)lt_dlsym(handle, #name);
#define DLINK(name) GLINK(name) if(!f_##name) return "function " #name " was not found";	#define DLINK(name) GLINK(name) if(!f_##name) return "function " #name " was not found";

static const char dlink(const char dlopen_file_spec) {	static const char dlink(char dlopen_file_spec) {
pa_dlinit();	pa_dlinit();

lt_dlhandle handle=lt_dlopen(dlopen_file_spec);	lt_dlhandle handle;
	do {
	char *next=lsplit(dlopen_file_spec, ',');
	handle=lt_dlopen(dlopen_file_spec);
	dlopen_file_spec=next;
	} while (!handle && dlopen_file_spec);

if(!handle){	if(!handle){
if(const char* result=lt_dlerror())	if(const char* result=lt_dlerror())
return result;	return result;
return "can not open the dynamic link module";	return "cannot open the dynamic link module";
}	}

DLINK(curl_easy_init);	DLINK(curl_easy_init);
Line 68 static const char dlink(const char dlo	Line 74 static const char dlink(const char dlo
return 0;	return 0;
}	}

	#ifdef WIN32
	#define CURL_LIBRARY "libcurl" LT_MODULE_EXT
	#else
	#define CURL_LIBRARY "libcurl" LT_MODULE_EXT ",libcurl" LT_MODULE_EXT ".4"
	#endif

	// 'parser' options can be set outside of 'session' operator
	const char *curl_library=CURL_LIBRARY;
	bool curl_parser_resolve = false;


struct ParserOptions : public PA_Allocated {	struct ParserOptions : public PA_Allocated {
// real options	// real options
Line 81 struct ParserOptions : public PA_Allocat	Line 97 struct ParserOptions : public PA_Allocat
struct curl_httppost *f_post;	struct curl_httppost *f_post;
FILE *f_stderr;	FILE *f_stderr;

	// resolving
	struct curl_slist *resolve;
	bool parser_resolve;
	long resolve_family;
	long port;

// if response content-length check required	// if response content-length check required
bool no_body;	bool no_body;
// stuff to walkaround curl request content-length bugs	// stuff to walkaround curl request content-length bugs
bool is_post;	bool is_post;
bool has_content_length;	bool has_content_length;

ParserOptions() : filename(0), content_type(0), is_text(true), charset(0), response_charset(0), url(0), f_post(0), f_stderr(0), no_body(false), is_post(false), has_content_length(false){}	ParserOptions() : filename(0), content_type(0), is_text(true), charset(0), response_charset(0), url(0), f_post(0), f_stderr(0), resolve(0),
	parser_resolve(curl_parser_resolve), resolve_family(CURL_IPRESOLVE_V4), port(0), no_body(false), is_post(false), has_content_length(false){}
~ParserOptions() {	~ParserOptions() {
f_curl_formfree(f_post);	f_curl_formfree(f_post);
if(f_stderr)	if(f_stderr)
Line 137 public:	Line 160 public:

bool curl_linked = false;	bool curl_linked = false;
const char *curl_status = 0;	const char *curl_status = 0;
const char *curl_library="libcurl" LT_MODULE_EXT;

static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){	static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){
if(!curl_linked)	if(!curl_linked)
curl_status=dlink(curl_library);	curl_status=dlink(pa_strdup(curl_library));

if(curl_status == 0){	if(curl_status == 0){
curl_linked=true;	curl_linked=true;
Temp_curl temp_curl;	Temp_curl temp_curl;
action(r,params);	action(r,params);
} else {	} else {
throw Exception("curl", 0, "failed to load curl library %s: %s", curl_library, curl_status);	const char *hint=strcmp(curl_library, CURL_LIBRARY) ? "" : " (before use, call ^curl:options[ $.library[correct.libcurl" LT_MODULE_EXT ".name] ])";
	throw Exception("curl", 0, "failed to load curl library %s%s", curl_status, hint);
}	}
}	}

Line 162 static void _curl_session(Request& r, Me	Line 185 static void _curl_session(Request& r, Me
}	}

static void _curl_version_action(Request& r, MethodParams& ){	static void _curl_version_action(Request& r, MethodParams& ){
r.write(new VString(new String(f_curl_version(), String::L_TAINTED)));	r.write(*new VString(f_curl_version()));
}	}

static void _curl_version(Request& r, MethodParams& params){	static void _curl_version(Request& r, MethodParams& params){
Line 181 struct CurlOption : public PA_Allocated{	Line 204 struct CurlOption : public PA_Allocated{
CURL_POSTFIELDS,	CURL_POSTFIELDS,
CURL_FORM,	CURL_FORM,
CURL_HEADERS,	CURL_HEADERS,
	CURL_RESOLVE,
CURL_FILE,	CURL_FILE,
CURL_STDERR,	CURL_STDERR,
CURL_HTTP_VERSION,	CURL_HTTP_VERSION,
Line 227 public:	Line 251 public:
CURL_OPT(CURL_INT, FOLLOWLOCATION);	CURL_OPT(CURL_INT, FOLLOWLOCATION);
CURL_OPT(CURL_INT, UNRESTRICTED_AUTH);	CURL_OPT(CURL_INT, UNRESTRICTED_AUTH);
CURL_OPT(CURL_INT, IPRESOLVE);	CURL_OPT(CURL_INT, IPRESOLVE);
	CURL_OPT(CURL_RESOLVE, RESOLVE);

CURL_OPT(CURL_POST, POST);	CURL_OPT(CURL_POST, POST);
CURL_OPT(CURL_INT, HTTPGET);	CURL_OPT(CURL_INT, HTTPGET);
Line 375 static struct curl_slist *curl_headers(H	Line 400 static struct curl_slist *curl_headers(H
return slist;	return slist;
}	}

	static struct curl_slist curl_resolve(HashStringValue value_hash) {
	struct curl_slist *slist=NULL;

	for(HashStringValue::Iterator i(*value_hash); i; i.next() ){
	const char *key=i.key().cstr();
	const String &address=i.value()->as_string();

	if(address.is_empty() \|\| strchr(key, ':')){
	slist=f_curl_slist_append(slist, address.is_empty() ? key : pa_strcat(key, ":", address.cstr()) );
	} else {
	// port-less specification covers both http and https ports by default
	slist=f_curl_slist_append(slist, pa_strcat(key, ":80:", address.cstr()) );
	slist=f_curl_slist_append(slist, pa_strcat(key, ":443:", address.cstr()) );
	}
	}
	return slist;
	}

static const char* curl_transcode(const String &s, Request& r){	static const char* curl_transcode(const String &s, Request& r){
return options().charset ? Charset::transcode(s.cstr(), r.charsets.source(), *options().charset).cstr() : s.cstr();	return options().charset ? Charset::transcode(s.cstr(), r.charsets.source(), *options().charset).cstr() : s.cstr();
}	}
Line 397 static void curl_form(HashStringValue *v	Line 440 static void curl_form(HashStringValue *v
CURLFORM_PTRCONTENTS, curl_transcode(String(tvalue->get(t)->get(0)->cstr()), r),	CURLFORM_PTRCONTENTS, curl_transcode(String(tvalue->get(t)->get(0)->cstr()), r),
CURLFORM_END);	CURLFORM_END);
}	}
} else if(VFile* fvalue=static_cast<VFile *>(i.value()->as("file"))){	} else if(VFile* fvalue=dynamic_cast<VFile *>(i.value())){
// file	// file
f_curl_formadd(&options().f_post, &f_last,	f_curl_formadd(&options().f_post, &f_last,
CURLFORM_PTRNAME, key,	CURLFORM_PTRNAME, key,
Line 477 static void curl_setopt(HashStringValue:	Line 520 static void curl_setopt(HashStringValue:
// integer curl option	// integer curl option
long value_int=(long)v.as_double();	long value_int=(long)v.as_double();
res=f_curl_easy_setopt(curl(), opt->id, value_int);	res=f_curl_easy_setopt(curl(), opt->id, value_int);
	if(opt->id==CURLOPT_PORT)
	options().port=value_int; // can be used instead of :port in url, saving for 'parser' resolve mode
	if(opt->id==CURLOPT_IPRESOLVE)
	options().resolve_family=value_int;
break;	break;
}	}
case CurlOption::CURL_NO_BODY:{	case CurlOption::CURL_NO_BODY:{
Line 525 static void curl_setopt(HashStringValue:	Line 572 static void curl_setopt(HashStringValue:
res=f_curl_easy_setopt(curl(), opt->id, value_hash ? curl_headers(value_hash, r) : 0);	res=f_curl_easy_setopt(curl(), opt->id, value_hash ? curl_headers(value_hash, r) : 0);
break;	break;
}	}
	case CurlOption::CURL_RESOLVE:{
	// 'curl\|parser' resolver mode
	if(const String* mode=v.get_string()){
	bool parser_resolve;
	if(*mode == "parser")
	parser_resolve=true;
	else if(*mode == "curl" \|\| mode->is_empty())
	parser_resolve=false;
	else
	throw Exception("curl", mode, "invalid resolve option value, must be 'parser' or 'curl'");

	if(foptions)
	foptions->parser_resolve=parser_resolve;
	else
	curl_parser_resolve=parser_resolve;
	break;
	}

	// hash of $.[host:port][address[,address]] entries to populate DNS cache with, bypassing the resolver
	HashStringValue *value_hash=v.as_hash("failed to set option 'resolve': value");
	res=f_curl_easy_setopt(curl(), opt->id, options().resolve=curl_resolve(value_hash));
	break;
	}
case CurlOption::CURL_FILE:{	case CurlOption::CURL_FILE:{
// file-spec curl option	// file-spec curl option
const char *file_spec_cstr=curl_check_file(r.full_disk_path(v.as_string()));	const char *file_spec_cstr=curl_check_file(r.full_disk_path(v.as_string()));
Line 552 static void curl_setopt(HashStringValue:	Line 622 static void curl_setopt(HashStringValue:
// 'library' parser option	// 'library' parser option
if(!curl_linked){	if(!curl_linked){
curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC);	curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC);
	if(!curl_library[0])
	curl_library=CURL_LIBRARY;
} else	} else
throw Exception("curl", 0, "failed to set option '%s': already loaded", key.cstr());	throw Exception("curl", 0, "failed to set option '%s': already loaded", key.cstr());
break;	break;
Line 628 static Value *curl_getinfo(const String:	Line 700 static Value *curl_getinfo(const String:
case CurlInfo::CURL_HTTP_VERSION:{	case CurlInfo::CURL_HTTP_VERSION:{
long l=0;	long l=0;
CURL_GETINFO(l);	CURL_GETINFO(l);
return new VString(*new String(curl_http_version_name(l), String::L_TAINTED));	return new VString(curl_http_version_name(l));
}	}
}	}
return VVoid::get();	return VVoid::get();
Line 708 static int curl_header(char *data, size_	Line 780 static int curl_header(char *data, size_
throw Exception("curl", 0, "failed to set " message ": %s", f_curl_easy_strerror(res)); \	throw Exception("curl", 0, "failed to set " message ": %s", f_curl_easy_strerror(res)); \
}	}

	// defined in inet.C
	const char pa_name2ip(const char name, int family, Table table, const char exception_type);

	static void pa_curl_parser_resolve(){
	const char *url=options().url;
	if(!url)
	return;

	// url scheme was validated to be http:// or https:// before
	bool is_http=!pa_strncasecmp(url, "http://");

	char *host=pa_strdup(url+(is_http ? 7 : 8));
	if(char *uri=strpbrk(host, "/?#"))
	*uri=0;

	char *name=host;
	if(char *userinfo_host=lsplit(name, '@'))
	name=userinfo_host;

	if(*name=='[') // no resolving for IPv6 literal
	return;

	long port=options().port;
	if(char *port_cstr=lsplit(name, ':'))
	if(!port)
	port=atol(port_cstr);
	if(!port)
	port=is_http ? 80 : 443;

	if(!*name)
	return;

	const char *prefix=pa_strcat(name, ":", pa_itoa(port));
	size_t prefix_length=strlen(prefix);

	// explicit entries have priority over auto-resolving + last duplicate wins in curl, so we should check
	for(struct curl_slist *e=options().resolve; e; e=e->next){
	const char *data=e->data;
	// support optional '+host:port:address' curl 7.75+ syntax, '-host:port' should not match
	if(*data=='+')
	data++;
	if(!pa_strncasecmp(data, prefix, prefix_length) && data[prefix_length]==':')
	return;
	}

	int family=options().resolve_family==CURL_IPRESOLVE_V4 ? AF_INET : options().resolve_family==CURL_IPRESOLVE_V6 ? AF_INET6 : AF_UNSPEC;
	const char *address=pa_name2ip(name, family, 0, "curl.host");
	if(!address)
	return;

	options().resolve=f_curl_slist_append(options().resolve, pa_strcat(prefix, ":", address));
	CURLcode res;
	CURL_SETOPT(CURLOPT_RESOLVE, options().resolve, "curl resolve list");
	}

static void _curl_load_action(Request& r, MethodParams& params){	static void _curl_load_action(Request& r, MethodParams& params){
if(params.count()==1)	if(params.count()==1)
_curl_options(r, params);	_curl_options(r, params);
Line 723 static void _curl_load_action(Request& r	Line 850 static void _curl_load_action(Request& r
CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function");	CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function");
CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer");	CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer");

	char error[CURL_ERROR_SIZE+1]="";
	CURL_SETOPT(CURLOPT_ERRORBUFFER, error, "curl error buffer");

if(options().is_post && !options().has_content_length){	if(options().is_post && !options().has_content_length){
// libcurl bug walkaround. Prior to 7.38 (Debian Jessie) curl passed Content-length: -1	// libcurl bug walkaround. Prior to 7.38 (Debian Jessie) curl passed Content-length: -1
// after that no Content-length header is passed, that hangs request to nginx.	// after that no Content-length header is passed, that hangs request to nginx.
CURL_SETOPT(CURLOPT_POSTFIELDSIZE, 0, "post content-length");	CURL_SETOPT(CURLOPT_POSTFIELDSIZE, 0, "post content-length");
}	}

	if(options().parser_resolve)
	pa_curl_parser_resolve();

ALTER_EXCEPTION_SOURCE(res=f_curl_easy_perform(curl()), new String(options().url));	ALTER_EXCEPTION_SOURCE(res=f_curl_easy_perform(curl()), new String(options().url));
if(res != CURLE_OK){	if(res != CURLE_OK){
const char *ex_type = 0;	const char *ex_type = 0;
Line 751 static void _curl_load_action(Request& r	Line 884 static void _curl_load_action(Request& r
check_file_size(response.content_length, new String(options().url)); break;	check_file_size(response.content_length, new String(options().url)); break;
default: break;	default: break;
}	}
throw Exception( PA_DEFAULT(ex_type, "curl.fail"), new String(options().url), "%s", f_curl_easy_strerror(res));	throw Exception( PA_DEFAULT(ex_type, "curl.fail"), new String(options().url), "%s", error[0] ? error : f_curl_easy_strerror(res));
}	}

// assure trailing zero	// assure trailing zero
Line 782 static void _curl_load_action(Request& r	Line 915 static void _curl_load_action(Request& r

long http_status = 0;	long http_status = 0;
if(f_curl_easy_getinfo(curl(), CURLINFO_RESPONSE_CODE, &http_status) == CURLE_OK){	if(f_curl_easy_getinfo(curl(), CURLINFO_RESPONSE_CODE, &http_status) == CURLE_OK){
result.fields().put("status", new VInt(http_status));	HASH_PUT_CSTR(result.fields(), "status", new VInt(http_status));
}	}

VHash* vtables=new VHash;	VHash* vtables=new VHash;
result.fields().put("tables", vtables);	HASH_PUT_CSTR(result.fields(), "tables", vtables);

for(Array_iterator<HTTP_Headers::Header> i(response.headers); i.has_next(); ){	for(Array_iterator<HTTP_Headers::Header> i(response.headers); i; ){
HTTP_Headers::Header header=i.next();	HTTP_Headers::Header header=i.next();

if(asked_charset)	if(asked_charset)
Line 802 static void _curl_load_action(Request& r	Line 935 static void _curl_load_action(Request& r

// filling $.cookies	// filling $.cookies
if(Value *vcookies=vtables->hash().get("SET-COOKIE"))	if(Value *vcookies=vtables->hash().get("SET-COOKIE"))
result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table())));	HASH_PUT_CSTR(result.fields(), HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table())));

r.write(result);	r.write(result);
}	}

E-mail:

Removed from v.1.68
changed lines
	Added in v.1.80