Annotation of parser3/src/classes/curl.C, revision 1.78
1.1 misha 1: /** @file
2: Parser: @b curl parser class.
3:
1.78 ! moko 4: Copyright (c) 2001-2026 Art. Lebedev Studio (https://www.artlebedev.com)
1.70 moko 5: Authors: Konstantin Morshnev <moko@design.ru>
1.1 misha 6: */
7:
1.3 misha 8: #include "pa_config_includes.h"
9:
1.1 misha 10: #include "pa_vmethod_frame.h"
11: #include "pa_request.h"
12: #include "pa_vfile.h"
13: #include "pa_charsets.h"
1.4 misha 14: #include "pa_vstring.h"
1.32 moko 15: #include "pa_vdate.h"
1.4 misha 16: #include "pa_vtable.h"
17: #include "pa_common.h"
1.11 moko 18: #include "pa_http.h"
1.5 misha 19: #include "ltdl.h"
1.1 misha 20:
1.78 ! moko 21: volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.77 2025/05/26 01:56:54 moko Exp $";
1.13 moko 22:
1.1 misha 23: class MCurl: public Methoded {
24: public:
25: MCurl();
26: };
27:
28: // global variables
29:
1.36 moko 30: DECLARE_CLASS_VAR(curl, new MCurl);
1.1 misha 31:
32: #include "curl.h"
33:
34: typedef CURL *(*t_curl_easy_init)(); t_curl_easy_init f_curl_easy_init;
35: typedef CURLcode (*t_curl_easy_setopt)(CURL *, CURLoption option, ...); t_curl_easy_setopt f_curl_easy_setopt;
36: typedef CURLcode (*t_curl_easy_perform)(CURL *); t_curl_easy_perform f_curl_easy_perform;
37: typedef void (*t_curl_easy_cleanup)(CURL *); t_curl_easy_cleanup f_curl_easy_cleanup;
38: typedef const char *(*t_curl_easy_strerror)(CURLcode); t_curl_easy_strerror f_curl_easy_strerror;
1.4 misha 39: typedef CURLcode (*t_curl_easy_getinfo)(CURL *curl, CURLINFO info, ...); t_curl_easy_getinfo f_curl_easy_getinfo;
1.1 misha 40: typedef struct curl_slist *(*t_curl_slist_append)(struct curl_slist *,const char *); t_curl_slist_append f_curl_slist_append;
1.4 misha 41: typedef const char *(*t_curl_version)(); t_curl_version f_curl_version;
42: typedef CURLFORMcode (*t_curl_formadd)(struct curl_httppost **httppost, struct curl_httppost **last_post, ...); t_curl_formadd f_curl_formadd;
43: typedef void (*t_curl_formfree)(struct curl_httppost *form); t_curl_formfree f_curl_formfree;
1.1 misha 44:
45: #define GLINK(name) f_##name=(t_##name)lt_dlsym(handle, #name);
46: #define DLINK(name) GLINK(name) if(!f_##name) return "function " #name " was not found";
47:
1.75 moko 48: static const char *dlink(char *dlopen_file_spec) {
1.28 moko 49: pa_dlinit();
1.1 misha 50:
1.75 moko 51: lt_dlhandle handle;
52: do {
53: char *next=lsplit(dlopen_file_spec, ',');
54: handle=lt_dlopen(dlopen_file_spec);
55: dlopen_file_spec=next;
56: } while (!handle && dlopen_file_spec);
1.1 misha 57:
58: if(!handle){
59: if(const char* result=lt_dlerror())
60: return result;
1.73 moko 61: return "cannot open the dynamic link module";
1.1 misha 62: }
63:
64: DLINK(curl_easy_init);
65: DLINK(curl_easy_cleanup);
1.4 misha 66: DLINK(curl_version);
1.1 misha 67: DLINK(curl_easy_setopt);
68: DLINK(curl_easy_perform);
69: DLINK(curl_easy_strerror);
1.4 misha 70: DLINK(curl_easy_getinfo);
1.1 misha 71: DLINK(curl_slist_append);
1.4 misha 72: DLINK(curl_formadd);
73: DLINK(curl_formfree);
1.1 misha 74: return 0;
75: }
76:
77:
1.55 moko 78: struct ParserOptions : public PA_Allocated {
1.39 moko 79: // real options
1.12 misha 80: const String *filename;
1.1 misha 81: const String *content_type;
82: bool is_text;
1.4 misha 83: Charset *charset, *response_charset;
1.39 moko 84:
85: // stuff for internal use
86: const char *url;
1.4 misha 87: struct curl_httppost *f_post;
1.19 moko 88: FILE *f_stderr;
1.1 misha 89:
1.60 moko 90: // if response content-length check required
91: bool no_body;
92: // stuff to walkaround curl request content-length bugs
1.57 moko 93: bool is_post;
94: bool has_content_length;
95:
1.60 moko 96: ParserOptions() : filename(0), content_type(0), is_text(true), charset(0), response_charset(0), url(0), f_post(0), f_stderr(0), no_body(false), is_post(false), has_content_length(false){}
1.4 misha 97: ~ParserOptions() {
98: f_curl_formfree(f_post);
1.19 moko 99: if(f_stderr)
100: fclose(f_stderr);
1.4 misha 101: }
1.55 moko 102:
1.1 misha 103: };
104:
1.9 moko 105: // using TLS instead of keeping variables in request
106: THREAD_LOCAL CURL *fcurl = 0;
107: THREAD_LOCAL ParserOptions *foptions = 0;
1.1 misha 108:
109: static CURL *curl(){
110: if(!fcurl)
111: throw Exception("curl", 0, "outside of 'session' operator");
112: return fcurl;
113: }
114:
115: static ParserOptions &options(){
116: if(!foptions)
117: throw Exception("curl", 0, "outside of 'session' operator");
118: return *foptions;
119: }
120:
121: // using temporal object scheme to garanty cleanup call
122: class Temp_curl {
1.4 misha 123: CURL *saved_curl;
124: ParserOptions *saved_options;
1.66 moko 125:
126: // every TLS should be referenced elsewhere, or GC will collect it
127: CURL *thread_curl;
128: ParserOptions *thread_options;
1.1 misha 129: public:
1.4 misha 130: Temp_curl() : saved_curl(fcurl), saved_options(foptions){
1.66 moko 131: thread_curl = fcurl = f_curl_easy_init();
132: thread_options = foptions = new ParserOptions();
1.27 moko 133: f_curl_easy_setopt(fcurl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); // avoid ipv6 by default
1.1 misha 134: }
1.56 moko 135:
1.1 misha 136: ~Temp_curl() {
137: f_curl_easy_cleanup(fcurl);
138: fcurl = saved_curl;
139: delete foptions;
140: foptions = saved_options;
141: }
142: };
143:
1.75 moko 144: #ifdef WIN32
145: #define CURL_LIBRARY "libcurl" LT_MODULE_EXT
146: #else
147: #define CURL_LIBRARY "libcurl" LT_MODULE_EXT ",libcurl" LT_MODULE_EXT ".4"
148: #endif
149:
1.1 misha 150: bool curl_linked = false;
1.41 moko 151: const char *curl_status = 0;
1.75 moko 152:
153: const char *curl_library=CURL_LIBRARY;
1.1 misha 154:
155: static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){
1.41 moko 156: if(!curl_linked)
1.75 moko 157: curl_status=dlink(pa_strdup(curl_library));
1.1 misha 158:
159: if(curl_status == 0){
1.41 moko 160: curl_linked=true;
1.1 misha 161: Temp_curl temp_curl;
162: action(r,params);
163: } else {
1.75 moko 164: const char *hint=strcmp(curl_library, CURL_LIBRARY) ? "" : " (before use, call ^curl:options[ $.library[correct.libcurl" LT_MODULE_EXT ".name] ])";
1.69 moko 165: throw Exception("curl", 0, "failed to load curl library %s%s", curl_status, hint);
1.1 misha 166: }
167: }
168:
169: static void _curl_session_action(Request& r, MethodParams& params){
170: Value& body_code=params.as_junction(0, "body must be code");
171: r.process_write(body_code);
172: }
173:
174: static void _curl_session(Request& r, MethodParams& params){
175: temp_curl(_curl_session_action, r, params);
176: }
177:
1.5 misha 178: static void _curl_version_action(Request& r, MethodParams& ){
1.77 moko 179: r.write(*new VString(f_curl_version()));
1.4 misha 180: }
181:
182: static void _curl_version(Request& r, MethodParams& params){
183: fcurl ? _curl_version_action(r, params) : temp_curl(_curl_version_action, r, params);
184: }
1.1 misha 185:
1.55 moko 186: struct CurlOption : public PA_Allocated{
1.1 misha 187:
188: enum OptionType {
189: CURL_STRING,
190: CURL_URLENCODE, // url-encoded string
1.4 misha 191: CURL_URL,
1.1 misha 192: CURL_INT,
1.60 moko 193: CURL_NO_BODY,
1.1 misha 194: CURL_POST,
1.57 moko 195: CURL_POSTFIELDS,
1.4 misha 196: CURL_FORM,
1.1 misha 197: CURL_HEADERS,
198: CURL_FILE,
1.15 moko 199: CURL_STDERR,
1.56 moko 200: CURL_HTTP_VERSION,
1.1 misha 201: PARSER_LIBRARY,
202: PARSER_NAME,
203: PARSER_CONTENT_TYPE,
204: PARSER_MODE,
1.4 misha 205: PARSER_CHARSET,
206: PARSER_RESPONSE_CHARSET
1.1 misha 207: };
208:
209: CURLoption id;
210: OptionType type;
211: CurlOption(CURLoption aid, OptionType atype): id(aid), type(atype) {}
212: };
213:
214: class CurlOptionHash: public HashString<CurlOption*> {
215: public:
216: CurlOptionHash() {
217: #define CURL_OPT(type, name) put(str_lower(#name),new CurlOption(CURLOPT_##name, CurlOption::type));
218: #define PARSER_OPT(type, name) put(name,new CurlOption((CURLoption)0, CurlOption::type));
1.4 misha 219: CURL_OPT(CURL_URL, URL);
1.1 misha 220: CURL_OPT(CURL_STRING, INTERFACE);
221: CURL_OPT(CURL_INT, LOCALPORT);
222: CURL_OPT(CURL_INT, PORT);
1.4 misha 223:
1.15 moko 224: CURL_OPT(CURL_INT, VERBOSE);
225: CURL_OPT(CURL_STDERR, STDERR);
226: CURL_OPT(CURL_INT, MAXFILESIZE);
227:
1.1 misha 228: CURL_OPT(CURL_INT, HTTPAUTH);
229: CURL_OPT(CURL_STRING, USERPWD);
230:
231: CURL_OPT(CURL_STRING, USERNAME);
232: CURL_OPT(CURL_STRING, PASSWORD);
233:
1.4 misha 234: CURL_OPT(CURL_URLENCODE, USERAGENT);
235: CURL_OPT(CURL_URLENCODE, REFERER);
1.1 misha 236: CURL_OPT(CURL_INT, AUTOREFERER);
1.25 misha 237:
1.1 misha 238: CURL_OPT(CURL_STRING, ENCODING); // gzip or deflate
1.25 misha 239: CURL_OPT(CURL_STRING, ACCEPT_ENCODING); // gzip or deflate
240:
1.1 misha 241: CURL_OPT(CURL_INT, FOLLOWLOCATION);
242: CURL_OPT(CURL_INT, UNRESTRICTED_AUTH);
1.27 moko 243: CURL_OPT(CURL_INT, IPRESOLVE);
1.1 misha 244:
1.57 moko 245: CURL_OPT(CURL_POST, POST);
1.1 misha 246: CURL_OPT(CURL_INT, HTTPGET);
1.60 moko 247: CURL_OPT(CURL_NO_BODY, NOBODY);
1.4 misha 248: CURL_OPT(CURL_STRING, CUSTOMREQUEST);
1.1 misha 249:
1.57 moko 250: CURL_OPT(CURL_POSTFIELDS, POSTFIELDS); // hopefully is safe too
251: CURL_OPT(CURL_POSTFIELDS, COPYPOSTFIELDS);
1.4 misha 252: CURL_OPT(CURL_FORM, HTTPPOST);
1.1 misha 253:
254: CURL_OPT(CURL_HEADERS, HTTPHEADER);
255: CURL_OPT(CURL_URLENCODE, COOKIE);
256: CURL_OPT(CURL_URLENCODE, COOKIELIST);
257: CURL_OPT(CURL_INT, COOKIESESSION);
258:
259: CURL_OPT(CURL_INT, IGNORE_CONTENT_LENGTH);
260: CURL_OPT(CURL_INT, HTTP_CONTENT_DECODING);
261: CURL_OPT(CURL_INT, HTTP_TRANSFER_DECODING);
262:
1.4 misha 263: CURL_OPT(CURL_INT, MAXREDIRS);
264: CURL_OPT(CURL_INT, POSTREDIR);
265:
266: CURL_OPT(CURL_STRING, RANGE);
267:
1.1 misha 268: CURL_OPT(CURL_INT, TIMEOUT);
269: CURL_OPT(CURL_INT, TIMEOUT_MS);
270: CURL_OPT(CURL_INT, LOW_SPEED_LIMIT);
271: CURL_OPT(CURL_INT, LOW_SPEED_TIME);
272: CURL_OPT(CURL_INT, MAXCONNECTS);
273:
1.4 misha 274: CURL_OPT(CURL_STRING, PROXY);
275: CURL_OPT(CURL_INT, PROXYPORT);
276: CURL_OPT(CURL_INT, PROXYTYPE);
277: CURL_OPT(CURL_INT, HTTPPROXYTUNNEL);
278: CURL_OPT(CURL_STRING, PROXYUSERPWD);
279: CURL_OPT(CURL_INT, PROXYAUTH);
280:
1.1 misha 281: CURL_OPT(CURL_INT, FRESH_CONNECT);
282: CURL_OPT(CURL_INT, FORBID_REUSE);
283: CURL_OPT(CURL_INT, CONNECTTIMEOUT);
284: CURL_OPT(CURL_INT, CONNECTTIMEOUT_MS);
1.4 misha 285: CURL_OPT(CURL_INT, FAILONERROR);
1.1 misha 286:
287: CURL_OPT(CURL_FILE, SSLCERT);
288: CURL_OPT(CURL_STRING, SSLCERTTYPE);
289: CURL_OPT(CURL_FILE, SSLKEY);
290: CURL_OPT(CURL_STRING, SSLKEYTYPE);
291: CURL_OPT(CURL_STRING, KEYPASSWD);
292: CURL_OPT(CURL_STRING, SSLENGINE);
293: CURL_OPT(CURL_STRING, SSLENGINE_DEFAULT);
294:
295: CURL_OPT(CURL_FILE, ISSUERCERT);
296: CURL_OPT(CURL_FILE, CRLFILE);
297:
298: CURL_OPT(CURL_STRING, CAINFO);
1.18 moko 299: CURL_OPT(CURL_FILE, CAPATH);
1.1 misha 300: CURL_OPT(CURL_INT, SSL_VERIFYPEER);
301: CURL_OPT(CURL_INT, SSL_VERIFYHOST);
302: CURL_OPT(CURL_STRING, SSL_CIPHER_LIST);
303: CURL_OPT(CURL_INT, SSL_SESSIONID_CACHE);
1.43 moko 304: CURL_OPT(CURL_INT, SSLVERSION);
1.56 moko 305: CURL_OPT(CURL_HTTP_VERSION, HTTP_VERSION);
1.1 misha 306:
307: PARSER_OPT(PARSER_LIBRARY, "library");
308: PARSER_OPT(PARSER_NAME, "name");
309: PARSER_OPT(PARSER_CONTENT_TYPE, "content-type");
310: PARSER_OPT(PARSER_MODE, "mode");
311: PARSER_OPT(PARSER_CHARSET, "charset");
1.4 misha 312: PARSER_OPT(PARSER_RESPONSE_CHARSET, "response-charset");
1.1 misha 313: }
314:
315: } *curl_options=0;
316:
1.55 moko 317: struct CurlInfo : public PA_Allocated{
1.32 moko 318:
319: enum OptionType {
320: CURL_STRING,
321: CURL_INT,
1.56 moko 322: CURL_DOUBLE,
323: CURL_HTTP_VERSION
1.32 moko 324: };
325:
326: CURLINFO id;
327: OptionType type;
328: CurlInfo(CURLINFO aid, OptionType atype): id(aid), type(atype) {}
329: };
330:
1.61 moko 331: class CurlInfoHash: public OrderedHashString<CurlInfo*> {
1.32 moko 332: public:
333: CurlInfoHash() {
334: #define CURL_INF(type, name) put(str_lower(#name),new CurlInfo(CURLINFO_##name, CurlInfo::type));
1.61 moko 335: CURL_INF(CURL_STRING, SCHEME);
336: CURL_INF(CURL_HTTP_VERSION, HTTP_VERSION);
337: CURL_INF(CURL_STRING, EFFECTIVE_URL);
338: CURL_INF(CURL_STRING, CONTENT_TYPE);
339: CURL_INF(CURL_INT, RESPONSE_CODE);
340: CURL_INF(CURL_INT, OS_ERRNO);
341:
342: CURL_INF(CURL_DOUBLE, NAMELOOKUP_TIME);
1.32 moko 343: CURL_INF(CURL_DOUBLE, APPCONNECT_TIME);
1.61 moko 344: CURL_INF(CURL_DOUBLE, PRETRANSFER_TIME);
345: CURL_INF(CURL_DOUBLE, STARTTRANSFER_TIME);
1.32 moko 346: CURL_INF(CURL_DOUBLE, CONNECT_TIME);
1.61 moko 347: CURL_INF(CURL_DOUBLE, TOTAL_TIME);
348:
1.32 moko 349: CURL_INF(CURL_DOUBLE, CONTENT_LENGTH_DOWNLOAD);
350: CURL_INF(CURL_DOUBLE, CONTENT_LENGTH_UPLOAD);
351: CURL_INF(CURL_INT, HEADER_SIZE);
1.61 moko 352: CURL_INF(CURL_INT, REQUEST_SIZE);
353: CURL_INF(CURL_DOUBLE, SIZE_DOWNLOAD);
354: CURL_INF(CURL_DOUBLE, SIZE_UPLOAD);
355: CURL_INF(CURL_DOUBLE, SPEED_DOWNLOAD);
356: CURL_INF(CURL_DOUBLE, SPEED_UPLOAD);
357:
1.32 moko 358: CURL_INF(CURL_INT, NUM_CONNECTS);
359: CURL_INF(CURL_STRING, PRIMARY_IP);
1.61 moko 360: CURL_INF(CURL_INT, HTTPAUTH_AVAIL);
1.32 moko 361: CURL_INF(CURL_INT, PROXYAUTH_AVAIL);
362: CURL_INF(CURL_INT, REDIRECT_COUNT);
363: CURL_INF(CURL_DOUBLE, REDIRECT_TIME);
364: CURL_INF(CURL_STRING, REDIRECT_URL);
365: CURL_INF(CURL_INT, SSL_VERIFYRESULT);
366: }
367:
368: } *curl_infos=0;
369:
1.1 misha 370: static const char *curl_urlencode(const String &s, Request& r){
371: if(options().charset){
372: Temp_client_charset temp(r.charsets, *options().charset);
373: return s.untaint_and_transcode_cstr(String::L_URI, &r.charsets);
374: } else
375: return s.untaint_cstr(String::L_URI);
376: }
377:
378: static struct curl_slist *curl_headers(HashStringValue *value_hash, Request& r) {
379: struct curl_slist *slist=NULL;
380:
381: for(HashStringValue::Iterator i(*value_hash); i; i.next() ){
382: String header =
1.10 moko 383: String(pa_http_safe_header_name(capitalize(i.key().cstr())), String::L_AS_IS)
1.1 misha 384: << ": "
1.10 moko 385: << String(i.value()->as_string(), String::L_HTTP_HEADER);
1.1 misha 386:
387: slist=f_curl_slist_append(slist, curl_urlencode(header, r));
388: }
389: return slist;
390: }
391:
1.4 misha 392: static const char* curl_transcode(const String &s, Request& r){
393: return options().charset ? Charset::transcode(s.cstr(), r.charsets.source(), *options().charset).cstr() : s.cstr();
394: }
395:
396: static void curl_form(HashStringValue *value_hash, Request& r){
397: struct curl_httppost *f_last=0;
398: for(HashStringValue::Iterator i(*value_hash); i; i.next() ){
399: const char *key = curl_transcode(String(i.key().cstr()), r);
400: if(const String* svalue = i.value()->get_string()){
401: // string
402: f_curl_formadd(&options().f_post, &f_last,
403: CURLFORM_PTRNAME, key,
404: CURLFORM_PTRCONTENTS, curl_transcode(String(svalue->cstr()), r),
405: CURLFORM_END);
406: } else if(Table* tvalue = i.value()->get_table()){
407: // table
408: for(size_t t = 0; t < tvalue->count(); t++) {
409: f_curl_formadd(&options().f_post, &f_last,
410: CURLFORM_PTRNAME, key,
411: CURLFORM_PTRCONTENTS, curl_transcode(String(tvalue->get(t)->get(0)->cstr()), r),
412: CURLFORM_END);
413: }
1.72 moko 414: } else if(VFile* fvalue=dynamic_cast<VFile *>(i.value())){
1.4 misha 415: // file
416: f_curl_formadd(&options().f_post, &f_last,
417: CURLFORM_PTRNAME, key,
418: CURLFORM_BUFFER, curl_transcode(String(fvalue->fields().get("name")->as_string(), String::L_FILE_SPEC), r),
419: CURLFORM_BUFFERLENGTH, (long)fvalue->value_size(),
420: CURLFORM_BUFFERPTR, fvalue->value_ptr(),
421: CURLFORM_CONTENTTYPE, fvalue->fields().get("content-type")->as_string().taint_cstr(String::L_URI),
422: CURLFORM_END);
423: } else {
1.56 moko 424: throw Exception("curl", new String(i.key(), String::L_TAINTED), "is %s, form option value can be string, table or file only", i.value()->type());
1.4 misha 425: }
426: }
427: }
428:
1.18 moko 429: static const char *curl_check_file(const String &file_spec){
430: const char *file_spec_cstr=file_spec.taint_cstr(String::L_FILE_SPEC);
431: struct stat finfo;
1.49 moko 432: if(pa_stat(file_spec_cstr, &finfo)==0)
1.18 moko 433: check_safe_mode(finfo, file_spec, file_spec_cstr);
434: return file_spec_cstr;
435: }
436:
1.56 moko 437: static long curl_http_version(const String &name){
438: if(name.is_empty()) return CURL_HTTP_VERSION_NONE;
439:
440: if(name == "1.0") return CURL_HTTP_VERSION_1_0;
441: if(name == "1.1") return CURL_HTTP_VERSION_1_1;
442: if(name == "2") return CURL_HTTP_VERSION_2;
443: if(name == "2.0") return CURL_HTTP_VERSION_2_0;
444:
445: const char *sname = str_upper(name.cstr());
446: if(!strcmp(sname,"2TLS")) return CURL_HTTP_VERSION_2TLS;
447: if(!strcmp(sname,"2ONLY")) return CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE;
448: throw Exception("curl", &name, "invalid http_version option value");
449: }
450:
451: static const char *curl_http_version_name(long value){
452: if(value == CURL_HTTP_VERSION_NONE) return "none";
453: if(value == CURL_HTTP_VERSION_1_0) return "1.0";
454: if(value == CURL_HTTP_VERSION_1_1) return "1.1";
455: if(value == CURL_HTTP_VERSION_2) return "2";
456: throw Exception("curl", 0, "invalid http version '%d' in info", value);
457: }
458:
1.1 misha 459: static void curl_setopt(HashStringValue::key_type key, HashStringValue::value_type value, Request& r) {
460: CurlOption *opt=curl_options->get(key);
461:
462: if(opt==0)
463: throw Exception("curl", 0, "called with invalid option '%s'", key.cstr());
464:
465: CURLcode res = CURLE_OK;
1.44 moko 466: Value &v=r.process(*value);
1.1 misha 467:
468: switch (opt->type){
469: case CurlOption::CURL_STRING:{
470: // string curl option
471: const char *value_str=v.as_string().cstr();
472: res=f_curl_easy_setopt(curl(), opt->id, value_str);
473: break;
474: }
475: case CurlOption::CURL_URLENCODE:{
476: // url-encoded string curl option
477: const char *value_str=curl_urlencode(v.as_string(), r);
478: res=f_curl_easy_setopt(curl(), opt->id, value_str);
479: break;
480: }
1.4 misha 481: case CurlOption::CURL_URL:{
482: // url-encoded string curl_url option
1.5 misha 483: const String url = v.as_string();
484: if(!url.starts_with("http://") && !url.starts_with("https://"))
485: throw Exception("curl", 0, "failed to set option '%s': invalid url scheme '%s'", key.cstr(), url.cstr());
1.39 moko 486: options().url=curl_urlencode(url, r);
487: res=f_curl_easy_setopt(curl(), opt->id, options().url);
1.4 misha 488: break;
489: }
1.1 misha 490: case CurlOption::CURL_INT:{
1.2 misha 491: // integer curl option
492: long value_int=(long)v.as_double();
1.1 misha 493: res=f_curl_easy_setopt(curl(), opt->id, value_int);
494: break;
495: }
1.60 moko 496: case CurlOption::CURL_NO_BODY:{
497: // integer curl option
498: long value_int=(long)v.as_double();
499: res=f_curl_easy_setopt(curl(), opt->id, value_int);
500: options().no_body=value_int != 0;
501: break;
502: }
1.1 misha 503: case CurlOption::CURL_POST:{
1.57 moko 504: // integer curl option
505: long value_int=(long)v.as_double();
506: res=f_curl_easy_setopt(curl(), opt->id, value_int);
507: options().is_post=value_int != 0;
508: break;
509: }
510: case CurlOption::CURL_POSTFIELDS:{
1.1 misha 511: // http post curl option
512: if(v.get_string()){
1.2 misha 513: if( (res=f_curl_easy_setopt(curl(), CURLOPT_POSTFIELDSIZE, -1L)) == CURLE_OK )
1.1 misha 514: res=f_curl_easy_setopt(curl(), opt->id, curl_urlencode(v.as_string(), r));
515: } else {
1.68 moko 516: VFile *file=v.as_vfile();
1.2 misha 517: if( (res=f_curl_easy_setopt(curl(), CURLOPT_POSTFIELDSIZE, (long)file->value_size())) == CURLE_OK )
1.1 misha 518: res=f_curl_easy_setopt(curl(), opt->id, file->value_ptr());
519: }
1.57 moko 520: options().has_content_length=true;
1.1 misha 521: break;
522: }
1.4 misha 523: case CurlOption::CURL_FORM:{
1.48 moko 524: HashStringValue *value_hash = v.as_hash("failed to set option 'httppost': value");
1.4 misha 525: if(value_hash){
526: curl_form(value_hash, r);
1.48 moko 527: } else {
1.57 moko 528: if(options().f_post)
529: f_curl_formfree(options().f_post);
1.4 misha 530: options().f_post = 0;
1.5 misha 531: }
1.4 misha 532: res=f_curl_easy_setopt(curl(), CURLOPT_HTTPPOST, foptions->f_post);
1.57 moko 533: options().has_content_length=true;
1.4 misha 534: break;
535: }
1.1 misha 536: case CurlOption::CURL_HEADERS:{
537: // http headers curl option
1.48 moko 538: HashStringValue *value_hash=v.as_hash("failed to set option 'httpheader': value");
1.1 misha 539: res=f_curl_easy_setopt(curl(), opt->id, value_hash ? curl_headers(value_hash, r) : 0);
540: break;
541: }
542: case CurlOption::CURL_FILE:{
543: // file-spec curl option
1.65 moko 544: const char *file_spec_cstr=curl_check_file(r.full_disk_path(v.as_string()));
1.18 moko 545: res=f_curl_easy_setopt(curl(), opt->id, file_spec_cstr);
1.1 misha 546: break;
547: }
1.16 moko 548: case CurlOption::CURL_STDERR:{
549: // verbose output redirection from stderr to file curl option
1.65 moko 550: const char *file_spec_cstr=curl_check_file(r.full_disk_path(v.as_string()));
1.53 moko 551: FILE *f_stderr=options().f_stderr=pa_fopen(file_spec_cstr, "wt");
1.19 moko 552: if (f_stderr){
553: res=f_curl_easy_setopt(curl(), opt->id, f_stderr);
1.16 moko 554: } else {
1.18 moko 555: throw Exception("curl", 0, "failed to set option '%s': unable to open file '%s'", key.cstr(), file_spec_cstr);
1.16 moko 556: }
557: break;
558: }
1.56 moko 559: case CurlOption::CURL_HTTP_VERSION:{
560: // http protocol version name curl option
561: long value_int=curl_http_version(v.as_string());
562: res=f_curl_easy_setopt(curl(), opt->id, value_int);
563: break;
564: }
1.1 misha 565: case CurlOption::PARSER_LIBRARY:{
566: // 'library' parser option
1.41 moko 567: if(!curl_linked){
1.1 misha 568: curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC);
1.75 moko 569: if(!curl_library[0])
570: curl_library=CURL_LIBRARY;
1.1 misha 571: } else
1.16 moko 572: throw Exception("curl", 0, "failed to set option '%s': already loaded", key.cstr());
1.1 misha 573: break;
574: }
575: case CurlOption::PARSER_NAME:{
576: // 'name' parser option
1.12 misha 577: options().filename=&v.as_string();
1.1 misha 578: break;
579: }
580: case CurlOption::PARSER_CONTENT_TYPE:{
581: // 'content-type' parser option
582: options().content_type=&v.as_string();
583: break;
584: }
585: case CurlOption::PARSER_MODE:{
586: // 'mode' parser option
1.12 misha 587: options().is_text=VFile::is_text_mode(v.as_string());
1.1 misha 588: break;
589: }
590: case CurlOption::PARSER_CHARSET:{
1.30 moko 591: // 'charset' parser option should be processed before other options
1.1 misha 592: break;
593: }
1.4 misha 594: case CurlOption::PARSER_RESPONSE_CHARSET:{
1.12 misha 595: // 'response-charset' parser option
1.42 moko 596: options().response_charset=&pa_charsets.get(v.as_string());
1.4 misha 597: break;
598: }
1.1 misha 599: }
600:
601: if(res != CURLE_OK)
602: throw Exception("curl", 0, "failed to set option '%s': %s", key.cstr(), f_curl_easy_strerror(res));
603: }
604:
1.3 misha 605: static void _curl_options(Request& r, MethodParams& params){
1.1 misha 606: if(curl_options==0)
607: curl_options=new CurlOptionHash();
608:
1.30 moko 609: if(HashStringValue* options_hash=params.as_hash(0)){
610: if(Value* value=options_hash->get("charset")){
611: // charset should be handled first as params may require transcode
1.44 moko 612: Value &v=r.process(*value);
1.42 moko 613: options().charset=&pa_charsets.get(v.as_string());
1.30 moko 614: }
615: options_hash->for_each<Request&>(curl_setopt, r);
616: }
1.1 misha 617: }
618:
1.32 moko 619: #define CURL_GETINFO(arg) \
620: if((res=f_curl_easy_getinfo(curl(), info->id, &arg)) != CURLE_OK){ \
1.58 moko 621: if (fail_on_error) \
622: throw Exception("curl", 0, "failed to get %s info: %s", key.cstr(), f_curl_easy_strerror(res)); \
623: return 0; \
1.32 moko 624: }
625:
1.58 moko 626: static Value *curl_getinfo(const String::Body &key, CurlInfo *info, bool fail_on_error=false) {
1.32 moko 627: CURLcode res;
628: switch (info->type){
629: case CurlInfo::CURL_STRING:{
630: char *str=0;
631: CURL_GETINFO(str);
1.33 moko 632: return new VString(str ? *new String(pa_strdup(str), String::L_TAINTED) : String::Empty);
1.32 moko 633: }
634: case CurlInfo::CURL_INT:{
635: long l=0;
636: CURL_GETINFO(l);
637: return new VInt(l);
638: }
639: case CurlInfo::CURL_DOUBLE:{
640: double d=0;
641: CURL_GETINFO(d);
642: return new VDouble(d);
643: }
1.56 moko 644: case CurlInfo::CURL_HTTP_VERSION:{
645: long l=0;
646: CURL_GETINFO(l);
1.77 moko 647: return new VString(curl_http_version_name(l));
1.56 moko 648: }
1.32 moko 649: }
650: return VVoid::get();
651: }
652:
653: static void _curl_info(Request& r, MethodParams& params){
654: if(curl_infos==0)
655: curl_infos=new CurlInfoHash();
656: if(params.count()==1){
657: const String &name=params.as_string(0, "name must be string");
1.58 moko 658: CurlInfo *info=curl_infos->get(name);
659: if(info==0)
660: throw Exception("curl", 0, "called with invalid parameter '%s'", name.cstr());
1.59 moko 661: r.write(*curl_getinfo(name, info, true));
1.32 moko 662: } else {
663: VHash& result=*new VHash;
664: for(CurlInfoHash::Iterator i(*curl_infos); i; i.next() ){
1.59 moko 665: Value *value=curl_getinfo(i.key(), i.value());
1.58 moko 666: if(value)
667: result.get_hash()->put(i.key(), value);
1.32 moko 668: }
1.46 moko 669: r.write(result);
1.32 moko 670: }
671: }
672:
1.1 misha 673: class Curl_buffer{
674: public:
675: char *buf;
676: size_t length;
677: size_t buf_size;
1.62 moko 678: HTTP_Headers& headers;
1.1 misha 679:
1.62 moko 680: Curl_buffer(HTTP_Headers& aheaders) : buf((char *)pa_malloc_atomic(MAX_STRING)), length(0), buf_size(MAX_STRING-1), headers(aheaders){}
1.50 moko 681:
682: void resize(size_t size){
683: buf_size=size;
684: buf=(char *)pa_realloc(buf, size+1);
685: }
1.1 misha 686: };
687:
688: static int curl_writer(char *data, size_t size, size_t nmemb, Curl_buffer *result){
689: if(result == 0)
690: return 0;
691:
692: size=size*nmemb;
693: if(size>0){
1.50 moko 694: size_t buf_required = result->length + size;
695: if(buf_required > result->buf_size)
1.51 moko 696: result->resize(buf_required <= result->headers.content_length ? (size_t)result->headers.content_length : result->buf_size*2 + size);
1.1 misha 697: memcpy(result->buf+result->length, data, size);
698: result->length += size;
699: }
700: return size;
701: }
702:
1.62 moko 703: static int curl_header(char *data, size_t size, size_t nmemb, HTTP_Headers *result){
1.1 misha 704: if(result == 0)
705: return 0;
706:
707: size=size*nmemb;
708: if(size>0){
1.47 moko 709: char *header=pa_strdup(data, size);
710: if(!pa_strncasecmp(header, "HTTP/") && !strchr(header, ':')){
711: // response code, clearing possible headers from previous requests
712: result->clear();
713: } else {
714: result->add_header(header);
1.60 moko 715: if(result->content_length>pa_file_size_limit && !options().no_body)
1.50 moko 716: return 0;
1.47 moko 717: }
1.1 misha 718: }
719: return size;
720: }
721:
722: #define CURL_SETOPT(option, arg, message) \
723: if( (res=f_curl_easy_setopt(curl(), option, arg)) != CURLE_OK){ \
724: throw Exception("curl", 0, "failed to set " message ": %s", f_curl_easy_strerror(res)); \
725: }
726:
727: static void _curl_load_action(Request& r, MethodParams& params){
728: if(params.count()==1)
1.3 misha 729: _curl_options(r, params);
1.1 misha 730:
731: CURLcode res;
732:
733: // we need a container for headers as VFile fields can be put only after VFile.set
1.62 moko 734: HTTP_Headers response;
1.1 misha 735: CURL_SETOPT(CURLOPT_HEADERFUNCTION, curl_header, "curl header function");
1.24 misha 736: CURL_SETOPT(CURLOPT_WRITEHEADER, &response, "curl header buffer");
1.1 misha 737:
1.50 moko 738: Curl_buffer body(response);
739: CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function");
740: CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer");
741:
1.57 moko 742: if(options().is_post && !options().has_content_length){
743: // libcurl bug walkaround. Prior to 7.38 (Debian Jessie) curl passed Content-length: -1
744: // after that no Content-length header is passed, that hangs request to nginx.
745: CURL_SETOPT(CURLOPT_POSTFIELDSIZE, 0, "post content-length");
746: }
747:
1.63 moko 748: ALTER_EXCEPTION_SOURCE(res=f_curl_easy_perform(curl()), new String(options().url));
749: if(res != CURLE_OK){
1.15 moko 750: const char *ex_type = 0;
1.4 misha 751: switch(res){
752: case CURLE_OPERATION_TIMEDOUT:
753: ex_type = "curl.timeout"; break;
754: case CURLE_COULDNT_RESOLVE_HOST:
755: ex_type = "curl.host"; break;
756: case CURLE_COULDNT_CONNECT:
757: ex_type = "curl.connect"; break;
758: case CURLE_HTTP_RETURNED_ERROR:
759: ex_type = "curl.status"; break;
760: case CURLE_SSL_CONNECT_ERROR:
761: case CURLE_SSL_CERTPROBLEM:
762: case CURLE_SSL_CIPHER:
763: case CURLE_SSL_CACERT:
764: case CURLE_SSL_ENGINE_INITFAILED:
765: ex_type = "curl.ssl"; break;
1.50 moko 766: case CURLE_WRITE_ERROR:
1.64 moko 767: check_file_size(response.content_length, new String(options().url)); break;
1.31 moko 768: default: break;
1.4 misha 769: }
1.63 moko 770: throw Exception( PA_DEFAULT(ex_type, "curl.fail"), new String(options().url), "%s", f_curl_easy_strerror(res));
1.1 misha 771: }
772:
773: // assure trailing zero
774: body.buf[body.length]=0;
775:
1.4 misha 776: VFile& result=*new VFile;
777:
1.8 moko 778: Charset *asked_charset = options().response_charset;
1.38 moko 779: if (!asked_charset && !response.content_type.is_empty())
780: asked_charset=detect_charset(response.content_type.cstr());
781:
782: if(options().is_text)
1.42 moko 783: asked_charset=pa_charsets.checkBOM(body.buf, body.length, asked_charset);
1.38 moko 784:
785: if (!asked_charset)
786: asked_charset = options().charset;
1.1 misha 787:
788: if(options().is_text && asked_charset != 0){
789: String::C c=Charset::transcode(String::C(body.buf, body.length), *asked_charset, r.charsets.source());
790: body.buf=(char *)c.str;
791: body.length=c.length;
792: }
793:
1.38 moko 794: const String *content_type = PA_DEFAULT(options().content_type, response.content_type.is_empty() ? 0 : new String(response.content_type, String::L_TAINTED));
1.39 moko 795: const String *filename = PA_DEFAULT(options().filename, new String(options().url));
1.38 moko 796:
1.39 moko 797: result.set(true/*tainted*/, options().is_text, body.buf, body.length, filename, content_type ? new VString(*content_type) : 0, &r);
1.38 moko 798:
1.4 misha 799: long http_status = 0;
800: if(f_curl_easy_getinfo(curl(), CURLINFO_RESPONSE_CODE, &http_status) == CURLE_OK){
1.76 moko 801: HASH_PUT_CSTR(result.fields(), "status", new VInt(http_status));
1.4 misha 802: }
803:
1.38 moko 804: VHash* vtables=new VHash;
1.76 moko 805: HASH_PUT_CSTR(result.fields(), "tables", vtables);
1.23 misha 806:
1.71 moko 807: for(Array_iterator<HTTP_Headers::Header> i(response.headers); i; ){
1.62 moko 808: HTTP_Headers::Header header=i.next();
1.24 misha 809:
810: if(asked_charset)
1.38 moko 811: header.transcode(*asked_charset, r.charsets.source());
812:
813: String &header_value=*new String(header.value, String::L_TAINTED);
814:
815: tables_update(vtables->hash(), header.name, header_value);
816: result.fields().put(header.name, new VString(header_value));
1.1 misha 817: }
818:
1.38 moko 819: // filling $.cookies
820: if(Value *vcookies=vtables->hash().get("SET-COOKIE"))
1.76 moko 821: HASH_PUT_CSTR(result.fields(), HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table())));
1.23 misha 822:
1.46 moko 823: r.write(result);
1.1 misha 824: }
825:
826: static void _curl_load(Request& r, MethodParams& params){
827: fcurl ? _curl_load_action(r, params) : temp_curl(_curl_load_action, r, params);
828: }
829:
830: // constructor
831: MCurl::MCurl(): Methoded("curl") {
832: add_native_method("session", Method::CT_STATIC, _curl_session, 1, 1);
1.4 misha 833: add_native_method("version", Method::CT_STATIC, _curl_version, 0, 0);
1.3 misha 834: add_native_method("options", Method::CT_STATIC, _curl_options, 1, 1);
1.32 moko 835: add_native_method("info", Method::CT_STATIC, _curl_info, 0, 1);
1.1 misha 836: add_native_method("load", Method::CT_STATIC, _curl_load, 0, 1);
1.3 misha 837: }
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to curl.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / classes