Annotation of parser3/src/classes/curl.C, revision 1.3
1.1 misha 1: /** @file
2: Parser: @b curl parser class.
3:
4: Copyright(c) 2001-2009 ArtLebedev Group(http://www.artlebedev.com)
5: */
6:
1.3 ! misha 7: #include "pa_config_includes.h"
! 8:
! 9: #ifdef HAVE_CURL
! 10:
! 11: static const char * const IDENT_INET_C="$Date: 2009-12-04 22:22:13 $";
1.1 misha 12:
13: #include "pa_vmethod_frame.h"
14: #include "pa_request.h"
15: #include "pa_vfile.h"
16: #include "pa_charsets.h"
17: #include "ltdl.h"
18:
19: class MCurl: public Methoded {
20: public:
21: MCurl();
22:
23: public: // Methoded
24: bool used_directly() { return true; }
25: };
26:
27: // global variables
28:
29: DECLARE_CLASS_VAR(curl, new MCurl, 0);
30:
31: // from file.C
32: extern bool is_text_mode(const String& mode);
33:
34: #include "curl.h"
35:
36: typedef CURL *(*t_curl_easy_init)(); t_curl_easy_init f_curl_easy_init;
37: typedef CURLcode (*t_curl_easy_setopt)(CURL *, CURLoption option, ...); t_curl_easy_setopt f_curl_easy_setopt;
38: typedef CURLcode (*t_curl_easy_perform)(CURL *); t_curl_easy_perform f_curl_easy_perform;
39: typedef void (*t_curl_easy_cleanup)(CURL *); t_curl_easy_cleanup f_curl_easy_cleanup;
40: typedef const char *(*t_curl_easy_strerror)(CURLcode); t_curl_easy_strerror f_curl_easy_strerror;
41: typedef struct curl_slist *(*t_curl_slist_append)(struct curl_slist *,const char *); t_curl_slist_append f_curl_slist_append;
42:
43: #define GLINK(name) f_##name=(t_##name)lt_dlsym(handle, #name);
44: #define DLINK(name) GLINK(name) if(!f_##name) return "function " #name " was not found";
45:
46: const char *dlink(const char *dlopen_file_spec) {
47: if(lt_dlinit())
48: return lt_dlerror();
49:
50: lt_dlhandle handle=lt_dlopen(dlopen_file_spec);
51:
52: if(!handle){
53: if(const char* result=lt_dlerror())
54: return result;
55: return "can not open the dynamic link module";
56: }
57:
58: DLINK(curl_easy_init);
59: DLINK(curl_easy_cleanup);
60: DLINK(curl_easy_setopt);
61: DLINK(curl_easy_perform);
62: DLINK(curl_easy_strerror);
63: DLINK(curl_slist_append);
64: return 0;
65: }
66:
67:
68: class ParserOptions {
69: public:
70: const char *filename;
71: const String *content_type;
72: bool is_text;
73: Charset *charset;
74:
75: ParserOptions() : filename(0), content_type(0), is_text(true), charset(0) {}
76: };
77:
78: // using thread local variables instead of keeping them in request
79: // not necessary for cgi version
80: #ifdef WIN32
81: #define __thread __declspec(thread)
82: #endif
83: __thread CURL *fcurl=0;
84: __thread ParserOptions *foptions;
85:
86: static CURL *curl(){
87: if(!fcurl)
88: throw Exception("curl", 0, "outside of 'session' operator");
89: return fcurl;
90: }
91:
92: static ParserOptions &options(){
93: if(!foptions)
94: throw Exception("curl", 0, "outside of 'session' operator");
95: return *foptions;
96: }
97:
98: // using temporal object scheme to garanty cleanup call
99: class Temp_curl {
100: CURL* saved_curl;
101: ParserOptions* saved_options;
102: public:
103: Temp_curl() : saved_curl(fcurl) {
104: fcurl = f_curl_easy_init();
105: foptions = new ParserOptions();
106: }
107: ~Temp_curl() {
108: f_curl_easy_cleanup(fcurl);
109: fcurl = saved_curl;
110: delete foptions;
111: foptions = saved_options;
112: }
113: };
114:
115: bool curl_linked = false;
116: #ifdef WIN32
117: const char *curl_library="libcurl.dll";
118: #else
119: const char *curl_library="libcurl.so";
120: #endif
121:
122: const char *curl_status = 0;
123:
124: static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){
125: if(!curl_linked){
126: curl_linked=true;
127: curl_status=dlink(curl_library);
128: }
129:
130: if(curl_status == 0){
131: Temp_curl temp_curl;
132: action(r,params);
133: } else {
134: throw Exception("curl", 0, "failed to load curl library %s: %s", curl_library, curl_status);
135: }
136: }
137:
138: static void _curl_session_action(Request& r, MethodParams& params){
139: Value& body_code=params.as_junction(0, "body must be code");
140: r.process_write(body_code);
141: }
142:
143: static void _curl_session(Request& r, MethodParams& params){
144: temp_curl(_curl_session_action, r, params);
145: }
146:
147:
148: static char *str_lower(const char *str){
149: char *result=pa_strdup(str);
150: for(char* c=result; *c; c++)
151: *c=(char)tolower((unsigned char)*c);
152: return result;
153: }
154:
155: static char *str_upper(const char *str){
156: char *result=pa_strdup(str);
157: for(char* c=result; *c; c++)
158: *c=(char)toupper((unsigned char)*c);
159: return result;
160: }
161:
162: class CurlOption {
163: public:
164:
165: enum OptionType {
166: CURL_STRING,
167: CURL_URLENCODE, // url-encoded string
168: CURL_INT,
169: CURL_POST,
170: CURL_HEADERS,
171: CURL_FILE,
172: PARSER_LIBRARY,
173: PARSER_NAME,
174: PARSER_CONTENT_TYPE,
175: PARSER_MODE,
176: PARSER_CHARSET
177: };
178:
179: CURLoption id;
180: OptionType type;
181: CurlOption(CURLoption aid, OptionType atype): id(aid), type(atype) {}
182: };
183:
184: class CurlOptionHash: public HashString<CurlOption*> {
185: public:
186: CurlOptionHash() {
187: #define CURL_OPT(type, name) put(str_lower(#name),new CurlOption(CURLOPT_##name, CurlOption::type));
188: #define PARSER_OPT(type, name) put(name,new CurlOption((CURLoption)0, CurlOption::type));
189: CURL_OPT(CURL_URLENCODE, URL);
190: CURL_OPT(CURL_STRING, INTERFACE);
191: CURL_OPT(CURL_INT, LOCALPORT);
192: CURL_OPT(CURL_INT, PORT);
193:
194: CURL_OPT(CURL_INT, HTTPAUTH);
195: CURL_OPT(CURL_STRING, USERPWD);
196:
1.2 misha 197: #ifdef CURLOPT_USERNAME
1.1 misha 198: CURL_OPT(CURL_STRING, USERNAME);
199: CURL_OPT(CURL_STRING, PASSWORD);
1.2 misha 200: #endif
1.1 misha 201:
202: CURL_OPT(CURL_INT, AUTOREFERER);
203: CURL_OPT(CURL_STRING, ENCODING); // gzip or deflate
204: CURL_OPT(CURL_INT, FOLLOWLOCATION);
205: CURL_OPT(CURL_INT, UNRESTRICTED_AUTH);
206:
207: CURL_OPT(CURL_INT, POST);
208: CURL_OPT(CURL_INT, HTTPGET);
209:
210: CURL_OPT(CURL_POST, POSTFIELDS); // hopefully is safe too
211: CURL_OPT(CURL_POST, COPYPOSTFIELDS);
212:
213: CURL_OPT(CURL_HEADERS, HTTPHEADER);
214: CURL_OPT(CURL_URLENCODE, COOKIE);
215: CURL_OPT(CURL_URLENCODE, COOKIELIST);
216: CURL_OPT(CURL_INT, COOKIESESSION);
217:
218: CURL_OPT(CURL_INT, IGNORE_CONTENT_LENGTH);
219: CURL_OPT(CURL_INT, HTTP_CONTENT_DECODING);
220: CURL_OPT(CURL_INT, HTTP_TRANSFER_DECODING);
221:
222: CURL_OPT(CURL_INT, TIMEOUT);
223: CURL_OPT(CURL_INT, TIMEOUT_MS);
224: CURL_OPT(CURL_INT, LOW_SPEED_LIMIT);
225: CURL_OPT(CURL_INT, LOW_SPEED_TIME);
226: CURL_OPT(CURL_INT, MAXCONNECTS);
227:
228: CURL_OPT(CURL_INT, FRESH_CONNECT);
229: CURL_OPT(CURL_INT, FORBID_REUSE);
230: CURL_OPT(CURL_INT, CONNECTTIMEOUT);
231: CURL_OPT(CURL_INT, CONNECTTIMEOUT_MS);
232:
233: CURL_OPT(CURL_FILE, SSLCERT);
234: CURL_OPT(CURL_STRING, SSLCERTTYPE);
235: CURL_OPT(CURL_FILE, SSLKEY);
236: CURL_OPT(CURL_STRING, SSLKEYTYPE);
237: CURL_OPT(CURL_STRING, KEYPASSWD);
238: CURL_OPT(CURL_STRING, SSLENGINE);
239: CURL_OPT(CURL_STRING, SSLENGINE_DEFAULT);
240:
1.2 misha 241: #ifdef CURLOPT_ISSUERCERT
1.1 misha 242: CURL_OPT(CURL_FILE, ISSUERCERT);
1.2 misha 243: #endif
244:
245: #ifdef CURLOPT_CRLFILE
1.1 misha 246: CURL_OPT(CURL_FILE, CRLFILE);
1.2 misha 247: #endif
1.1 misha 248:
249: CURL_OPT(CURL_STRING, CAINFO);
250: CURL_OPT(CURL_STRING, CAPATH);
251: CURL_OPT(CURL_INT, SSL_VERIFYPEER);
252: CURL_OPT(CURL_INT, SSL_VERIFYHOST);
253: CURL_OPT(CURL_STRING, SSL_CIPHER_LIST);
254: CURL_OPT(CURL_INT, SSL_SESSIONID_CACHE);
255:
256: PARSER_OPT(PARSER_LIBRARY, "library");
257: PARSER_OPT(PARSER_NAME, "name");
258: PARSER_OPT(PARSER_CONTENT_TYPE, "content-type");
259: PARSER_OPT(PARSER_MODE, "mode");
260: PARSER_OPT(PARSER_CHARSET, "charset");
261: }
262:
263: } *curl_options=0;
264:
265: static const char *curl_urlencode(const String &s, Request& r){
266: if(options().charset){
267: Temp_client_charset temp(r.charsets, *options().charset);
268: return s.untaint_and_transcode_cstr(String::L_URI, &r.charsets);
269: } else
270: return s.untaint_cstr(String::L_URI);
271: }
272:
273: static struct curl_slist *curl_headers(HashStringValue *value_hash, Request& r) {
274: struct curl_slist *slist=NULL;
275:
276: for(HashStringValue::Iterator i(*value_hash); i; i.next() ){
277: String header =
278: String(capitalize(i.key().cstr()), String::L_URI)
279: << ": "
280: << String(i.value()->as_string(), String::L_URI);
281:
282: slist=f_curl_slist_append(slist, curl_urlencode(header, r));
283: }
284: return slist;
285: }
286:
287: static void curl_setopt(HashStringValue::key_type key, HashStringValue::value_type value, Request& r) {
288: CurlOption *opt=curl_options->get(key);
289:
290: if(opt==0)
291: throw Exception("curl", 0, "called with invalid option '%s'", key.cstr());
292:
293: CURLcode res = CURLE_OK;
294: Value &v=r.process_to_value(*value);
295:
296: switch (opt->type){
297: case CurlOption::CURL_STRING:{
298: // string curl option
299: const char *value_str=v.as_string().cstr();
300: res=f_curl_easy_setopt(curl(), opt->id, value_str);
301: break;
302: }
303: case CurlOption::CURL_URLENCODE:{
304: // url-encoded string curl option
305: const char *value_str=curl_urlencode(v.as_string(), r);
306: res=f_curl_easy_setopt(curl(), opt->id, value_str);
307: break;
308: }
309: case CurlOption::CURL_INT:{
1.2 misha 310: // integer curl option
311: long value_int=(long)v.as_double();
1.1 misha 312: res=f_curl_easy_setopt(curl(), opt->id, value_int);
313: break;
314: }
315: case CurlOption::CURL_POST:{
316: // http post curl option
317: if(v.get_string()){
1.2 misha 318: if( (res=f_curl_easy_setopt(curl(), CURLOPT_POSTFIELDSIZE, -1L)) == CURLE_OK )
1.1 misha 319: res=f_curl_easy_setopt(curl(), opt->id, curl_urlencode(v.as_string(), r));
320: } else {
321: VFile *file=v.as_vfile(String::L_AS_IS);
1.2 misha 322: if( (res=f_curl_easy_setopt(curl(), CURLOPT_POSTFIELDSIZE, (long)file->value_size())) == CURLE_OK )
1.1 misha 323: res=f_curl_easy_setopt(curl(), opt->id, file->value_ptr());
324: }
325: break;
326: }
327: case CurlOption::CURL_HEADERS:{
328: // http headers curl option
329: HashStringValue *value_hash=v.get_hash();
330: res=f_curl_easy_setopt(curl(), opt->id, value_hash ? curl_headers(value_hash, r) : 0);
331: break;
332: }
333: case CurlOption::CURL_FILE:{
334: // file-spec curl option
335: const char *value_str=r.absolute(v.as_string()).taint_cstr(String::L_FILE_SPEC);
336: res=f_curl_easy_setopt(curl(), opt->id, value_str);
337: break;
338: }
339: case CurlOption::PARSER_LIBRARY:{
340: // 'library' parser option
341: if(fcurl==0){
342: curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC);
343: } else
344: throw Exception("curl", 0, "failed to set option '%s': %s", key.cstr(), "already loaded");
345: break;
346: }
347: case CurlOption::PARSER_NAME:{
348: // 'name' parser option
349: options().filename=v.as_string().taint_cstr(String::L_FILE_SPEC);
350: break;
351: }
352: case CurlOption::PARSER_CONTENT_TYPE:{
353: // 'content-type' parser option
354: options().content_type=&v.as_string();
355: break;
356: }
357: case CurlOption::PARSER_MODE:{
358: // 'mode' parser option
359: options().is_text=is_text_mode(v.as_string());
360: break;
361: }
362: case CurlOption::PARSER_CHARSET:{
363: // 'charset' parser option
364: options().charset=&::charsets.get(v.as_string().change_case(r.charsets.source(), String::CC_UPPER));
365: break;
366: }
367: }
368:
369: if(res != CURLE_OK)
370: throw Exception("curl", 0, "failed to set option '%s': %s", key.cstr(), f_curl_easy_strerror(res));
371: }
372:
1.3 ! misha 373: static void _curl_options(Request& r, MethodParams& params){
1.1 misha 374: if(curl_options==0)
375: curl_options=new CurlOptionHash();
376:
377: if(HashStringValue* options=params.as_no_junction(0, OPTIONS_MUST_NOT_BE_CODE).get_hash()){
378: options->for_each<Request&>(curl_setopt, r);
379: } else
380: throw Exception("curl", 0, "options must be hash");
381: }
382:
383:
384: class Curl_buffer{
385: public:
386: char *buf;
387: size_t length;
388: size_t buf_size;
389:
390: Curl_buffer() : buf((char *)pa_malloc(MAX_STRING+1)), length(0), buf_size(MAX_STRING){}
391: };
392:
393: static int curl_writer(char *data, size_t size, size_t nmemb, Curl_buffer *result){
394: if(result == 0)
395: return 0;
396:
397: size=size*nmemb;
398: if(size>0){
399: if(result->length + size >= result->buf_size){
400: result->buf_size = result->buf_size*2 + size;
401: result->buf = (char *)pa_realloc(result->buf, result->buf_size+1);
402: }
403: memcpy(result->buf+result->length, data, size);
404: result->length += size;
405: }
406: return size;
407: }
408:
409: static int curl_header(char *data, size_t size, size_t nmemb, HASH_STRING<char *> *result){
410: if(result == 0)
411: return 0;
412:
413: size=size*nmemb;
414: if(size>0){
415: char *line=pa_strdup(data, size);
416: char *value=lsplit(line,':');
417: if(value && *line){
418: // we need only headers, not the response code
419: result->put(str_upper(line), value);
420: }
421: }
422: return size;
423: }
424:
425: #define CURL_SETOPT(option, arg, message) \
426: if( (res=f_curl_easy_setopt(curl(), option, arg)) != CURLE_OK){ \
427: throw Exception("curl", 0, "failed to set " message ": %s", f_curl_easy_strerror(res)); \
428: }
429:
430: static void _curl_load_action(Request& r, MethodParams& params){
431: if(params.count()==1)
1.3 ! misha 432: _curl_options(r, params);
1.1 misha 433:
434: CURLcode res;
435:
436: Curl_buffer body;
437: CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function");
438: CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer");
439:
440: // we need a container for headers as VFile fields can be put only after VFile.set
441: HASH_STRING<char *> headers;
442: CURL_SETOPT(CURLOPT_HEADERFUNCTION, curl_header, "curl header function");
443: CURL_SETOPT(CURLOPT_WRITEHEADER, &headers, "curl header buffer");
444:
445: if((res=f_curl_easy_perform(curl())) != CURLE_OK){
446: throw Exception("curl", 0, "failed to exec curl session: %s", f_curl_easy_strerror(res));
447: }
448:
449: // assure trailing zero
450: body.buf[body.length]=0;
451:
452: Charset *asked_charset=options().charset;
453:
454: if(options().is_text && asked_charset != 0){
455: String::C c=Charset::transcode(String::C(body.buf, body.length), *asked_charset, r.charsets.source());
456: body.buf=(char *)c.str;
457: body.length=c.length;
458: }
459:
460: Value* vcontent_type=
461: options().content_type ? new VString(*options().content_type) :
462: options().filename ? new VString(r.mime_type_of(options().filename)) : 0;
463:
464: VFile& result=*new VFile;
465: result.set(true /*tainted*/, body.buf, body.length, options().filename, vcontent_type);
466: result.set_mode(options().is_text);
467:
468: for(HASH_STRING<char *>::Iterator i(headers); i; i.next() ){
469: String::Body key=i.key();
470: String::Body value=i.value();
471: if(asked_charset){
472: key=Charset::transcode(key, *asked_charset, r.charsets.source());
473: value=Charset::transcode(value, *asked_charset, r.charsets.source());
474: }
475: result.fields().put(key, new VString(*new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED)));
476: }
477:
478: r.write_no_lang(result);
479: }
480:
481: static void _curl_load(Request& r, MethodParams& params){
482: fcurl ? _curl_load_action(r, params) : temp_curl(_curl_load_action, r, params);
483: }
484:
485: // constructor
486: MCurl::MCurl(): Methoded("curl") {
487: add_native_method("session", Method::CT_STATIC, _curl_session, 1, 1);
1.3 ! misha 488: add_native_method("options", Method::CT_STATIC, _curl_options, 1, 1);
1.1 misha 489: add_native_method("load", Method::CT_STATIC, _curl_load, 0, 1);
1.3 ! misha 490: }
! 491:
! 492: #else // HAVE_CURL
! 493:
! 494: #include "classes.h"
! 495: // global variable
! 496: DECLARE_CLASS_VAR(curl, 0, 0); // fictive
! 497:
1.1 misha 498: #endif // HAVE_CURL
E-mail:
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to curl.C CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [parser3project] / parser3 / src / classes