--- parser3/src/classes/curl.C 2013/03/10 23:36:00 1.23 +++ parser3/src/classes/curl.C 2014/06/27 22:25:11 1.30 @@ -16,7 +16,7 @@ #include "pa_http.h" #include "ltdl.h" -volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.23 2013/03/10 23:36:00 misha Exp $"; +volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.30 2014/06/27 22:25:11 moko Exp $"; class MCurl: public Methoded { public: @@ -44,8 +44,7 @@ typedef void (*t_curl_formfree)(struct c #define DLINK(name) GLINK(name) if(!f_##name) return "function " #name " was not found"; static const char *dlink(const char *dlopen_file_spec) { - if(lt_dlinit()) - return lt_dlerror(); + pa_dlinit(); lt_dlhandle handle=lt_dlopen(dlopen_file_spec); @@ -112,6 +111,7 @@ public: fcurl = f_curl_easy_init(); foptions = new ParserOptions(); f_curl_easy_setopt(fcurl, CURLOPT_POSTFIELDSIZE, 0); // fix libcurl bug + f_curl_easy_setopt(fcurl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); // avoid ipv6 by default } ~Temp_curl() { f_curl_easy_cleanup(fcurl); @@ -122,11 +122,7 @@ public: }; bool curl_linked = false; -#ifdef WIN32 -const char *curl_library="libcurl.dll"; -#else -const char *curl_library="libcurl.so"; -#endif +const char *curl_library="libcurl" LT_MODULE_EXT; const char *curl_status = 0; @@ -224,9 +220,13 @@ public: CURL_OPT(CURL_URLENCODE, USERAGENT); CURL_OPT(CURL_URLENCODE, REFERER); CURL_OPT(CURL_INT, AUTOREFERER); + CURL_OPT(CURL_STRING, ENCODING); // gzip or deflate + CURL_OPT(CURL_STRING, ACCEPT_ENCODING); // gzip or deflate + CURL_OPT(CURL_INT, FOLLOWLOCATION); CURL_OPT(CURL_INT, UNRESTRICTED_AUTH); + CURL_OPT(CURL_INT, IPRESOLVE); CURL_OPT(CURL_INT, POST); CURL_OPT(CURL_INT, HTTPGET); @@ -474,8 +474,7 @@ static void curl_setopt(HashStringValue: break; } case CurlOption::PARSER_CHARSET:{ - // 'charset' parser option - options().charset=&::charsets.get(v.as_string().change_case(r.charsets.source(), String::CC_UPPER)); + // 'charset' parser option should be processed before other options break; } case CurlOption::PARSER_RESPONSE_CHARSET:{ @@ -493,8 +492,14 @@ static void _curl_options(Request& r, Me if(curl_options==0) curl_options=new CurlOptionHash(); - if(HashStringValue* options=params.as_hash(0)) - options->for_each(curl_setopt, r); + if(HashStringValue* options_hash=params.as_hash(0)){ + if(Value* value=options_hash->get("charset")){ + // charset should be handled first as params may require transcode + Value &v=r.process_to_value(*value); + options().charset=&::charsets.get(v.as_string().change_case(r.charsets.source(), String::CC_UPPER)); + } + options_hash->for_each(curl_setopt, r); + } } @@ -523,7 +528,13 @@ static int curl_writer(char *data, size_ return size; } -static int curl_header(char *data, size_t size, size_t nmemb, HASH_STRING *result){ +class Curl_response { +public: + HASH_STRING headers; + Array cookies; +}; + +static int curl_header(char *data, size_t size, size_t nmemb, Curl_response *result){ if(result == 0) return 0; @@ -533,7 +544,10 @@ static int curl_header(char *data, size_ char *value=lsplit(line,':'); if(value && *line){ // we need only headers, not the response code - result->put(str_upper(line), value); + const char* HEADER_NAME=str_upper(line); + result->headers.put(HEADER_NAME, value); + if(strcmp(HEADER_NAME, "SET-COOKIE")==0) + result->cookies+=value; } } return size; @@ -555,9 +569,9 @@ static void _curl_load_action(Request& r CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer"); // we need a container for headers as VFile fields can be put only after VFile.set - HASH_STRING headers; + Curl_response response; CURL_SETOPT(CURLOPT_HEADERFUNCTION, curl_header, "curl header function"); - CURL_SETOPT(CURLOPT_WRITEHEADER, &headers, "curl header buffer"); + CURL_SETOPT(CURLOPT_WRITEHEADER, &response, "curl header buffer"); if((res=f_curl_easy_perform(curl())) != CURLE_OK){ const char *ex_type = 0; @@ -585,7 +599,7 @@ static void _curl_load_action(Request& r VFile& result=*new VFile; - String::Body ct_header = headers.get(HTTP_CONTENT_TYPE_UPPER); + String::Body ct_header = response.headers.get(HTTP_CONTENT_TYPE_UPPER); Charset *asked_charset = options().response_charset; if (asked_charset == 0){ Charset *remote_charset = ct_header.is_empty() ? 0 : detect_charset(ct_header.trim(String::TRIM_BOTH, " \t\n\r").cstr()); @@ -605,33 +619,35 @@ static void _curl_load_action(Request& r result.fields().put("status", new VInt(http_status)); } - Table *cookies=0; - for(HASH_STRING::Iterator i(headers); i; i.next() ){ + for(HASH_STRING::Iterator i(response.headers); i; i.next() ){ String::Body HEADER_NAME=i.key(); String::Body value=i.value(); if(asked_charset){ HEADER_NAME=Charset::transcode(HEADER_NAME, *asked_charset, r.charsets.source()); value=Charset::transcode(value, *asked_charset, r.charsets.source()); } - const String& header_value=*new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED); - result.fields().put(HEADER_NAME, new VString(header_value)); + result.fields().put(HEADER_NAME, new VString(*new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED))); + } - if(HEADER_NAME == "SET-COOKIE") { - if(!cookies){ - // first appearence - Table::columns_type columns=new ArrayString(1); - *columns+=new String("value"); - cookies=new Table(columns); - } - ArrayString& row=*new ArrayString(1); - row+=&header_value; - *cookies+=&row; + // filling $.cookies + Table* tcookies=0; + + for(Array_iterator i(response.cookies); i.has_next(); ){ + if(!tcookies){ + Table::columns_type columns=new ArrayString(1); + *columns+=new String("value"); + tcookies=new Table(columns); } + String::Body value=i.next(); + if(asked_charset) + value=Charset::transcode(value, *asked_charset, r.charsets.source()); + ArrayString& row=*new ArrayString(1); + row+=new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED); + *tcookies+=&row; } - // filling $.cookies - if(cookies) - result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, cookies))); + if(tcookies) + result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, tcookies))); r.write_no_lang(result); }