--- parser3/src/classes/curl.C 2015/10/26 01:21:53 1.35 +++ parser3/src/classes/curl.C 2016/10/25 23:37:52 1.45 @@ -17,7 +17,7 @@ #include "pa_http.h" #include "ltdl.h" -volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.35 2015/10/26 01:21:53 moko Exp $"; +volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.45 2016/10/25 23:37:52 moko Exp $"; class MCurl: public Methoded { public: @@ -26,7 +26,7 @@ public: // global variables -DECLARE_CLASS_VAR(curl, new MCurl, 0); +DECLARE_CLASS_VAR(curl, new MCurl); #include "curl.h" @@ -71,14 +71,18 @@ static const char *dlink(const char *dlo class ParserOptions { public: + // real options const String *filename; const String *content_type; bool is_text; Charset *charset, *response_charset; + + // stuff for internal use + const char *url; struct curl_httppost *f_post; FILE *f_stderr; - ParserOptions() : filename(0), content_type(0), is_text(true), charset(0), response_charset(0), f_post(0), f_stderr(0){} + ParserOptions() : filename(0), content_type(0), is_text(true), charset(0), response_charset(0), url(0), f_post(0), f_stderr(0){} ~ParserOptions() { f_curl_formfree(f_post); if(f_stderr) @@ -123,17 +127,15 @@ public: }; bool curl_linked = false; -const char *curl_library="libcurl" LT_MODULE_EXT; - const char *curl_status = 0; +const char *curl_library="libcurl" LT_MODULE_EXT; static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){ - if(!curl_linked){ - curl_linked=true; + if(!curl_linked) curl_status=dlink(curl_library); - } if(curl_status == 0){ + curl_linked=true; Temp_curl temp_curl; action(r,params); } else { @@ -158,20 +160,6 @@ static void _curl_version(Request& r, Me fcurl ? _curl_version_action(r, params) : temp_curl(_curl_version_action, r, params); } -static char *str_lower(const char *str){ - char *result=pa_strdup(str); - for(char* c=result; *c; c++) - *c=(char)tolower((unsigned char)*c); - return result; -} - -static char *str_upper(const char *str){ - char *result=pa_strdup(str); - for(char* c=result; *c; c++) - *c=(char)toupper((unsigned char)*c); - return result; -} - class CurlOption { public: @@ -288,6 +276,7 @@ public: CURL_OPT(CURL_INT, SSL_VERIFYHOST); CURL_OPT(CURL_STRING, SSL_CIPHER_LIST); CURL_OPT(CURL_INT, SSL_SESSIONID_CACHE); + CURL_OPT(CURL_INT, SSLVERSION); PARSER_OPT(PARSER_LIBRARY, "library"); PARSER_OPT(PARSER_NAME, "name"); @@ -421,7 +410,7 @@ static void curl_setopt(HashStringValue: throw Exception("curl", 0, "called with invalid option '%s'", key.cstr()); CURLcode res = CURLE_OK; - Value &v=r.process_to_value(*value); + Value &v=r.process(*value); switch (opt->type){ case CurlOption::CURL_STRING:{ @@ -441,8 +430,8 @@ static void curl_setopt(HashStringValue: const String url = v.as_string(); if(!url.starts_with("http://") && !url.starts_with("https://")) throw Exception("curl", 0, "failed to set option '%s': invalid url scheme '%s'", key.cstr(), url.cstr()); - const char *value_str=curl_urlencode(url, r); - res=f_curl_easy_setopt(curl(), opt->id, value_str); + options().url=curl_urlencode(url, r); + res=f_curl_easy_setopt(curl(), opt->id, options().url); break; } case CurlOption::CURL_INT:{ @@ -501,7 +490,7 @@ static void curl_setopt(HashStringValue: } case CurlOption::PARSER_LIBRARY:{ // 'library' parser option - if(fcurl==0){ + if(!curl_linked){ curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC); } else throw Exception("curl", 0, "failed to set option '%s': already loaded", key.cstr()); @@ -528,7 +517,7 @@ static void curl_setopt(HashStringValue: } case CurlOption::PARSER_RESPONSE_CHARSET:{ // 'response-charset' parser option - options().response_charset=&::charsets.get(v.as_string().change_case(r.charsets.source(), String::CC_UPPER)); + options().response_charset=&pa_charsets.get(v.as_string()); break; } } @@ -544,8 +533,8 @@ static void _curl_options(Request& r, Me if(HashStringValue* options_hash=params.as_hash(0)){ if(Value* value=options_hash->get("charset")){ // charset should be handled first as params may require transcode - Value &v=r.process_to_value(*value); - options().charset=&::charsets.get(v.as_string().change_case(r.charsets.source(), String::CC_UPPER)); + Value &v=r.process(*value); + options().charset=&pa_charsets.get(v.as_string()); } options_hash->for_each(curl_setopt, r); } @@ -586,7 +575,7 @@ static void _curl_info(Request& r, Metho curl_infos=new CurlInfoHash(); if(params.count()==1){ const String &name=params.as_string(0, "name must be string"); - r.write_assign_lang(*curl_getinfo(name)); + r.write_pass_lang(*curl_getinfo(name)); } else { VHash& result=*new VHash; for(CurlInfoHash::Iterator i(*curl_infos); i; i.next() ){ @@ -622,27 +611,13 @@ static int curl_writer(char *data, size_ return size; } -class Curl_response { -public: - HASH_STRING headers; - Array cookies; -}; - -static int curl_header(char *data, size_t size, size_t nmemb, Curl_response *result){ +static int curl_header(char *data, size_t size, size_t nmemb, ResponseHeaders *result){ if(result == 0) return 0; size=size*nmemb; if(size>0){ - char *line=pa_strdup(data, size); - char *value=lsplit(line,':'); - if(value && *line){ - // we need only headers, not the response code - const char* HEADER_NAME=str_upper(line); - result->headers.put(HEADER_NAME, value); - if(strcmp(HEADER_NAME, "SET-COOKIE")==0) - result->cookies+=value; - } + result->add_header(pa_strdup(data, size)); } return size; } @@ -663,7 +638,7 @@ static void _curl_load_action(Request& r CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer"); // we need a container for headers as VFile fields can be put only after VFile.set - Curl_response response; + ResponseHeaders response; CURL_SETOPT(CURLOPT_HEADERFUNCTION, curl_header, "curl header function"); CURL_SETOPT(CURLOPT_WRITEHEADER, &response, "curl header buffer"); @@ -686,7 +661,7 @@ static void _curl_load_action(Request& r ex_type = "curl.ssl"; break; default: break; } - throw Exception( ex_type ? ex_type : "curl.fail", 0, "%s", f_curl_easy_strerror(res)); + throw Exception( PA_DEFAULT(ex_type, "curl.fail"), 0, "%s", f_curl_easy_strerror(res)); } // assure trailing zero @@ -694,12 +669,15 @@ static void _curl_load_action(Request& r VFile& result=*new VFile; - String::Body ct_header = response.headers.get(HTTP_CONTENT_TYPE_UPPER); Charset *asked_charset = options().response_charset; - if (asked_charset == 0){ - Charset *remote_charset = ct_header.is_empty() ? 0 : detect_charset(ct_header.trim(String::TRIM_BOTH, " \t\n\r").cstr()); - asked_charset = remote_charset ? remote_charset : options().charset; - } + if (!asked_charset && !response.content_type.is_empty()) + asked_charset=detect_charset(response.content_type.cstr()); + + if(options().is_text) + asked_charset=pa_charsets.checkBOM(body.buf, body.length, asked_charset); + + if (!asked_charset) + asked_charset = options().charset; if(options().is_text && asked_charset != 0){ String::C c=Charset::transcode(String::C(body.buf, body.length), *asked_charset, r.charsets.source()); @@ -707,42 +685,34 @@ static void _curl_load_action(Request& r body.length=c.length; } - result.set(true/*tainted*/, options().is_text, body.buf, body.length, options().filename - , options().content_type ? new VString(*options().content_type) : 0, &r); + const String *content_type = PA_DEFAULT(options().content_type, response.content_type.is_empty() ? 0 : new String(response.content_type, String::L_TAINTED)); + const String *filename = PA_DEFAULT(options().filename, new String(options().url)); + + result.set(true/*tainted*/, options().is_text, body.buf, body.length, filename, content_type ? new VString(*content_type) : 0, &r); + long http_status = 0; if(f_curl_easy_getinfo(curl(), CURLINFO_RESPONSE_CODE, &http_status) == CURLE_OK){ result.fields().put("status", new VInt(http_status)); } - for(HASH_STRING::Iterator i(response.headers); i; i.next() ){ - String::Body HEADER_NAME=i.key(); - String::Body value=i.value(); - if(asked_charset){ - HEADER_NAME=Charset::transcode(HEADER_NAME, *asked_charset, r.charsets.source()); - value=Charset::transcode(value, *asked_charset, r.charsets.source()); - } - result.fields().put(HEADER_NAME, new VString(*new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED))); - } + VHash* vtables=new VHash; + result.fields().put("tables", vtables); - // filling $.cookies - Table* tcookies=0; + for(Array_iterator i(response.headers); i.has_next(); ){ + ResponseHeaders::Header header=i.next(); - for(Array_iterator i(response.cookies); i.has_next(); ){ - if(!tcookies){ - Table::columns_type columns=new ArrayString(1); - *columns+=new String("value"); - tcookies=new Table(columns); - } - String::Body value=i.next(); if(asked_charset) - value=Charset::transcode(value, *asked_charset, r.charsets.source()); - ArrayString& row=*new ArrayString(1); - row+=new String(value.trim(String::TRIM_BOTH, " \t\n\r"), String::L_TAINTED); - *tcookies+=&row; + header.transcode(*asked_charset, r.charsets.source()); + + String &header_value=*new String(header.value, String::L_TAINTED); + + tables_update(vtables->hash(), header.name, header_value); + result.fields().put(header.name, new VString(header_value)); } - if(tcookies) - result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, tcookies))); + // filling $.cookies + if(Value *vcookies=vtables->hash().get("SET-COOKIE")) + result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table()))); r.write_no_lang(result); }