--- parser3/src/classes/curl.C 2016/07/29 20:24:16 1.40 +++ parser3/src/classes/curl.C 2017/01/28 15:26:50 1.52 @@ -17,7 +17,7 @@ #include "pa_http.h" #include "ltdl.h" -volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.40 2016/07/29 20:24:16 moko Exp $"; +volatile const char * IDENT_CURL_C="$Id: curl.C,v 1.52 2017/01/28 15:26:50 moko Exp $"; class MCurl: public Methoded { public: @@ -127,17 +127,15 @@ public: }; bool curl_linked = false; -const char *curl_library="libcurl" LT_MODULE_EXT; - const char *curl_status = 0; +const char *curl_library="libcurl" LT_MODULE_EXT; static void temp_curl(void (*action)(Request&, MethodParams&), Request& r, MethodParams& params){ - if(!curl_linked){ - curl_linked=true; + if(!curl_linked) curl_status=dlink(curl_library); - } if(curl_status == 0){ + curl_linked=true; Temp_curl temp_curl; action(r,params); } else { @@ -155,7 +153,7 @@ static void _curl_session(Request& r, Me } static void _curl_version_action(Request& r, MethodParams& ){ - r.write_no_lang(*new VString(*new String(f_curl_version(), String::L_TAINTED))); + r.write(*new VString(*new String(f_curl_version(), String::L_TAINTED))); } static void _curl_version(Request& r, MethodParams& params){ @@ -278,6 +276,7 @@ public: CURL_OPT(CURL_INT, SSL_VERIFYHOST); CURL_OPT(CURL_STRING, SSL_CIPHER_LIST); CURL_OPT(CURL_INT, SSL_SESSIONID_CACHE); + CURL_OPT(CURL_INT, SSLVERSION); PARSER_OPT(PARSER_LIBRARY, "library"); PARSER_OPT(PARSER_NAME, "name"); @@ -399,7 +398,7 @@ static void curl_form(HashStringValue *v static const char *curl_check_file(const String &file_spec){ const char *file_spec_cstr=file_spec.taint_cstr(String::L_FILE_SPEC); struct stat finfo; - if(stat(file_spec_cstr, &finfo)==0) + if(pa_stat(file_spec_cstr, &finfo)==0) check_safe_mode(finfo, file_spec, file_spec_cstr); return file_spec_cstr; } @@ -411,7 +410,7 @@ static void curl_setopt(HashStringValue: throw Exception("curl", 0, "called with invalid option '%s'", key.cstr()); CURLcode res = CURLE_OK; - Value &v=r.process_to_value(*value); + Value &v=r.process(*value); switch (opt->type){ case CurlOption::CURL_STRING:{ @@ -454,21 +453,19 @@ static void curl_setopt(HashStringValue: break; } case CurlOption::CURL_FORM:{ - HashStringValue *value_hash = v.get_hash(); + HashStringValue *value_hash = v.as_hash("failed to set option 'httppost': value"); if(value_hash){ curl_form(value_hash, r); - } else if(v.get_string()->is_empty()){ + } else { f_curl_formfree(options().f_post); options().f_post = 0; - } else { - throw Exception("curl", 0, "failed to set option '%s': value must be a hash", key.cstr()); } res=f_curl_easy_setopt(curl(), CURLOPT_HTTPPOST, foptions->f_post); break; } case CurlOption::CURL_HEADERS:{ // http headers curl option - HashStringValue *value_hash=v.get_hash(); + HashStringValue *value_hash=v.as_hash("failed to set option 'httpheader': value"); res=f_curl_easy_setopt(curl(), opt->id, value_hash ? curl_headers(value_hash, r) : 0); break; } @@ -491,7 +488,7 @@ static void curl_setopt(HashStringValue: } case CurlOption::PARSER_LIBRARY:{ // 'library' parser option - if(fcurl==0){ + if(!curl_linked){ curl_library=v.as_string().taint_cstr(String::L_FILE_SPEC); } else throw Exception("curl", 0, "failed to set option '%s': already loaded", key.cstr()); @@ -518,7 +515,7 @@ static void curl_setopt(HashStringValue: } case CurlOption::PARSER_RESPONSE_CHARSET:{ // 'response-charset' parser option - options().response_charset=&::charsets.get(v.as_string()); + options().response_charset=&pa_charsets.get(v.as_string()); break; } } @@ -534,8 +531,8 @@ static void _curl_options(Request& r, Me if(HashStringValue* options_hash=params.as_hash(0)){ if(Value* value=options_hash->get("charset")){ // charset should be handled first as params may require transcode - Value &v=r.process_to_value(*value); - options().charset=&::charsets.get(v.as_string()); + Value &v=r.process(*value); + options().charset=&pa_charsets.get(v.as_string()); } options_hash->for_each(curl_setopt, r); } @@ -576,13 +573,13 @@ static void _curl_info(Request& r, Metho curl_infos=new CurlInfoHash(); if(params.count()==1){ const String &name=params.as_string(0, "name must be string"); - r.write_assign_lang(*curl_getinfo(name)); + r.write(*curl_getinfo(name)); } else { VHash& result=*new VHash; for(CurlInfoHash::Iterator i(*curl_infos); i; i.next() ){ result.get_hash()->put(i.key(), curl_getinfo(i.key(), i.value())); } - r.write_no_lang(result); + r.write(result); } } @@ -592,8 +589,14 @@ public: char *buf; size_t length; size_t buf_size; + ResponseHeaders& headers; - Curl_buffer() : buf((char *)pa_malloc(MAX_STRING+1)), length(0), buf_size(MAX_STRING){} + Curl_buffer(ResponseHeaders& aheaders) : buf((char *)pa_malloc_atomic(MAX_STRING)), length(0), buf_size(MAX_STRING-1), headers(aheaders){} + + void resize(size_t size){ + buf_size=size; + buf=(char *)pa_realloc(buf, size+1); + } }; static int curl_writer(char *data, size_t size, size_t nmemb, Curl_buffer *result){ @@ -602,10 +605,9 @@ static int curl_writer(char *data, size_ size=size*nmemb; if(size>0){ - if(result->length + size >= result->buf_size){ - result->buf_size = result->buf_size*2 + size; - result->buf = (char *)pa_realloc(result->buf, result->buf_size+1); - } + size_t buf_required = result->length + size; + if(buf_required > result->buf_size) + result->resize(buf_required <= result->headers.content_length ? (size_t)result->headers.content_length : result->buf_size*2 + size); memcpy(result->buf+result->length, data, size); result->length += size; } @@ -618,7 +620,15 @@ static int curl_header(char *data, size_ size=size*nmemb; if(size>0){ - result->add_header(pa_strdup(data, size)); + char *header=pa_strdup(data, size); + if(!pa_strncasecmp(header, "HTTP/") && !strchr(header, ':')){ + // response code, clearing possible headers from previous requests + result->clear(); + } else { + result->add_header(header); + if(result->content_length>pa_file_size_limit) + return 0; + } } return size; } @@ -634,15 +644,15 @@ static void _curl_load_action(Request& r CURLcode res; - Curl_buffer body; - CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function"); - CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer"); - // we need a container for headers as VFile fields can be put only after VFile.set ResponseHeaders response; CURL_SETOPT(CURLOPT_HEADERFUNCTION, curl_header, "curl header function"); CURL_SETOPT(CURLOPT_WRITEHEADER, &response, "curl header buffer"); + Curl_buffer body(response); + CURL_SETOPT(CURLOPT_WRITEFUNCTION, curl_writer, "curl writer function"); + CURL_SETOPT(CURLOPT_WRITEDATA, &body, "curl write buffer"); + if((res=f_curl_easy_perform(curl())) != CURLE_OK){ const char *ex_type = 0; switch(res){ @@ -660,6 +670,8 @@ static void _curl_load_action(Request& r case CURLE_SSL_CACERT: case CURLE_SSL_ENGINE_INITFAILED: ex_type = "curl.ssl"; break; + case CURLE_WRITE_ERROR: + check_file_size(response.content_length, *new String(options().url)); break; default: break; } throw Exception( PA_DEFAULT(ex_type, "curl.fail"), 0, "%s", f_curl_easy_strerror(res)); @@ -675,7 +687,7 @@ static void _curl_load_action(Request& r asked_charset=detect_charset(response.content_type.cstr()); if(options().is_text) - asked_charset=charsets.checkBOM(body.buf, body.length, asked_charset); + asked_charset=pa_charsets.checkBOM(body.buf, body.length, asked_charset); if (!asked_charset) asked_charset = options().charset; @@ -715,7 +727,7 @@ static void _curl_load_action(Request& r if(Value *vcookies=vtables->hash().get("SET-COOKIE")) result.fields().put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table()))); - r.write_no_lang(result); + r.write(result); } static void _curl_load(Request& r, MethodParams& params){