|
|
| version 1.25, 2009/05/14 08:10:09 | version 1.41, 2009/09/10 09:44:07 |
|---|---|
| Line 41 static const char * const IDENT_HTTP_C=" | Line 41 static const char * const IDENT_HTTP_C=" |
| #undef CRLF | #undef CRLF |
| #define CRLF "\r\n" | #define CRLF "\r\n" |
| #define DCRLF "\r\n\r\n" | |
| static bool set_addr(struct sockaddr_in *addr, const char* host, const short port){ | static bool set_addr(struct sockaddr_in *addr, const char* host, const short port){ |
| memset(addr, 0, sizeof(*addr)); | memset(addr, 0, sizeof(*addr)); |
| Line 67 size_t guess_content_length(char* buf) { | Line 66 size_t guess_content_length(char* buf) { |
| char* ptr; | char* ptr; |
| if((ptr=strstr(buf, "Content-Length:"))) // Apache | if((ptr=strstr(buf, "Content-Length:"))) // Apache |
| goto found; | goto found; |
| if((ptr=strstr(buf, "content-length:"))) // Parser 3 | if((ptr=strstr(buf, "content-length:"))) // Parser 3 before 3.4.0 |
| goto found; | goto found; |
| if((ptr=strstr(buf, "Content-length:"))) // maybe 1 | if((ptr=strstr(buf, "Content-length:"))) // maybe 1 |
| goto found; | goto found; |
| Line 76 size_t guess_content_length(char* buf) { | Line 75 size_t guess_content_length(char* buf) { |
| return 0; | return 0; |
| found: | found: |
| char *error_pos; | char *error_pos; |
| size_t result=(size_t)strtol(ptr+15/*strlen("CONTENT-LENGTH:")*/, &error_pos, 0); | size_t result=(size_t)strtol(ptr+15/*strlen("Content-Length:")*/, &error_pos, 0); |
| const size_t reasonable_initial_max=0x400*0x400*10 /*10M*/; | const size_t reasonable_initial_max=0x400*0x400*10 /*10M*/; |
| if(result>reasonable_initial_max) // sanity check | if(result>reasonable_initial_max) // sanity check |
| Line 86 found: | Line 85 found: |
| static int http_read_response(char*& response, size_t& response_size, int sock, bool fail_on_status_ne_200) { | static int http_read_response(char*& response, size_t& response_size, int sock, bool fail_on_status_ne_200) { |
| int result=0; | int result=0; |
| // fetching some to local buffer, guessing on possible content-length | // fetching some to local buffer, guessing on possible Content-Length |
| response_size=0x400*20; // initial size if content-length could not be determined | response_size=0x400*20; // initial size if Content-Length could not be determined |
| const size_t preview_size=0x400*20; | const size_t preview_size=0x400*20; |
| char preview_buf[preview_size+1/*terminator*/]; // 20K buffer to preview headers | char preview_buf[preview_size+1/*terminator*/]; // 20K buffer to preview headers |
| ssize_t received_size=recv(sock, preview_buf, preview_size, 0); | ssize_t received_size=recv(sock, preview_buf, preview_size, 0); |
| Line 136 static int http_read_response(char*& res | Line 135 static int http_read_response(char*& res |
| // we use terminator byte for two purposes here: | // we use terminator byte for two purposes here: |
| // 1. we return there zero always, not knowing: maybe they would want to create String form $file.body? | // 1. we return there zero always, not knowing: maybe they would want to create String form $file.body? |
| // invariant: all Strings should have zero-terminated buffers | // invariant: all Strings should have zero-terminated buffers |
| // 2. we use that out-of-size byte to detect if our content-length guess was wrong | // 2. we use that out-of-size byte to detect if our Content-Length guess was wrong |
| // when recv gets more than we expected | // when recv gets more than we expected |
| // a) we know that the content-length guess was wrong | // a) we know that the Content-Length guess was wrong |
| // b) we have space to put the first byte of extra data | // b) we have space to put the first byte of extra data |
| // c) we use less code to detect normal situation: on last while-cycle recv expected to just return 0 | // c) we use less code to detect normal situation: on last while-cycle recv expected to just return 0 |
| while(true) { | while(true) { |
| Line 314 static int http_request(char*& response, | Line 313 static int http_request(char*& response, |
| struct Http_pass_header_info { | struct Http_pass_header_info { |
| Request_charsets* charsets; | Request_charsets* charsets; |
| String* request; | String* request; |
| bool user_agent_specified; | bool* user_agent_specified; |
| bool content_type_specified; | bool* content_type_specified; |
| bool* content_type_url_encoded; | |
| }; | }; |
| #endif | #endif |
| static void http_pass_header(HashStringValue::key_type name, | static void http_pass_header(HashStringValue::key_type aname, |
| HashStringValue::value_type value, | HashStringValue::value_type avalue, |
| Http_pass_header_info *info) { | Http_pass_header_info *info) { |
| String aname=String(name, String::L_URI); | const char* name_cstr=aname.cstr(); |
| String name=String(capitalize(name_cstr), String::L_URI); | |
| if(strcasecmp(name_cstr, HTTP_CONTENT_LENGTH)==0) | |
| return; | |
| *info->request << aname << ": " | String value=attributed_meaning_to_string(*avalue, String::L_URI, false); |
| << attributed_meaning_to_string(*value, String::L_URI, false) | |
| << CRLF; | *info->request << name << ": " << value << CRLF; |
| const String::Body name_upper=aname.change_case(info->charsets->source(), String::CC_UPPER); | if(strcasecmp(name_cstr, HTTP_USER_AGENT)==0) |
| if(name_upper==HTTP_USER_AGENT_UPPER) | *info->user_agent_specified=true; |
| info->user_agent_specified=true; | if(strcasecmp(name_cstr, HTTP_CONTENT_TYPE)==0){ |
| if(name_upper==HTTP_CONTENT_TYPE_UPPER) | *info->content_type_specified=true; |
| info->content_type_specified=true; | *info->content_type_url_encoded=StrStartFromNC(value.cstr(), HTTP_CONTENT_TYPE_FORM_URLENCODED); |
| } | |
| } | } |
| static void http_pass_cookie(HashStringValue::key_type name, | static void http_pass_cookie(HashStringValue::key_type name, |
| Line 340 static void http_pass_cookie(HashStringV | Line 346 static void http_pass_cookie(HashStringV |
| Http_pass_header_info *info) { | Http_pass_header_info *info) { |
| *info->request << String(name, String::L_HTTP_COOKIE) << "=" | *info->request << String(name, String::L_HTTP_COOKIE) << "=" |
| << attributed_meaning_to_string(*value, String::L_HTTP_COOKIE, false) | << attributed_meaning_to_string(*value, String::L_HTTP_COOKIE, true) |
| << "; "; | << "; "; |
| } | } |
| Line 366 static void form_string_value2string( | Line 372 static void form_string_value2string( |
| const String& value, | const String& value, |
| String& result) | String& result) |
| { | { |
| result << String(key, String::L_URI) << "="; | result << String(key, String::L_URI) << "=" << String(value, String::L_URI) << "&"; |
| result.append(value, String::L_URI, true); | |
| result << "&"; | |
| } | } |
| #ifndef DOXYGEN | #ifndef DOXYGEN |
| Line 402 static void form_value2string( | Line 406 static void form_value2string( |
| const char* pa_form2string(HashStringValue& form, Request_charsets& charsets) { | const char* pa_form2string(HashStringValue& form, Request_charsets& charsets) { |
| String string; | String string; |
| form.for_each<String*>(form_value2string, &string); | form.for_each<String*>(form_value2string, &string); |
| return string.cstr(String::L_UNSPECIFIED, 0, &charsets); | return string.transcode_and_untaint_cstr(String::L_URI, &charsets); |
| } | } |
| struct FormPart { | struct FormPart { |
| Line 412 struct FormPart { | Line 416 struct FormPart { |
| Form_table_value2string_info* info; | Form_table_value2string_info* info; |
| }; | }; |
| static void form_part_boundary_header(FormPart& part, String name, const char* file_name=0){ | static void form_part_boundary_header(FormPart& part, String::Body name, const char* file_name=0){ |
| part.string << "--" << part.boundary; | part.string << "--" << part.boundary |
| part.string << CRLF HTTP_CONTENT_DISPOSITION ": form-data; name=\"" << name << "\""; | << CRLF CONTENT_DISPOSITION_CAPITALIZED ": form-data; name=\"" |
| << Charset::transcode(name, part.r->charsets.source(), part.r->charsets.client()) | |
| << "\""; | |
| if(file_name){ | if(file_name){ |
| if(strcmp(file_name, NONAME_DAT)!=0) | if(strcmp(file_name, NONAME_DAT)!=0) |
| part.string << "; filename=\"" << file_name << "\""; | part.string << "; filename=\"" << file_name << "\""; |
| part.string << CRLF HTTP_CONTENT_TYPE ": " << part.r->mime_type_of(file_name); | part.string << CRLF HTTP_CONTENT_TYPE_CAPITALIZED ": " << part.r->mime_type_of(file_name); |
| } | } |
| part.string << DCRLF; | part.string << CRLF CRLF; |
| } | } |
| static void form_string_value2part( | static void form_string_value2part( |
| HashStringValue::key_type key, | HashStringValue::key_type key, |
| const String& value, | const String& value, |
| FormPart& part) | FormPart& part) |
| { | { |
| form_part_boundary_header(part, String(key, String::L_URI)); | form_part_boundary_header(part, key); |
| part.string.append(value, String::L_AS_IS, true); | part.string << Charset::transcode(value, part.r->charsets.source(), part.r->charsets.client()) << CRLF; |
| part.string << CRLF; | |
| } | } |
| static void form_file_value2part( | static void form_file_value2part( |
| HashStringValue::key_type key, | HashStringValue::key_type key, |
| VFile& vfile, | VFile& vfile, |
| FormPart& part) | FormPart& part) |
| { | { |
| form_part_boundary_header(part, String(key, String::L_URI), vfile.fields().get(name_name)->as_string().cstr()); | form_part_boundary_header(part, key, vfile.fields().get(name_name)->as_string().cstr()); |
| part.string.append_know_length(vfile.value_ptr(), vfile.value_size(), String::L_FILE_POST); | part.string.append_know_length(vfile.value_ptr(), vfile.value_size(), String::L_FILE_POST); |
| part.string << CRLF; | part.string << CRLF; |
| } | } |
| Line 448 static void form_table_value2part(Table: | Line 453 static void form_table_value2part(Table: |
| } | } |
| static void form_value2part( | static void form_value2part( |
| HashStringValue::key_type key, | HashStringValue::key_type key, |
| HashStringValue::value_type value, | HashStringValue::value_type value, |
| FormPart& part) | FormPart& part) |
| { | { |
| if(const String* svalue=value->get_string()) | if(const String* svalue=value->get_string()) |
| form_string_value2part(key, *svalue, part); | form_string_value2part(key, *svalue, part); |
| Line 458 static void form_value2part( | Line 463 static void form_value2part( |
| Form_table_value2string_info info(key, part.string); | Form_table_value2string_info info(key, part.string); |
| part.info = &info; | part.info = &info; |
| tvalue->for_each(form_table_value2part, &part); | tvalue->for_each(form_table_value2part, &part); |
| } else if(VFile* vfile=static_cast<VFile *>(value->as("file", false))){ | } else if(VFile* vfile=static_cast<VFile *>(value->as("file"))){ |
| form_file_value2part(key, *vfile, part); | form_file_value2part(key, *vfile, part); |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| Line 473 const char* pa_form2string_multipart(Has | Line 478 const char* pa_form2string_multipart(Has |
| formpart.info=NULL; | formpart.info=NULL; |
| form.for_each<FormPart&>(form_value2part, formpart); | form.for_each<FormPart&>(form_value2part, formpart); |
| formpart.string << "--" << boundary << "--"; | formpart.string << "--" << boundary << "--"; |
| post_size=formpart.string.length(); | post_size=formpart.string.length(); // very surprizing, but it calculates correct post_size even with binary files! |
| return formpart.string.cstr(String::L_UNSPECIFIED, 0, &(r.charsets)); | return formpart.string.untaint_cstr(String::L_AS_IS); // without transcoding |
| } | } |
| static void find_headers_end(char* p, | static void find_headers_end(char* p, |
| Line 575 File_read_http_result pa_internal_file_r | Line 580 File_read_http_result pa_internal_file_r |
| if(valid_options!=options->count()) | if(valid_options!=options->count()) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| 0, | 0, |
| "invalid option passed"); | INVALID_OPTION_PASSED); |
| } | } |
| if(!asked_remote_charset) // defaulting to $request:charset | if(!asked_remote_charset) // defaulting to $request:charset |
| asked_remote_charset=&(r.charsets).source(); | asked_remote_charset=&(r.charsets).source(); |
| Line 607 File_read_http_result pa_internal_file_r | Line 612 File_read_http_result pa_internal_file_r |
| } | } |
| //preparing request | //preparing request |
| String& connect_string=*new String; | String& connect_string=*new String(file_spec); |
| // not in ^sql{... L_SQL ...} spirit, but closer to ^file::load one | |
| connect_string.append(file_spec, String::L_URI); // tainted pieces -> URI pieces | |
| String request_head_and_body; | String request_head_and_body; |
| { | { |
| // influence URLencoding of tainted pieces to String::L_URI lang | // influence URLencoding of tainted pieces to String::L_URI lang |
| Temp_client_charset temp(r.charsets, *asked_remote_charset); | Temp_client_charset temp(r.charsets, *asked_remote_charset); |
| const char* connect_string_cstr=connect_string.cstr(String::L_UNSPECIFIED, 0, &(r.charsets)); | const char* connect_string_cstr=connect_string.transcode_and_untaint_cstr(String::L_URI, &(r.charsets)); |
| const char* current=connect_string_cstr; | const char* current=connect_string_cstr; |
| if(strncmp(current, "http://", 7)!=0) | if(strncmp(current, "http://", 7)!=0) |
| Line 626 File_read_http_result pa_internal_file_r | Line 629 File_read_http_result pa_internal_file_r |
| current+=7; | current+=7; |
| strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0; | strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0; |
| char* host_uri=lsplit(host, '/'); | char* host_uri=lsplit(host, '/'); |
| uri=host_uri?current+(host_uri-1-host):"/"; | uri=host_uri?current+(host_uri-1-host):"/"; |
| char* port_cstr=lsplit(host, ':'); | char* port_cstr=lsplit(host, ':'); |
| char* error_pos=0; | char* error_pos=0; |
| port=port_cstr?(short)strtol(port_cstr, &error_pos, 0):80; | port=port_cstr?(short)strtol(port_cstr, &error_pos, 0):80; |
| bool uri_has_query_string=strchr(uri, '?')!=0; | |
| // making request head | // making request head |
| String head; | String head; |
| head << method << " " << uri; | head << method << " " << uri; |
| if(form && method_is_get) | if(method_is_get && form) |
| head << (uri_has_query_string?"&":"?") << pa_form2string(*form, r.charsets); | head << (strchr(uri, '?')!=0?"&":"?") << pa_form2string(*form, r.charsets); |
| head <<" HTTP/1.0" CRLF "host: "<< host << CRLF; | head <<" HTTP/1.0" CRLF "Host: "<< host << CRLF; |
| char* boundary; | char* boundary=0; |
| if(multipart){ | if(multipart){ |
| uuid uuid=get_uuid(); | uuid uuid=get_uuid(); |
| Line 656 File_read_http_result pa_internal_file_r | Line 657 File_read_http_result pa_internal_file_r |
| uuid.node[3], uuid.node[4], uuid.node[5]); | uuid.node[3], uuid.node[4], uuid.node[5]); |
| } | } |
| String user_headers; | |
| bool user_agent_specified=false; | |
| bool content_type_specified=false; | |
| bool content_type_url_encoded=false; | |
| if(vheaders && !vheaders->is_string()) { // allow empty | |
| if(HashStringValue *headers=vheaders->get_hash()) { | |
| Http_pass_header_info info={ | |
| &(r.charsets), | |
| &user_headers, | |
| &user_agent_specified, | |
| &content_type_specified, | |
| &content_type_url_encoded}; | |
| headers->for_each<Http_pass_header_info*>(http_pass_header, &info); | |
| } else | |
| throw Exception(PARSER_RUNTIME, | |
| 0, | |
| "headers param must be hash"); | |
| }; | |
| size_t post_size=0; | size_t post_size=0; |
| if(form && !method_is_get) { | if(form && !method_is_get) { |
| head << HTTP_CONTENT_TYPE ": "; | head << "Content-Type: " << (multipart ? HTTP_CONTENT_TYPE_MULTIPART_FORMDATA : HTTP_CONTENT_TYPE_FORM_URLENCODED); |
| if(!omit_post_charset) | |
| head << "; charset=" << asked_remote_charset->NAME_CSTR(); | |
| if(multipart) { | if(multipart) { |
| head << HTTP_CONTENT_TYPE_MULTIPART_FORMDATA "; boundary=" << boundary << CRLF; | head << "; boundary=" << boundary; |
| // !!! charset? | body_cstr=pa_form2string_multipart(*form, r/*charsets & mime_type needed*/, boundary, post_size/*correct post_size returned here*/); |
| body_cstr=pa_form2string_multipart(*form, r, boundary, post_size); | |
| } else { | } else { |
| head << HTTP_CONTENT_TYPE_FORM_URLENCODED; | |
| if(!omit_post_charset) | |
| head << "; charset=" << asked_remote_charset->NAME_CSTR() << ";"; | |
| head << CRLF; | |
| body_cstr=pa_form2string(*form, r.charsets); | body_cstr=pa_form2string(*form, r.charsets); |
| post_size=strlen(body_cstr); | post_size=strlen(body_cstr); |
| } | } |
| } else if (vbody) { | head << CRLF; |
| body_cstr=vbody->as_string().cstr(String::L_UNSPECIFIED, 0, &(r.charsets)); | } else if(vbody) { |
| // needed for transcoded $.body[] first of all | // $.body was specified |
| body_cstr=Charset::transcode( | if(content_type_url_encoded){ |
| String::C(body_cstr, strlen(body_cstr)), | // transcode + url-encode |
| r.charsets.source(), | body_cstr=vbody->as_string().transcode_and_untaint_cstr(String::L_URI, &(r.charsets)); |
| *asked_remote_charset | } else { |
| ); | // content-type != application/x-www-form-urlencoded -> transcode only, don't url-encode! |
| body_cstr=Charset::transcode( | |
| String::C(vbody->as_string().cstr(), vbody->as_string().length()), | |
| r.charsets.source(), | |
| *asked_remote_charset | |
| ); | |
| } | |
| post_size=strlen(body_cstr); | |
| } | } |
| // http://www.ietf.org/rfc/rfc2617.txt | // http://www.ietf.org/rfc/rfc2617.txt |
| if(const String* authorization_field_value=basic_authorization_field(user_cstr, password_cstr)) | if(const String* authorization_field_value=basic_authorization_field(user_cstr, password_cstr)) |
| head<<"authorization: "<<*authorization_field_value<<CRLF; | head << "Authorization: " << *authorization_field_value << CRLF; |
| head << user_headers; | |
| bool user_agent_specified=false; | |
| bool content_type_specified=false; | |
| if(vheaders && !vheaders->is_string()) { // allow empty | |
| if(HashStringValue *headers=vheaders->get_hash()) { | |
| Http_pass_header_info info={&(r.charsets), &head, false}; | |
| headers->for_each<Http_pass_header_info*>(http_pass_header, &info); | |
| user_agent_specified=info.user_agent_specified; | |
| content_type_specified=info.content_type_specified; | |
| } else | |
| throw Exception(PARSER_RUNTIME, | |
| &connect_string, | |
| "headers param must be hash"); | |
| }; | |
| if(!user_agent_specified) // defaulting | if(!user_agent_specified) // defaulting |
| head << HTTP_USER_AGENT ": " DEFAULT_USER_AGENT CRLF; | head << "User-Agent: " DEFAULT_USER_AGENT CRLF; |
| if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified | if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| &connect_string, | 0, |
| "$.content-type can't be specified with method POST"); | "$.content-type can't be specified with method POST"); |
| if(vcookies && !vcookies->is_string()){ // allow empty | if(vcookies && !vcookies->is_string()){ // allow empty |
| if(HashStringValue* cookies=vcookies->get_hash()) { | if(HashStringValue* cookies=vcookies->get_hash()) { |
| head << "cookie: "; | head << "Cookie: "; |
| Http_pass_header_info info={&(r.charsets), &head, false}; | Http_pass_header_info info={&(r.charsets), &head, 0, 0, 0}; |
| cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info); | cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info); |
| head << CRLF; | head << CRLF; |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| &connect_string, | 0, |
| "cookies param must be hash"); | "cookies param must be hash"); |
| } | } |
| if(body_cstr) | if(body_cstr) |
| head << "content-length: " << format(post_size, "%u") << CRLF; | head << "Content-Length: " << format(post_size, "%u") << CRLF; |
| const char* head_cstr=head.cstr(String::L_UNSPECIFIED, 0, &(r.charsets)); | |
| // head + end of header | // head + end of header |
| request_head_and_body << head_cstr << CRLF; | request_head_and_body << head.transcode_and_untaint_cstr(String::L_URI, &(r.charsets)) << CRLF; |
| // body | // body |
| if(body_cstr) | if(body_cstr) |
| request_head_and_body << body_cstr; | request_head_and_body << body_cstr; |
| } | } |
| //sending request | const char* request_cstr=request_head_and_body.cstr(); |
| char* response; | size_t request_size=strlen(request_cstr); |
| size_t response_size; | |
| const char* request=request_head_and_body.cstr(); | |
| size_t request_size=strlen(request); | |
| if(multipart) | if(multipart) |
| request_size=file_untaint(request, request_size); | request_size=file_untaint(request_cstr, request_size); |
| char* response; | |
| size_t response_size; | |
| // sending request | |
| int status_code=http_request(response, response_size, | int status_code=http_request(response, response_size, |
| host, port, request, request_size, | host, port, request_cstr, request_size, |
| timeout_secs, fail_on_status_ne_200); | timeout_secs, fail_on_status_ne_200); |
| //processing results | // processing results |
| char* raw_body; size_t raw_body_size; | char* raw_body; size_t raw_body_size; |
| char* headers_end_at; | char* headers_end_at; |
| find_headers_end(response, | find_headers_end(response, |
| Line 768 File_read_http_result pa_internal_file_r | Line 781 File_read_http_result pa_internal_file_r |
| size_t pos_after=0; | size_t pos_after=0; |
| header_block.split(aheaders, pos_after, "\n"); | header_block.split(aheaders, pos_after, "\n"); |
| //processing headers | // processing headers |
| size_t aheaders_count=aheaders.count(); | size_t aheaders_count=aheaders.count(); |
| for(size_t i=1; i<aheaders_count; i++) { | for(size_t i=1; i<aheaders_count; i++) { |
| const String& line=*aheaders.get(i); | const String& line=*aheaders.get(i); |
| Line 780 File_read_http_result pa_internal_file_r | Line 793 File_read_http_result pa_internal_file_r |
| const String::Body HEADER_NAME=line.mid(0, pos).change_case(r.charsets.source(), String::CC_UPPER); | const String::Body HEADER_NAME=line.mid(0, pos).change_case(r.charsets.source(), String::CC_UPPER); |
| const String& HEADER_VALUE=line.mid(pos+1, line.length()).trim(String::TRIM_BOTH, " \t\r"); | const String& HEADER_VALUE=line.mid(pos+1, line.length()).trim(String::TRIM_BOTH, " \t\r"); |
| if(as_text && HEADER_NAME==HTTP_CONTENT_TYPE_UPPER) | if(as_text && HEADER_NAME==HTTP_CONTENT_TYPE_UPPER) |
| real_remote_charset=detect_charset(r.charsets.source(), HEADER_VALUE); | real_remote_charset=detect_charset(HEADER_VALUE.cstr()); |
| // tables | // tables |
| { | { |