|
|
| version 1.57, 2013/03/11 05:51:17 | version 1.72, 2016/07/26 13:20:23 |
|---|---|
| Line 1 | Line 1 |
| /** @file | /** @file |
| Parser: http support functions. | Parser: http support functions. |
| Copyright (c) 2001-2012 Art. Lebedev Studio (http://www.artlebedev.com) | Copyright (c) 2001-2015 Art. Lebedev Studio (http://www.artlebedev.com) |
| Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru) | Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru) |
| */ | */ |
| Line 15 | Line 15 |
| volatile const char * IDENT_PA_HTTP_C="$Id$" IDENT_PA_HTTP_H; | volatile const char * IDENT_PA_HTTP_C="$Id$" IDENT_PA_HTTP_H; |
| #ifdef _MSC_VER | |
| #include <windows.h> | |
| #else | |
| #define closesocket close | |
| #endif | |
| // defines | // defines |
| #define HTTP_METHOD_NAME "method" | #define HTTP_METHOD_NAME "method" |
| Line 24 volatile const char * IDENT_PA_HTTP_C="$ | Line 30 volatile const char * IDENT_PA_HTTP_C="$ |
| #define HTTP_HEADERS_NAME "headers" | #define HTTP_HEADERS_NAME "headers" |
| #define HTTP_FORM_ENCTYPE_NAME "enctype" | #define HTTP_FORM_ENCTYPE_NAME "enctype" |
| #define HTTP_ANY_STATUS_NAME "any-status" | #define HTTP_ANY_STATUS_NAME "any-status" |
| #define HTTP_OMIT_POST_CHARSET_NAME "omit-post-charset" // ^file::load[...;http://...;$.form[...]$.method[post]] | #define HTTP_OMIT_POST_CHARSET_NAME "omit-post-charset" // ^file::load[...;http://...;$.method[post]] by default adds charset to content-type |
| // by default add charset to content-type | |
| #define HTTP_TABLES_NAME "tables" | |
| #define HTTP_USER "user" | #define HTTP_USER "user" |
| #define HTTP_PASSWORD "password" | #define HTTP_PASSWORD "password" |
| #define HTTP_USER_AGENT "user-agent" | |
| #define DEFAULT_USER_AGENT "parser3" | #define DEFAULT_USER_AGENT "parser3" |
| # ifndef INADDR_NONE | #ifndef INADDR_NONE |
| # define INADDR_NONE ((ulong) -1) | #define INADDR_NONE ((ulong) -1) |
| # endif | #endif |
| #undef CRLF | #undef CRLF |
| #define CRLF "\r\n" | #define CRLF "\r\n" |
| Line 63 static bool set_addr(struct sockaddr_in | Line 67 static bool set_addr(struct sockaddr_in |
| addr->sin_family=AF_INET; | addr->sin_family=AF_INET; |
| addr->sin_port=htons(port); | addr->sin_port=htons(port); |
| if(host) { | if(host) { |
| ulong packed_ip=inet_addr(host); | struct hostent *hostIP=gethostbyname(host); |
| if(packed_ip!=INADDR_NONE) | if(hostIP && hostIP->h_addrtype == AF_INET){ |
| memcpy(&addr->sin_addr, &packed_ip, sizeof(packed_ip)); | memcpy(&addr->sin_addr, hostIP->h_addr, hostIP->h_length); |
| else { | return true; |
| struct hostent *hostIP=gethostbyname(host); | } |
| if(hostIP) | } |
| memcpy(&addr->sin_addr, hostIP->h_addr, hostIP->h_length); | return false; |
| else | |
| return false; | |
| } | |
| } else | |
| addr->sin_addr.s_addr=INADDR_ANY; | |
| return true; | |
| } | } |
| size_t guess_content_length(char* buf) { | size_t guess_content_length(char* buf) { |
| Line 342 static void http_pass_header(HashStringV | Line 340 static void http_pass_header(HashStringV |
| *info->user_agent_specified=true; | *info->user_agent_specified=true; |
| if(strcasecmp(name_cstr, HTTP_CONTENT_TYPE)==0){ | if(strcasecmp(name_cstr, HTTP_CONTENT_TYPE)==0){ |
| *info->content_type_specified=true; | *info->content_type_specified=true; |
| *info->content_type_url_encoded=StrStartFromNC(value.cstr(), HTTP_CONTENT_TYPE_FORM_URLENCODED); | *info->content_type_url_encoded=pa_strncasecmp(value.cstr(), HTTP_CONTENT_TYPE_FORM_URLENCODED)==0; |
| } | } |
| } | } |
| Line 405 static void form_value2string( | Line 403 static void form_value2string( |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| new String(key, String::L_TAINTED), | new String(key, String::L_TAINTED), |
| "is %s, "HTTP_FORM_NAME" option value can be string or table only (file is allowed for $."HTTP_METHOD_NAME"[POST] + $."HTTP_FORM_ENCTYPE_NAME"["HTTP_CONTENT_TYPE_MULTIPART_FORMDATA"])", value->type()); | "is %s, " HTTP_FORM_NAME " option value can be string or table only (file is allowed for $." HTTP_METHOD_NAME "[POST] + $." HTTP_FORM_ENCTYPE_NAME "[" HTTP_CONTENT_TYPE_MULTIPART_FORMDATA "])", value->type()); |
| } | } |
| const char* pa_form2string(HashStringValue& form, Request_charsets& charsets) { | const char* pa_form2string(HashStringValue& form, Request_charsets& charsets) { |
| Line 511 static void form_value2part( | Line 509 static void form_value2part( |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, |
| new String(key, String::L_TAINTED), | new String(key, String::L_TAINTED), |
| "is %s, "HTTP_FORM_NAME" option value can be string, table or file only", value->type()); | "is %s, " HTTP_FORM_NAME " option value can be string, table or file only", value->type()); |
| } | } |
| const char* pa_form2string_multipart(HashStringValue& form, Request& r, const char* boundary, size_t& post_size){ | const char* pa_form2string_multipart(HashStringValue& form, Request& r, const char* boundary, size_t& post_size){ |
| Line 543 static void find_headers_end(char* p, | Line 541 static void find_headers_end(char* p, |
| // Set-Cookie: name=value; Domain=docs.foo.com; Path=/accounts; Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; HttpOnly | // Set-Cookie: name=value; Domain=docs.foo.com; Path=/accounts; Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; HttpOnly |
| static ArrayString* parse_cookie(Request& r, const String& cookie) { | static ArrayString* parse_cookie(Request& r, const String& cookie) { |
| char *current=strdup(cookie.cstr()); | char *current=pa_strdup(cookie.cstr()); |
| const String* name=0; | const String* name=0; |
| const String* value=&String::Empty; | const String* value=&String::Empty; |
| Line 617 Table* parse_cookies(Request& r, Table * | Line 615 Table* parse_cookies(Request& r, Table * |
| return &result; | return &result; |
| } | } |
| void *tables_update(HashStringValue& tables, const String::Body name, const String& value){ | |
| Table *table; | |
| if(Value *valready=tables.get(name)) { | |
| // second+ appearence | |
| table=valready->get_table(); | |
| } else { | |
| // first appearence | |
| Table::columns_type columns=new ArrayString(1); | |
| *columns+=new String("value"); | |
| table=new Table(columns); | |
| tables.put(name, new VTable(table)); | |
| } | |
| // this string becomes next row | |
| ArrayString& row=*new ArrayString(1); | |
| row+=&value; | |
| *table+=&row; | |
| } | |
| /// @todo build .cookies field. use ^file.tables.SET-COOKIES.menu{ for now | /// @todo build .cookies field. use ^file.tables.SET-COOKIES.menu{ for now |
| File_read_http_result pa_internal_file_read_http(Request& r, | File_read_http_result pa_internal_file_read_http(Request& r, const String& file_spec, bool as_text, HashStringValue *options, bool transcode_text_result) { |
| const String& file_spec, | |
| bool as_text, | |
| HashStringValue *options, | |
| bool transcode_text_result) { | |
| File_read_http_result result; | File_read_http_result result; |
| char host[MAX_STRING]; | char host[MAX_STRING]; |
| const char *idna_host; | |
| const char* uri; | const char* uri; |
| short port=80; | short port=80; |
| const char* method="GET"; | const char* method="GET"; |
| Line 636 File_read_http_result pa_internal_file_r | Line 649 File_read_http_result pa_internal_file_r |
| Value* vheaders=0; | Value* vheaders=0; |
| Value* vcookies=0; | Value* vcookies=0; |
| Value* vbody=0; | Value* vbody=0; |
| Charset *asked_remote_charset=0; | Charset* asked_remote_charset=0; |
| Charset* real_remote_charset=0; | |
| const char* user_cstr=0; | const char* user_cstr=0; |
| const char* password_cstr=0; | const char* password_cstr=0; |
| const char* encode=0; | const char* encode=0; |
| Line 680 File_read_http_result pa_internal_file_r | Line 694 File_read_http_result pa_internal_file_r |
| omit_post_charset=vomit_post_charset->as_bool(); | omit_post_charset=vomit_post_charset->as_bool(); |
| } | } |
| if(Value* vcharset_name=options->get(PA_CHARSET_NAME)) { | if(Value* vcharset_name=options->get(PA_CHARSET_NAME)) { |
| asked_remote_charset=&charsets.get(vcharset_name->as_string(). | asked_remote_charset=&charsets.get(vcharset_name->as_string().change_case(r.charsets.source(), String::CC_UPPER)); |
| change_case(r.charsets.source(), String::CC_UPPER)); | } |
| if(Value* vresponse_charset_name=options->get(PA_RESPONSE_CHARSET_NAME)) { | |
| valid_options++; | |
| real_remote_charset=&charsets.get(vresponse_charset_name->as_string().change_case(r.charsets.source(), String::CC_UPPER)); | |
| } | } |
| if(Value* vuser=options->get(HTTP_USER)) { | if(Value* vuser=options->get(HTTP_USER)) { |
| valid_options++; | valid_options++; |
| Line 700 File_read_http_result pa_internal_file_r | Line 717 File_read_http_result pa_internal_file_r |
| if(encode){ | if(encode){ |
| if(method_is_get) | if(method_is_get) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "you can not use $." HTTP_FORM_ENCTYPE_NAME " option with method GET"); |
| 0, | |
| "you can not use $."HTTP_FORM_ENCTYPE_NAME" option with method GET"); | |
| multipart=strcasecmp(encode, HTTP_CONTENT_TYPE_MULTIPART_FORMDATA)==0; | multipart=strcasecmp(encode, HTTP_CONTENT_TYPE_MULTIPART_FORMDATA)==0; |
| if(!multipart && strcasecmp(encode, HTTP_CONTENT_TYPE_FORM_URLENCODED)!=0) | if(!multipart && strcasecmp(encode, HTTP_CONTENT_TYPE_FORM_URLENCODED)!=0) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "$." HTTP_FORM_ENCTYPE_NAME " option value can be " HTTP_CONTENT_TYPE_FORM_URLENCODED " or " HTTP_CONTENT_TYPE_MULTIPART_FORMDATA " only"); |
| 0, | |
| "$."HTTP_FORM_ENCTYPE_NAME" option value can be "HTTP_CONTENT_TYPE_FORM_URLENCODED" or "HTTP_CONTENT_TYPE_MULTIPART_FORMDATA" only"); | |
| } | } |
| if(vbody){ | if(vbody){ |
| if(method_is_get) | if(method_is_get) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "you can not use $." HTTP_BODY_NAME " option with method GET"); |
| 0, | |
| "you can not use $."HTTP_BODY_NAME" option with method GET"); | |
| if(form) | if(form) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "you can not use options $." HTTP_BODY_NAME " and $." HTTP_FORM_NAME " together"); |
| 0, | |
| "you can not use options $."HTTP_BODY_NAME" and $."HTTP_FORM_NAME" together"); | |
| } | } |
| //preparing request | //preparing request |
| Line 737 File_read_http_result pa_internal_file_r | Line 746 File_read_http_result pa_internal_file_r |
| const char* current=connect_string_cstr; | const char* current=connect_string_cstr; |
| if(strncmp(current, "http://", 7)!=0) | if(strncmp(current, "http://", 7)!=0) |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, &connect_string, "does not start with http://"); //never |
| &connect_string, | |
| "does not start with http://"); //never | |
| current+=7; | current+=7; |
| strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0; | strncpy(host, current, sizeof(host)-1); host[sizeof(host)-1]=0; |
| Line 754 File_read_http_result pa_internal_file_r | Line 761 File_read_http_result pa_internal_file_r |
| throw Exception(PARSER_RUNTIME, &connect_string, "invalid port number '%s'", port_cstr); | throw Exception(PARSER_RUNTIME, &connect_string, "invalid port number '%s'", port_cstr); |
| } | } |
| idna_host=pa_idna_encode(host, r.charsets.source()); | |
| // making request head | // making request head |
| String head; | String head; |
| head << method << " " << uri; | head << method << " " << uri; |
| if(method_is_get && form) | if(method_is_get && form) |
| head << (strchr(uri, '?')!=0?"&":"?") << pa_form2string(*form, r.charsets); | head << (strchr(uri, '?')!=0?"&":"?") << pa_form2string(*form, r.charsets); |
| head <<" HTTP/1.0" CRLF "Host: "<< host; | head <<" HTTP/1.0" CRLF "Host: "<< idna_host; |
| if (port != 80) | if (port != 80) |
| head << ":" << port_cstr; | head << ":" << port_cstr; |
| head << CRLF; | head << CRLF; |
| char* boundary=0; | char* boundary= multipart ? get_uuid_boundary() : 0; |
| if(multipart){ | |
| uuid uuid=get_uuid(); | |
| const int boundary_bufsize=10+32+1/*for zero-teminator*/+1/*for faulty snprintfs*/; | |
| boundary=new(PointerFreeGC) char[boundary_bufsize]; | |
| snprintf(boundary, boundary_bufsize, | |
| "----------%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X", | |
| uuid.time_low, uuid.time_mid, uuid.time_hi_and_version, | |
| uuid.clock_seq >> 8, uuid.clock_seq & 0xFF, | |
| uuid.node[0], uuid.node[1], uuid.node[2], | |
| uuid.node[3], uuid.node[4], uuid.node[5]); | |
| } | |
| String user_headers; | String user_headers; |
| bool user_agent_specified=false; | bool user_agent_specified=false; |
| Line 793 File_read_http_result pa_internal_file_r | Line 790 File_read_http_result pa_internal_file_r |
| &content_type_url_encoded}; | &content_type_url_encoded}; |
| headers->for_each<Http_pass_header_info*>(http_pass_header, &info); | headers->for_each<Http_pass_header_info*>(http_pass_header, &info); |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "headers param must be hash"); |
| 0, | |
| "headers param must be hash"); | |
| }; | }; |
| const char* request_body=0; | const char* request_body=0; |
| Line 821 File_read_http_result pa_internal_file_r | Line 816 File_read_http_result pa_internal_file_r |
| request_body=vbody->as_string().untaint_and_transcode_cstr(String::L_URI, &(r.charsets)); | request_body=vbody->as_string().untaint_and_transcode_cstr(String::L_URI, &(r.charsets)); |
| } else { | } else { |
| // content-type != application/x-www-form-urlencoded -> transcode only, don't url-encode! | // content-type != application/x-www-form-urlencoded -> transcode only, don't url-encode! |
| request_body=Charset::transcode( | const String &sbody=vbody->as_string(); |
| String::C(vbody->as_string().cstr(), vbody->as_string().length()), | request_body=Charset::transcode(String::C(sbody.cstr(), sbody.length()), r.charsets.source(), *asked_remote_charset).str; |
| r.charsets.source(), | |
| *asked_remote_charset | |
| ); | |
| } | } |
| post_size=strlen(request_body); | post_size=strlen(request_body); |
| } | } |
| Line 840 File_read_http_result pa_internal_file_r | Line 832 File_read_http_result pa_internal_file_r |
| head << "User-Agent: " DEFAULT_USER_AGENT CRLF; | head << "User-Agent: " DEFAULT_USER_AGENT CRLF; |
| if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified | if(form && !method_is_get && content_type_specified) // POST + form + content-type was specified |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "$.content-type can't be specified with method POST"); |
| 0, | |
| "$.content-type can't be specified with method POST"); | |
| if(vcookies && !vcookies->is_string()){ // allow empty | if(vcookies && !vcookies->is_string()){ // allow empty |
| if(HashStringValue* cookies=vcookies->get_hash()) { | if(HashStringValue* cookies=vcookies->get_hash()) { |
| Line 851 File_read_http_result pa_internal_file_r | Line 841 File_read_http_result pa_internal_file_r |
| cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info); | cookies->for_each<Http_pass_header_info*>(http_pass_cookie, &info); |
| head << CRLF; | head << CRLF; |
| } else | } else |
| throw Exception(PARSER_RUNTIME, | throw Exception(PARSER_RUNTIME, 0, "cookies param must be hash"); |
| 0, | |
| "cookies param must be hash"); | |
| } | } |
| if(request_body) | if(request_body) |
| Line 876 File_read_http_result pa_internal_file_r | Line 864 File_read_http_result pa_internal_file_r |
| } | } |
| } | } |
| char* response; | char* response_str; |
| size_t response_size; | size_t response_size; |
| // sending request | // sending request |
| int status_code=http_request(response, response_size, | int status_code=http_request(response_str, response_size, idna_host, port, request, request_size, timeout_secs, fail_on_status_ne_200); |
| host, port, request, request_size, | |
| timeout_secs, fail_on_status_ne_200); | |
| // processing results | // processing results |
| char* raw_body; size_t raw_body_size; | char* raw_body; size_t raw_body_size; |
| char* headers_end_at; | char* headers_end_at; |
| find_headers_end(response, | find_headers_end(response_str, headers_end_at, raw_body); |
| headers_end_at, | raw_body_size=response_size-(raw_body-response_str); |
| raw_body); | |
| raw_body_size=response_size-(raw_body-response); | |
| result.headers=new HashStringValue; | result.headers=new HashStringValue; |
| VHash* vtables=new VHash; | VHash* vtables=new VHash; |
| result.headers->put(HTTP_TABLES_NAME, vtables); | result.headers->put("tables", vtables); |
| Charset* real_remote_charset=0; // undetected, yet | |
| ResponseHeaders response; | |
| if(headers_end_at) { | if(headers_end_at) { |
| *headers_end_at=0; | *headers_end_at=0; |
| const String header_block(String::C(response, headers_end_at-response), String::L_TAINTED); | const String header_block(String::C(response_str, headers_end_at-response_str), String::L_TAINTED); |
| ArrayString aheaders; | ArrayString aheaders; |
| HashStringValue& tables=vtables->hash(); | |
| size_t pos_after=0; | size_t pos_after=0; |
| header_block.split(aheaders, pos_after, "\n"); | header_block.split(aheaders, pos_after, "\n"); |
| // processing headers | |
| size_t aheaders_count=aheaders.count(); | |
| for(size_t i=1; i<aheaders_count; i++) { | |
| const String& line=*aheaders.get(i); | |
| size_t pos=line.pos(':'); | |
| if(pos==STRING_NOT_FOUND || pos<1) | |
| throw Exception("http.response", | |
| &connect_string, | |
| "bad response from host - bad header \"%s\"", line.cstr()); | |
| const String::Body HEADER_NAME=line.mid(0, pos).change_case(r.charsets.source(), String::CC_UPPER); | |
| const String& HEADER_VALUE=line.mid(pos+1, line.length()).trim(String::TRIM_BOTH, " \t\r"); | |
| if(as_text && HEADER_NAME==HTTP_CONTENT_TYPE_UPPER) | |
| real_remote_charset=detect_charset(HEADER_VALUE.cstr()); | |
| // tables | |
| { | |
| Value *valready=(Value *)tables.get(HEADER_NAME); | |
| bool existed=valready!=0; | |
| Table *table; | |
| if(existed) { | |
| // second+ appearence | |
| table=valready->get_table(); | |
| } else { | |
| // first appearence | |
| Table::columns_type columns=new ArrayString(1); | |
| *columns+=new String("value"); | |
| table=new Table(columns); | |
| } | |
| // this string becomes next row | |
| ArrayString& row=*new ArrayString(1); | |
| row+=&HEADER_VALUE; | |
| *table+=&row; | |
| // not existed before? add it | |
| if(!existed) | |
| tables.put(HEADER_NAME, new VTable(table)); | |
| } | |
| result.headers->put(HEADER_NAME, new VString(HEADER_VALUE)); | Array_iterator<const String*> i(aheaders); |
| i.next(); // skipping status | |
| for(;i.has_next();){ | |
| const char *line=i.next()->cstr(); | |
| if(!response.add_header(line)) | |
| throw Exception("http.response", &connect_string, "bad response from host - bad header \"%s\"", line); | |
| } | } |
| // filling $.cookies | |
| if(Value *vcookies=(Value *)tables.get("SET-COOKIE")) | |
| result.headers->put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table()))); | |
| } | } |
| if(as_text && raw_body_size>=3 && strncmp(raw_body, "\xEF\xBB\xBF", 3)==0){ | if (!real_remote_charset && !response.content_type.is_empty()) |
| // skip UTF-8 signature (BOM code) | real_remote_charset= detect_charset(response.content_type.cstr()); |
| raw_body+=3; | |
| raw_body_size-=3; | if(as_text) |
| if(!real_remote_charset) | real_remote_charset=charsets.checkBOM(raw_body, raw_body_size, real_remote_charset); |
| real_remote_charset=&UTF8_charset; | |
| if (!real_remote_charset) | |
| real_remote_charset=asked_remote_charset; // never null | |
| for(Array_iterator<ResponseHeaders::Header> i(response.headers); i.has_next(); ){ | |
| ResponseHeaders::Header header=i.next(); | |
| header.transcode(*real_remote_charset, r.charsets.source()); | |
| String &header_value=*new String(header.value, String::L_TAINTED); | |
| tables_update(vtables->hash(), header.name, header_value); | |
| result.headers->put(header.name, new VString(header_value)); | |
| } | } |
| // filling $.cookies | |
| if(Value *vcookies=vtables->hash().get("SET-COOKIE")) | |
| result.headers->put(HTTP_COOKIES_NAME, new VTable(parse_cookies(r, vcookies->get_table()))); | |
| // output response | // output response |
| String::C real_body=String::C(raw_body, raw_body_size); | String::C real_body=String::C(raw_body, raw_body_size); |
| if(as_text && transcode_text_result && raw_body_size) { // raw_body_size must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below | if(as_text && transcode_text_result && raw_body_size) { // raw_body_size must be checked because transcode returns CONST string in case length==0, which contradicts hacking few lines below |
| // defaulting to used-asked charset [it's never empty!] | |
| if(!real_remote_charset) | |
| real_remote_charset=asked_remote_charset; | |
| real_body=Charset::transcode(real_body, *real_remote_charset, r.charsets.source()); | real_body=Charset::transcode(real_body, *real_remote_charset, r.charsets.source()); |
| } | } |
| result.str=const_cast<char *>(real_body.str); // hacking a little | result.str=const_cast<char *>(real_body.str); // hacking a little |