--- parser3/src/main/pa_string.C 2004/10/07 09:22:01 1.201 +++ parser3/src/main/pa_string.C 2009/05/13 07:35:42 1.224 @@ -1,19 +1,18 @@ /** @file Parser: string class. @see untalength_t.C. - Copyright (c) 2001-2004 ArtLebedev Group (http://www.artlebedev.com) + Copyright (c) 2001-2009 ArtLebedev Group (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) */ -static const char * const IDENT_STRING_C="$Date: 2004/10/07 09:22:01 $"; - -#include "pcre.h" +static const char * const IDENT_STRING_C="$Date: 2009/05/13 07:35:42 $"; #include "pa_string.h" #include "pa_exception.h" #include "pa_table.h" #include "pa_dictionary.h" #include "pa_charset.h" +#include "pa_vregex.h" const String String::Empty; @@ -104,7 +103,7 @@ double pa_atod(const char* str, const St #ifndef DOXYGEN typedef struct { ssize_t countdown; - char target; /* Character we're looking for */ + int target; /* Character we're looking for */ } chr_data; #endif static int CORD_range_contains_chr_greater_then_proc(char c, size_t size, void* client_data) @@ -163,7 +162,7 @@ Table string_match_table_template(new St String::Body String::Body::Format(int value) { char local[MAX_NUMBER]; size_t length=snprintf(local, MAX_NUMBER, "%d", value); - return String::Body(pa_strdup(local, length), length); + return String::Body(pa_strdup(local, length)); } String::Body String::Body::trim(String::Trim_kind kind, const char* chars, @@ -229,21 +228,18 @@ static int CORD_batched_iter_fn_generic_ }; uint String::Body::hash_code() const { uint result=0; - CORD_iter5(body, 0, - CORD_batched_iter_fn_generic_hash_code, - CORD_batched_iter_fn_generic_hash_code, &result); + if (body && CORD_IS_STRING(body)){ + generic_hash_code(result, body); + } else { + CORD_iter5(body, 0, + CORD_batched_iter_fn_generic_hash_code, + CORD_batched_iter_fn_generic_hash_code, &result); + } return result; } // String methods -String::String(const char* cstr, size_t helper_length, bool tainted): body(CORD_EMPTY) { - append_help_length(cstr, helper_length, tainted?L_TAINTED:L_CLEAN); -} -String::String(const String::C cstr, bool tainted): body(CORD_EMPTY) { - append_know_length(cstr.str, cstr.length, tainted?L_TAINTED:L_CLEAN); -} - String& String::append_know_length(const char* str, size_t known_length, Language lang) { if(!known_length) return *this; @@ -279,6 +275,26 @@ String& String::append_strdup(const char return *this; } +int CORD_batched_len(const char* s, size_t* len){ + (*len) += lengthUTF8( (const XMLByte *)s, (const XMLByte *)s+strlen(s)); + return 0; +} + +// can be called only for IS_FUNCTION(CORD) which are used only in Lang +int CORD_batched_len(const char, size_t *len){ + (*len)++; + return 0; +} + +size_t String::length(Charset& charset) const { + if(charset.isUTF8()){ + size_t len=0; + body.for_each(CORD_batched_len, CORD_batched_len, &len); + return len; + } else + return body.length(); +} + /// @todo check in doc: whether it documents NOW bad situation "abc".mid(-1, 3) =were?="ab" String& String::mid(size_t substr_begin, size_t substr_end) const { String& result=*new String; @@ -299,6 +315,43 @@ String& String::mid(size_t substr_begin, return result; } +// from, to and helper_length in characters, not in bytes (it's important for utf-8) +String& String::mid(Charset& charset, size_t from, size_t to, size_t helper_length) const { + String& result=*new String; + + size_t self_length=(helper_length)?helper_length:length(charset); + + if(!self_length) + return result; + + from=min(min(to, from), self_length); + to=min(max(to, from), self_length); + + size_t substr_length=to-from; + + if(!substr_length) + return result; + + if(charset.isUTF8()){ + const XMLByte* srcPtr=(const XMLByte*)cstrm(); + const XMLByte* srcEnd=srcPtr+body.length(); + + // convert 'from' and 'substr_length' from 'characters' to 'bytes' + from=getUTF8BytePos(srcPtr, srcEnd, from); + substr_length=getUTF8BytePos(srcPtr+from, srcEnd, substr_length); + if(!substr_length) + return result; + } + + // first: their langs + result.langs.append(result.body, langs, from, substr_length); + // next: letters themselves + result.body=body.mid(from, substr_length); + + ASSERT_STRING_INVARIANT(result); + return result; +} + size_t String::pos(const String::Body substr, size_t this_offset, Language lang) const { size_t substr_length=substr.length(); while(true) { @@ -319,6 +372,32 @@ size_t String::pos(const String& substr, return pos(substr.body, this_offset, lang); } +size_t String::pos(Charset& charset, const String& substr, + size_t this_offset, Language lang) const { + + size_t result; + if(charset.isUTF8()){ + const XMLByte* srcPtr=(const XMLByte*)cstrm(); + const XMLByte* srcEnd=srcPtr+body.length(); + + // convert 'this_offset' from 'characters' to 'bytes' + this_offset=getUTF8BytePos(srcPtr, srcEnd, this_offset); + + result=pos(substr.body, this_offset, lang); + if(result==CORD_NOT_FOUND) + return STRING_NOT_FOUND; + + // convert 'result' from 'bytes' to 'characters' + result=getUTF8CharPos(srcPtr, srcEnd, result); + } else { + result=pos(substr.body, this_offset, lang); + if(result==CORD_NOT_FOUND) + return STRING_NOT_FOUND; + } + + return result; +} + void String::split(ArrayString& result, size_t& pos_after, const char* delim, @@ -364,107 +443,31 @@ void String::split(ArrayString& result, } } -static void regex_options(const String* options, int *result, bool& need_pre_post_match){ - struct Regex_option { - const char* keyL; - const char* keyU; - int clear, set; - int *result; - bool *flag; - } regex_option[]={ - {"i", "I", 0, PCRE_CASELESS, result, 0}, // a=A - {"s", "S", 0, PCRE_DOTALL, result, 0}, // \n\n$ [default] - {"x", "U", 0, PCRE_EXTENDED, result, 0}, // whitespace in regex ignored - {"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result, 0}, // ^aaa\n$^bbb\n$ - {"g", "G", 0, 1, result+1, 0}, // many rows - {"'", 0, 0, 0, 0, &need_pre_post_match}, - {0, 0, 0, 0, 0, 0} - }; - result[0]=PCRE_EXTRA | PCRE_DOTALL | PCRE_DOLLAR_ENDONLY; - result[1]=0; - - if(options && !options->is_empty()) - for(Regex_option *o=regex_option; o->keyL; o++) - if(options->pos(o->keyL)!=STRING_NOT_FOUND - || (o->keyU && options->pos(o->keyU)!=STRING_NOT_FOUND)) { - if(o->flag) - *o->flag=true; - else { // result - *o->result &= ~o->clear; - *o->result |= o->set; - } - } -} +Table* String::match(VRegex* vregex, + Row_action row_action, void *info, + int& matches_count) const { -Table* String::match(Charset& source_charset, - const String& regexp, - const String* options, - Row_action row_action, void *info, - bool& just_matched) const { - if(regexp.is_empty()) - throw Exception(0, - 0, - "regexp is empty"); - - const char* pattern=regexp.cstr(); - const char* errptr; - int erroffset; - bool need_pre_post_match=false; - int option_bits[2]={0}; regex_options(options, option_bits, need_pre_post_match); - bool global=option_bits[1]!=0; - pcre *code=pcre_compile(pattern, option_bits[0], - &errptr, &erroffset, - source_charset.pcre_tables); - - if(!code) - throw Exception(0, - ®exp.mid(erroffset, regexp.length()), - "regular expression syntax error - %s", errptr); - - int subpatterns=pcre_info(code, 0, 0); - if(subpatterns<0) { - pcre_free(code); - throw Exception(0, - ®exp, - "pcre_info error (%d)", - subpatterns); - } + // vregex->info(); // I have no idea what does it for? + + bool need_pre_post_match=vregex->is_pre_post_match_needed(); + bool global=vregex->is_global_search(); const char* subject=cstr(); size_t subject_length=strlen(subject); - const int oveclength=(1/*match*/+MAX_MATCH_GROUPS)*3; - int ovector[oveclength]; + const int ovector_size=(1/*match*/+MAX_MATCH_GROUPS)*3; + int ovector[ovector_size]; - // create table Table::Action_options table_options; Table& table=*new Table(string_match_table_template, table_options); - int exec_option_bits=0; int prestart=0; int poststart=0; int postfinish=length(); while(true) { - int exec_substrings=pcre_exec(code, 0, - subject, subject_length, prestart, - exec_option_bits, ovector, oveclength); - - if(exec_substrings==PCRE_ERROR_NOMATCH) { - pcre_free(code); - row_action(table, 0/*last time, no raw*/, 0, 0, poststart, postfinish, info); - if(global || subpatterns) - return &table; // global or with subpatterns=true+result - else { - just_matched=false; return 0; // not global=no result - } - } + int exec_result=vregex->exec(subject, subject_length, ovector, ovector_size, prestart); - if(exec_substrings<0) { - pcre_free(code); - throw Exception(0, - ®exp, - "regular expression execute error (%d)", - exec_substrings); - } + if(exec_result<0) // only PCRE_ERROR_NOMATCH might be here, other negative results cause an exception + break; int prefinish=ovector[0]; poststart=ovector[1]; @@ -479,25 +482,22 @@ Table* String::match(Charset& source_cha *row+=&Empty; // .postmatch } - for(int i=1; iis_just_count() ? 0 : &table; } String& String::change_case(Charset& source_charset, Change_case_kind kind) const { @@ -506,8 +506,9 @@ String& String::change_case(Charset& sou return result; char* new_cstr=cstrm(); - size_t new_cstr_len=length(); + if(source_charset.isUTF8()) { + size_t new_cstr_len=length(); switch(kind) { case CC_UPPER: change_case_UTF8((const XMLByte*)new_cstr, new_cstr_len, (XMLByte*)new_cstr, new_cstr_len, UTF8CaseToUpper); @@ -556,6 +557,13 @@ String& String::change_case(Charset& sou return result; } +const String& String::escape(Charset& source_charset) const { + if(is_empty()) + return *this; + + return Charset::escape(*this, source_charset); +} + const String& String::replace(const Dictionary& dict) const { String& result=*new String(); const char* old_cstr=cstr(); @@ -609,52 +617,75 @@ static int serialize_lang_piece(char ala } String::Cm String::serialize(size_t prolog_length) const { size_t fragments_count=langs.count(); + size_t body_length=body.length(); size_t buf_length= prolog_length //1 +sizeof(size_t) //2 - +fragments_count*(sizeof(char)+sizeof(size_t)) //3 - +body.length() //4 - +1; // for zero terminator used in deserialize + +body_length //3 + +1 // 4 for zero terminator used in deserialize + +sizeof(size_t) //5 + +fragments_count*(sizeof(char)+sizeof(size_t)); //6 + String::Cm result(new(PointerFreeGC) char[buf_length], buf_length); // 1: prolog char *cur=result.str+prolog_length; - // 2: langs.count [WARNING: not cast, addresses must be %4=0 on sparc] + // 2: chars.count [WARNING: not cast, addresses must be %4=0 on sparc] + memcpy(cur, &body_length, sizeof(body_length)); cur+=sizeof(body_length); + // 3: letters + body.for_each(serialize_body_char, serialize_body_piece, &cur); + // 4: zero terminator + *cur++=0; + // 5: langs.count [WARNING: not cast, addresses must be %4=0 on sparc] memcpy(cur, &fragments_count, sizeof(fragments_count)); cur+=sizeof(fragments_count); - // 3: lang info + // 6: lang info langs.for_each(body, serialize_lang_piece, &cur); - // 4: letters - body.for_each(serialize_body_char, serialize_body_piece, &cur); - // 5: zero terminator - *cur=0; return result; } -bool String::deserialize(size_t prolog_length, void *buf, size_t buf_length) { - if(buf_length<=prolog_length) +bool String::deserialize(size_t prolog_size, void *buf, size_t buf_size) { + size_t in_buf=buf_size; + if(in_buf<=prolog_size) return false; - buf_length-=prolog_length; - buf_length-=1; // 5: zero terminator + in_buf-=prolog_size; // 1: prolog - const char* cur=(const char* )buf+prolog_length; + const char* cur=(const char* )buf+prolog_size; - // 2: langs.count + // 2: chars.count + size_t body_length; + if(in_bufbody_length) + return false; // file curruption // uchar needed to prevent propagating 0x80 bit to upper bytes langs.append(total_length, (String::Language)(uchar)lang, fragment_length); - total_length+=fragment_length; - - buf_length-=piece_length; + total_length=combined_length; + in_buf-=piece_length; } - // 4: letters - if(buf_length!=total_length) + if(total_length!=body_length) // length(all language fragments) vs length(letters) return false; - - // serialize wrote extra zero byte there, we can rely on that - body=String::Body(cur, buf_length); } + if(in_buf!=0) // some strange extra bytes + return false; ASSERT_STRING_INVARIANT(*this); return true; @@ -719,7 +750,7 @@ void String::dump() const { langs.dump(); } const String& String::trim(String::Trim_kind kind, const char* chars) const { - if(!length()) + if(is_empty()) return *this; size_t substr_begin, substr_length;