--- parser3/src/main/pa_string.C 2006/04/09 13:38:47 1.207 +++ parser3/src/main/pa_string.C 2008/08/21 15:58:12 1.217 @@ -5,9 +5,7 @@ Author: Alexandr Petrosian (http://paf.design.ru) */ -static const char * const IDENT_STRING_C="$Date: 2006/04/09 13:38:47 $"; - -#include "pcre.h" +static const char * const IDENT_STRING_C="$Date: 2008/08/21 15:58:12 $"; #include "pa_string.h" #include "pa_exception.h" @@ -15,6 +13,8 @@ static const char * const IDENT_STRING_C #include "pa_dictionary.h" #include "pa_charset.h" +#include "pcre.h" + const String String::Empty; int pa_atoi(const char* str, const String* problem_source) { @@ -279,6 +279,14 @@ String& String::append_strdup(const char return *this; } +size_t String::length(Charset& charset) const { + if(charset.isUTF8()){ + const XMLByte* srcPtr=(const XMLByte*)cstrm(); + return lengthUTF8(srcPtr, srcPtr+body.length()); + } else + return body.length(); +} + /// @todo check in doc: whether it documents NOW bad situation "abc".mid(-1, 3) =were?="ab" String& String::mid(size_t substr_begin, size_t substr_end) const { String& result=*new String; @@ -299,6 +307,43 @@ String& String::mid(size_t substr_begin, return result; } +// from, to and helper_length in characters, not in bytes (it's important for utf-8) +String& String::mid(Charset& charset, size_t from, size_t to, size_t helper_length) const { + String& result=*new String; + + size_t self_length=(helper_length)?helper_length:length(charset); + + if(!self_length) + return result; + + from=min(min(to, from), self_length); + to=min(max(to, from), self_length); + + size_t substr_length=to-from; + + if(!substr_length) + return result; + + if(charset.isUTF8()){ + const XMLByte* srcPtr=(const XMLByte*)cstrm(); + const XMLByte* srcEnd=srcPtr+body.length(); + + // convert 'from' and 'substr_length' from 'characters' to 'bytes' + from=getUTF8BytePos(srcPtr, srcEnd, from); + substr_length=getUTF8BytePos(srcPtr+from, srcEnd, substr_length); + if(!substr_length) + return result; + } + + // first: their langs + result.langs.append(result.body, langs, from, substr_length); + // next: letters themselves + result.body=body.mid(from, substr_length); + + ASSERT_STRING_INVARIANT(result); + return result; +} + size_t String::pos(const String::Body substr, size_t this_offset, Language lang) const { size_t substr_length=substr.length(); while(true) { @@ -319,6 +364,32 @@ size_t String::pos(const String& substr, return pos(substr.body, this_offset, lang); } +size_t String::pos(Charset& charset, const String& substr, + size_t this_offset, Language lang) const { + + size_t result; + if(charset.isUTF8()){ + const XMLByte* srcPtr=(const XMLByte*)cstrm(); + const XMLByte* srcEnd=srcPtr+body.length(); + + // convert 'this_offset' from 'characters' to 'bytes' + this_offset=getUTF8BytePos(srcPtr, srcEnd, this_offset); + + result=pos(substr.body, this_offset, lang); + if(result==CORD_NOT_FOUND) + return STRING_NOT_FOUND; + + // convert 'result' from 'bytes' to 'characters' + result=getUTF8CharPos(srcPtr, srcEnd, result); + } else { + result=pos(substr.body, this_offset, lang); + if(result==CORD_NOT_FOUND) + return STRING_NOT_FOUND; + } + + return result; +} + void String::split(ArrayString& result, size_t& pos_after, const char* delim, @@ -364,32 +435,42 @@ void String::split(ArrayString& result, } } -static void regex_options(const String* options, int *result, bool& need_pre_post_match){ +enum Match_feature { + MF_NEED_PRE_POST_MATCH = 0x01, + MF_JUST_COUNT_MATCHES = 0x02 +}; + +static void regex_options(const String* options, int* result, int* match_features){ struct Regex_option { - const char* keyL; - const char* keyU; - int clear, set; + const char* key; + const char* keyAlt; + int clear; + int set; int *result; - bool *flag; + int flag; } regex_option[]={ {"i", "I", 0, PCRE_CASELESS, result, 0}, // a=A - {"s", "S", 0, PCRE_DOTALL, result, 0}, // \n\n$ [default] - {"x", "U", 0, PCRE_EXTENDED, result, 0}, // whitespace in regex ignored + {"s", "S", 0, PCRE_DOTALL, result, 0}, // ^\n\n$ [default] {"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result, 0}, // ^aaa\n$^bbb\n$ + {"x", 0, 0, PCRE_EXTENDED, result, 0}, // whitespace in regex ignored + {"U", 0, 0, PCRE_UNGREEDY, result, 0}, // ungreedy patterns (greedy by default) {"g", "G", 0, 1, result+1, 0}, // many rows - {"'", 0, 0, 0, 0, &need_pre_post_match}, + {"'", 0, 0, 0, 0, MF_NEED_PRE_POST_MATCH}, + {"n", 0, 0, 0, 0, MF_JUST_COUNT_MATCHES}, {0, 0, 0, 0, 0, 0} }; result[0]=PCRE_EXTRA | PCRE_DOTALL | PCRE_DOLLAR_ENDONLY; result[1]=0; if(options && !options->is_empty()) - for(Regex_option *o=regex_option; o->keyL; o++) - if(options->pos(o->keyL)!=STRING_NOT_FOUND - || (o->keyU && options->pos(o->keyU)!=STRING_NOT_FOUND)) { - if(o->flag) - *o->flag=true; - else { // result + for(Regex_option *o=regex_option; o->key; o++) + if( + options->pos(o->key)!=STRING_NOT_FOUND + || (o->keyAlt && options->pos(o->keyAlt)!=STRING_NOT_FOUND) + ){ + if(o->flag){ + (*match_features) |= o->flag; + } else { *o->result &= ~o->clear; *o->result |= o->set; } @@ -400,7 +481,7 @@ Table* String::match(Charset& source_cha const String& regexp, const String* options, Row_action row_action, void *info, - bool& just_matched) const { + int& matches_count) const { if(regexp.is_empty()) throw Exception(0, 0, @@ -409,9 +490,16 @@ Table* String::match(Charset& source_cha const char* pattern=regexp.cstr(String::L_UNSPECIFIED); // fix any tainted with L_REGEX const char* errptr; int erroffset; - bool need_pre_post_match=false; - int option_bits[2]={0}; regex_options(options, option_bits, need_pre_post_match); + int option_bits[2]={0}; + int match_features=0; + regex_options(options, option_bits, &match_features); + bool need_pre_post_match=(match_features & MF_NEED_PRE_POST_MATCH) != 0; + bool just_count_matches=(match_features & MF_JUST_COUNT_MATCHES) != 0; bool global=option_bits[1]!=0; + + if(source_charset.isUTF8()) + option_bits[0]=option_bits[0] | PCRE_UTF8; + pcre *code=pcre_compile(pattern, option_bits[0], &errptr, &erroffset, source_charset.pcre_tables); @@ -451,18 +539,19 @@ Table* String::match(Charset& source_cha if(exec_substrings==PCRE_ERROR_NOMATCH) { pcre_free(code); row_action(table, 0/*last time, no raw*/, 0, 0, poststart, postfinish, info); - if(global || subpatterns) - return &table; // global or with subpatterns=true+result - else { - just_matched=false; return 0; // not global=no result - } + // if(global || subpatterns) + // return &table; // global or with subpatterns=true+result + // else { + // just_matched=false; return 0; // not global=no result + // } + return just_count_matches ? 0 : &table; } if(exec_substrings<0) { pcre_free(code); throw Exception(0, ®exp, - "regular expression execute error (%d)", + print_pcre_exec_error_text(exec_substrings), exec_substrings); } @@ -484,12 +573,14 @@ Table* String::match(Charset& source_cha *row+=&mid(ovector[i*2+0], ovector[i*2+1]); // .i column value } + matches_count++; row_action(table, row, prestart, prefinish, poststart, postfinish, info); if(!global || prestart==poststart) { // not global | going to hang pcre_free(code); row_action(table, 0/*last time, no row*/, 0, 0, poststart, postfinish, info); - return &table; + return just_count_matches ? 0 : &table; + // return &table; } prestart=poststart; @@ -506,8 +597,9 @@ String& String::change_case(Charset& sou return result; char* new_cstr=cstrm(); - size_t new_cstr_len=length(); + if(source_charset.isUTF8()) { + size_t new_cstr_len=length(); switch(kind) { case CC_UPPER: change_case_UTF8((const XMLByte*)new_cstr, new_cstr_len, (XMLByte*)new_cstr, new_cstr_len, UTF8CaseToUpper); @@ -556,6 +648,13 @@ String& String::change_case(Charset& sou return result; } +const String& String::escape(Charset& source_charset) const { + if(is_empty()) + return *this; + + return Charset::escape(*this, source_charset); +} + const String& String::replace(const Dictionary& dict) const { String& result=*new String(); const char* old_cstr=cstr();