--- parser3/src/main/pa_string.C 2009/06/20 02:16:18 1.228 +++ parser3/src/main/pa_string.C 2011/05/05 23:45:45 1.237 @@ -5,7 +5,7 @@ Author: Alexandr Petrosian (http://paf.design.ru) */ -static const char * const IDENT_STRING_C="$Date: 2009/06/20 02:16:18 $"; +static const char * const IDENT_STRING_C="$Date: 2011/05/05 23:45:45 $"; #include "pa_string.h" #include "pa_exception.h" @@ -166,41 +166,108 @@ String::Body String::Body::Format(int va } String::Body String::Body::trim(String::Trim_kind kind, const char* chars, - size_t* out_start, size_t* out_length) const { + size_t* out_start, size_t* out_length, Charset* source_charset) const { size_t our_length=length(); if(!our_length) return *this; - if(!chars) - chars=" \t\n"; // white space + + // check if any UTF-8 in chars + bool fast=true; + if(chars && source_charset && source_charset->isUTF8()){ + const char* pos=chars; + while(unsigned char c=*pos++) + if(c>127){ + fast=false; + break; + } + } size_t start=0; size_t end=our_length; - // from left... - if(kind!=TRIM_END) { - CORD_pos pos; set_pos(pos, 0); - while(true) { - char c=CORD_pos_fetch(pos); - if(strchr(chars, c)) { - if(++start==our_length) - return 0; // all chars are empty, just return empty string - } else - break; - - CORD_next(pos); - } - } - // from right.. - if(kind!=TRIM_START) { - CORD_pos pos; set_pos(pos, end-1); - while(true) { - char c=CORD_pos_fetch(pos); - if(strchr(chars, c)) { - if(--end==0) // optimization: NO need to check for 'end>=start', that's(<) impossible - return 0; // all chars are empty, just return empty string - } else - break; + if(!chars) + chars=" \t\n"; // white space + + if(fast){ + // from left... + if(kind!=TRIM_END) { + CORD_pos pos; set_pos(pos, 0); + while(true) { + char c=CORD_pos_fetch(pos); + if(strchr(chars, c)) { + if(++start==our_length) + return 0; // all chars are empty, just return empty string + } else + break; + + CORD_next(pos); + } + } + + // from right.. + if(kind!=TRIM_START) { + CORD_pos pos; set_pos(pos, end-1); + while(true) { + char c=CORD_pos_fetch(pos); + if(strchr(chars, c)) { + if(--end==0) // optimization: NO need to check for 'end>=start', that's(<) impossible + return 0; // all chars are empty, just return empty string + } else + break; + + CORD_prev(pos); + } + } + } else { + const XMLByte* src_begin=(const XMLByte*)cstr(); + const XMLByte* src_end=src_begin+our_length; - CORD_prev(pos); + // from left... + if(kind!=TRIM_END) { + while(src_begin127 && *src_begin<0xC0)) + char_length++; + + bool found=false; + for(const char* chars_byte=chars; chars_byte=strchr(chars_byte, *ptr); chars_byte++) + if(strncmp(chars_byte, (const char*)ptr, char_length)==0){ + found=true; + break; + } + + if(found){ + start+=char_length; + if(start==our_length) + return 0; // all chars are empty, just return empty string + } else + break; + } + } + + // from right.. + if(kind!=TRIM_START) { + while(src_begin127 && *src_end<0xC0)) + char_length++; + + bool found=false; + for(const char* chars_byte=chars; chars_byte=strchr(chars_byte, *src_end); chars_byte++) + if(strncmp(chars_byte, (const char*)src_end, char_length)==0){ + found=true; + break; + } + + if(found){ + end-=char_length; + if(end==0) + return 0; // all chars are empty, just return empty string + } else + break; + } } } @@ -284,22 +351,31 @@ String& String::append_strdup(const char return *this; } -int CORD_batched_len(const char* s, size_t* len){ - (*len) += lengthUTF8( (const XMLByte *)s, (const XMLByte *)s+strlen(s)); - return 0; +struct CORD_length_info { + size_t len; + size_t skip; +}; + +int CORD_batched_len(const char* s, CORD_length_info* info){ + info->len += lengthUTF8( (const XMLByte *)s, (const XMLByte *)s+strlen(s)); return 0; } -// can be called only for IS_FUNCTION(CORD) which are used only in Lang -int CORD_batched_len(const char, size_t *len){ - (*len)++; +// can be called only for IS_FUNCTION(CORD) which are used in large String::Body::mid +int CORD_batched_len(const char c, CORD_length_info* info){ + if (info->skip==0){ + info->len++; + info->skip = lengthUTF8Char(c)-1; + } else { + info->skip--; + } return 0; } size_t String::length(Charset& charset) const { if(charset.isUTF8()){ - size_t len=0; - body.for_each(CORD_batched_len, CORD_batched_len, &len); - return len; + CORD_length_info info = {0, 0}; + body.for_each(CORD_batched_len, CORD_batched_len, &info); + return info.len; } else return body.length(); } @@ -342,12 +418,12 @@ String& String::mid(Charset& charset, si return result; if(charset.isUTF8()){ - const XMLByte* srcPtr=(const XMLByte*)cstrm(); - const XMLByte* srcEnd=srcPtr+body.length(); + const XMLByte* src_begin=(const XMLByte*)cstr(); + const XMLByte* src_end=src_begin+body.length(); // convert 'from' and 'substr_length' from 'characters' to 'bytes' - from=getUTF8BytePos(srcPtr, srcEnd, from); - substr_length=getUTF8BytePos(srcPtr+from, srcEnd, substr_length); + from=getUTF8BytePos(src_begin, src_end, from); + substr_length=getUTF8BytePos(src_begin+from, src_end, substr_length); if(!substr_length) return result; } @@ -384,33 +460,31 @@ size_t String::pos(const String& substr, size_t String::pos(Charset& charset, const String& substr, size_t this_offset, Language lang) const { - size_t result; if(charset.isUTF8()){ - const XMLByte* srcPtr=(const XMLByte*)cstrm(); + const XMLByte* srcPtr=(const XMLByte*)cstr(); const XMLByte* srcEnd=srcPtr+body.length(); // convert 'this_offset' from 'characters' to 'bytes' this_offset=getUTF8BytePos(srcPtr, srcEnd, this_offset); - result=pos(substr.body, this_offset, lang); - if(result==CORD_NOT_FOUND) - return STRING_NOT_FOUND; - - // convert 'result' from 'bytes' to 'characters' - result=getUTF8CharPos(srcPtr, srcEnd, result); + size_t result=pos(substr.body, this_offset, lang); + return (result==CORD_NOT_FOUND) + ? STRING_NOT_FOUND + : getUTF8CharPos(srcPtr, srcEnd, result); // convert 'result' from 'bytes' to 'characters' } else { - result=pos(substr.body, this_offset, lang); - if(result==CORD_NOT_FOUND) - return STRING_NOT_FOUND; + size_t result=pos(substr.body, this_offset, lang); + return (result==CORD_NOT_FOUND) + ? STRING_NOT_FOUND + : result; } - - return result; } void String::split(ArrayString& result, size_t& pos_after, const char* delim, Language lang, int limit) const { + if(is_empty()) + return; size_t self_length=length(); if(size_t delim_length=strlen(delim)) { size_t pos_before; @@ -434,6 +508,8 @@ void String::split(ArrayString& result, size_t& pos_after, const String& delim, Language lang, int limit) const { + if(is_empty()) + return; if(!delim.is_empty()) { size_t pos_before; // while we have 'delim'... @@ -462,7 +538,7 @@ Table* String::match(VRegex* vregex, bool global=vregex->is_global_search(); const char* subject=cstr(); - size_t subject_length=strlen(subject); + size_t subject_length=length(); const int ovector_size=(1/*match*/+MAX_MATCH_GROUPS)*3; int ovector[ovector_size]; @@ -480,7 +556,13 @@ Table* String::match(VRegex* vregex, int prefinish=ovector[0]; poststart=ovector[1]; - ArrayString* row=new ArrayString; + + if (prestart==poststart && subject[poststart]=='\n'){ + prestart++; + continue; + } + + ArrayString* row=new ArrayString(3); if(need_pre_post_match) { *row+=&mid(0, prefinish); // .prematch column value *row+=&mid(prefinish, poststart); // .match @@ -493,7 +575,7 @@ Table* String::match(VRegex* vregex, for(int i=1; i=0 && ovector[i*2+1]>0)?&mid(ovector[i*2+0], ovector[i*2+1]):new String; // .i column value } matches_count++; @@ -676,9 +758,9 @@ bool String::deserialize(size_t prolog_s return false; // 3: letters body=String::Body(*cur?cur:0); -#ifdef STRING_LENGTH_CACHING - body.set_length(body_length); -#endif +#ifdef STRING_LENGTH_CACHING + body.set_length(body_length); +#endif cur+=body_length+1; in_buf-=body_length+1; @@ -725,7 +807,7 @@ bool String::deserialize(size_t prolog_s } const char* String::Body::v() const { - return CORD_to_const_char_star(body); + return CORD_to_const_char_star(body, length()); } void String::Body::dump() const { CORD_dump(body); @@ -733,7 +815,7 @@ void String::Body::dump() const { const char* String::Languages::v() const { if(opt.is_not_just_lang) - return CORD_to_const_char_star(langs); + return CORD_to_const_char_star(langs, 0); else return (const char*)&langs; } @@ -757,16 +839,18 @@ const char* String::v() const { return buf; } + void String::dump() const { body.dump(); langs.dump(); } -const String& String::trim(String::Trim_kind kind, const char* chars) const { + +const String& String::trim(String::Trim_kind kind, const char* chars, Charset* source_charset) const { if(is_empty()) return *this; size_t substr_begin, substr_length; - Body new_body=body.trim(kind, chars, &substr_begin, &substr_length); + Body new_body=body.trim(kind, chars, &substr_begin, &substr_length, source_charset); if(new_body==body) // we received unchanged pointer, do likewise return *this; // new_body differs from body, adjust langs along