--- parser3/src/main/pa_string.C 2003/03/05 11:42:04 1.172.2.19 +++ parser3/src/main/pa_string.C 2003/03/20 13:09:54 1.172.2.21.2.8 @@ -5,7 +5,7 @@ Author: Alexandr Petrosian (http://paf.design.ru) */ -static const char* IDENT_STRING_C="$Date: 2003/03/05 11:42:04 $"; +static const char* IDENT_STRING_C="$Date: 2003/03/20 13:09:54 $"; #include "pcre.h" @@ -22,39 +22,76 @@ static const char* IDENT_STRING_C="$Date const int MAX_MATCH_GROUPS=100; class String_match_table_template_columns: public ArrayString { - Pool pool; public: String_match_table_template_columns() { - *this+=StringPtr(new String("prematch")); - *this+=StringPtr(new String("match")); - *this+=StringPtr(new String("postmatch")); + *this+=new String("prematch"); + *this+=new String("match"); + *this+=new String("postmatch"); for(int i=0; i(1), fsize(0) { +String::ArrayFragment& String::ArrayFragment::append(const ArrayFragment& src, + size_t substr_start, size_t substr_end) { + if(substr_start==substr_end) + return *this; + + size_t fragment_start=0; + size_t fragment_end; + for(Array_iterator i(src); ; fragment_start=fragment_end) { + const element_type& fragment=i.next(); + fragment_end=fragment_start+fragment.size; + + if(substr_startfragment_end) { // are there more fragments including pieces of 'substr'? + const element_type& fragment=i.next(); + fragment_end=fragment_start+fragment.size; + + if(substr_end>fragment_end) // are there still more? + simple_append(Fragment(fragment.lang, fragment.size)); // appending whole fragment + else { // no, it was last + simple_append(Fragment(fragment.lang, substr_end-fragment_start)); + goto break2; + } + } + + break; + } +break2: + + return *this; +} +/* +void String::ArrayFragment::mid(ArrayFragment& result, size_t substr_start, size_t substr_end) { +} +*/ + +// String methods + +String::String(const char* src, size_t src_size, bool tainted): body(CORD_EMPTY) { if(src) if(tainted) - APPEND_TAINTED(src, src_size, 0, 0); + APPEND_TAINTED(src, src_size); else - APPEND_CLEAN(src, src_size, 0, 0); + APPEND_CLEAN(src, src_size); } -String::String(const String& src): Array(src.count()) { - append(src, UL_PASS_APPENDED); -} +String::String(const String& src): body(src.body), fragments(src.fragments) {} -String& String::real_append(STRING_APPEND_PARAMS) { +String& String::append(const char* src, size_t size, Untaint_lang lang) { if(!src) return *this; if(!size) @@ -62,297 +99,125 @@ String& String::real_append(STRING_APPEN if(!size) return *this; - if(is_full()) - expand(fdelta); - - String_fragment *fragment=&felements[fused++]; - fragment->ptr=src; - fragment->size=size; - fragment->lang=lang; -#ifndef NO_STRING_ORIGIN - fragment->origin.file=file; - fragment->origin.line=line; -#endif - - fsize+=size; + body=CORD_cat_char_star(body, src, size); + fragments+=Fragment(lang, size); return *this; } +static int CORD_batched_iter_fn_generic_hash_code(const char* s, void * client_data) { + uint& result=*static_cast(client_data); + generic_hash_code(result, s); + return 0; +}; uint String::hash_code() const { uint result=0; - STRING_FOREACH_FRAGMENT( - result=generic_hash_code(result, fragment->ptr, fragment->size); - ); + CORD_iter5(body, 0, 0, CORD_batched_iter_fn_generic_hash_code, &result); return result; } -/// @todo move 'lang' skipping to pos -int String::cmp(int& partial, const String& src, - size_t this_offset, Untaint_lang lang) const { - partial=-1; - size_t a_size=size(); - this_offset=min(this_offset, a_size-1); - - const String_fragment *a_current=felements; - const String_fragment *b_current=src.felements; - size_t a_offset=this_offset; - size_t b_offset=0; - String_fragment *a_end=felements+fused; - String_fragment *b_end=src.felements+src.fused; - int result; - size_t pos=0; +const String& String::mid(size_t substr_start, size_t substr_end) const { + String& result=*new String; - bool a_break; - bool b_break; - while(true) { - a_break=(a_current==a_end); - b_break=(b_current==b_end); - if(a_break || b_break) - break; - - if(pos+a_current->size > this_offset) { - if(lang!=UL_UNSPECIFIED && a_current->lang>(String_UL)lang) - return -1; // wrong lang -- bail out - - int size_diff= - (a_current->size-a_offset)- - (b_current->size-b_offset); - - if(size_diff==0) { // a has same size as b - result=memcmp(a_current->ptr+a_offset, b_current->ptr+b_offset, - a_current->size-a_offset); - if(result) - return result; - pos+=a_current->size; - a_current++; a_offset=0; - b_current++; b_offset=0; - } else if(size_diff>0) { // a longer - result=memcmp(a_current->ptr+a_offset, b_current->ptr+b_offset, - b_current->size-b_offset); - if(result) - return result; - a_offset+=b_current->size-b_offset; - b_current++; b_offset=0; - } else { // b longer - result=memcmp(a_current->ptr+a_offset, b_current->ptr+b_offset, - a_current->size-a_offset); - if(result) - return result; - b_offset+=a_current->size-a_offset; - pos+=a_current->size; - a_current++; a_offset=0; - } - } else { - a_offset-=a_current->size; - pos+=a_current->size; - a_current++; - } - } - if(a_break==b_break) { // ended simultaneously - partial=0; return 0; - } else if(a_break) { // first bytes equal, but a ended before b - partial=1; return -1; - } else { - partial=2; return +1; - } -} - -/// @todo move 'lang' skipping to pos -int String::cmp(int& partial, const char* b_ptr, size_t src_size, - size_t this_offset, Untaint_lang lang) const { - partial=-1; - size_t a_size=size(); - size_t b_size=src_size?src_size:b_ptr?strlen(b_ptr):0; - this_offset=min(this_offset, a_size-1); - - const String_fragment *a_current=felements; - size_t a_offset=this_offset; - size_t b_offset=0; - String_fragment *a_end=felements+fused; - size_t pos=0; + substr_start=min(substr_start, size()); + substr_end=max(substr_start, substr_end); + if(substr_start==substr_end) + return result; - bool a_break; - bool b_break=b_size==0; - while(true) { - a_break=(a_current==a_end); - if(a_break || b_break) - break; - if(pos+a_current->size > this_offset) { - if(lang!=UL_UNSPECIFIED && a_current->lang>(String_UL)lang) - return -1; // wrong lang -- bail out - - int size_diff= - (a_current->size-a_offset)- - (b_size-b_offset); - - if(size_diff==0) { // a has same size as b - if(int result=memcmp(a_current->ptr+a_offset, b_ptr+b_offset, - a_current->size-a_offset)!=0) - return result; - pos+=a_current->size; - a_current++; a_offset=0; - b_break=true; - } else if (size_diff>0) { // a longer - if(int result=memcmp(a_current->ptr+a_offset, b_ptr+b_offset, - b_size-b_offset)!=0) - return result; - a_offset+=b_size-b_offset; - b_break=true; - } else { // b longer - if(int result=memcmp(a_current->ptr+a_offset, b_ptr+b_offset, - a_current->size-a_offset)!=0) - return result; - b_offset+=a_current->size-a_offset; - pos+=a_current->size; - a_current++; a_offset=0; - } - } else { - a_offset-=a_current->size; - pos+=a_current->size; - a_current++; - } - } - if(a_break==b_break) { // ended simultaneously - partial=0; return 0; - } else if(a_break) { // first bytes equal, but a ended before b - partial=1; return -1; - } else { - partial=2; return +1; - } -} + // first: letters themselves + result.body=CORD_substr(body, substr_start, substr_end-substr_start); -#ifndef NO_STRING_ORIGIN -const String_fragment::Origin& String::origin() const { - if(is_empty()) { - static const String_fragment::Origin empty_origin={"empty string"}; - return empty_origin; - } - - // determining origin by first piece or last appended piece - // because any of them can be constant=without origin: - // ex: ^load[/file] "document_root" + "/file" - // when last peice is constant, - // ex: parser_root_auto_path{dynamic} / auto.p{const} - // using first piece - String_fragment::Origin& first_origin=felements[0].origin; - return first_origin.file ? first_origin : felements[fused-1].origin; -} -#endif - -StringPtr String::mid(size_t start, size_t finish) const { - StringPtr result(new String()); - - start=min(start, size()); - finish=max(start, finish); - if(start==finish) - return result; + // next: their langs + result.fragments.append(fragments, substr_start, substr_end); - size_t pos=0; - STRING_FOREACH_FRAGMENT( - size_t item_finish=pos+fragment->size; - if(item_finish > start) { // started now or already? - bool started=result->is_empty(); // started now? - bool finished=finish <= item_finish; // finished now? - size_t offset=started?start-pos:0; - size_t size=finished?finish-pos:fragment->size; - result->APPEND( - fragment->ptr+offset, size-offset, - fragment->lang, - fragment->origin.file, fragment->origin.line); - if(finished) - goto break2; - } - pos+=fragment->size; - ); -break2: // SAPI::log(pool(), "piece of '%s' from %d to %d is '%s'", - //cstr(), start, finish, result.cstr()); + //cstr(), substr_start, substr_end, result.cstr()); return result; } -int String::pos(const String& substr, - int result, Untaint_lang lang) const { - size_t self_size=size(); - for(; size_t(result) i(fragments); i.has_next(); fragment_start=fragment_end) { + const Fragment& fragment=i.next(); + fragment_end=fragment_start+fragment.size; + + if(substr_startfragment_end) // end of substr OUT of current fragment? + break; + + return fragment.lang==lang?substr_end:STRING_NOT_FOUND; + } + + return STRING_NOT_FOUND; } -int String::pos(const char* substr, size_t substr_size, - int result, Untaint_lang lang) const { - size_t self_size=size(); - for(; size_t(result)=0 && limit; limit--) { - result+=mid(pos_after, pos_before); + for(; (pos_before=pos(delim, pos_after, lang))>=0 && limit; limit--) { + result+=&mid(pos_after, pos_before); pos_after=pos_before+delim_size; } // last piece if(pos_after=0 && limit; limit--) { - result+=mid(pos_after, pos_before); + result+=&mid(pos_after, pos_before); pos_after=pos_before+delim.size(); } // last piece if(pos_afterkeyL; o++) - if(options->pos(o->keyL)>=0 - || (o->keyU && options->pos(o->keyU)>=0)) { + if(options.pos(o->keyL)>=0 + || (o->keyU && options.pos(o->keyU)>=0)) { if(o->flag) *o->flag=true; else { // result @@ -384,50 +249,47 @@ static void regex_options(StringPtr opti } } -TablePtr String::match(Charset& source_charset, - StringPtr aorigin, +Table* String::match(Charset& source_charset, const String& regexp, - StringPtr options, + const String& options, Row_action row_action, void *info, - bool *was_global) const { - - if(regexp.is_empty()) + bool& just_matched) const { + if(!regexp) throw Exception(0, - aorigin, + 0, "regexp is empty"); - CharPtr pattern=regexp.cstr(); + const char* pattern=regexp.cstr(); const char* errptr; int erroffset; - bool need_pre_post_match=false; + bool need_pre_post_match=false; int option_bits[2]; regex_options(options, option_bits, need_pre_post_match); - if(was_global) - *was_global=option_bits[1]!=0; - pcre *code=pcre_compile(pattern.get(), option_bits[0], + bool global=option_bits[1]!=0; + pcre *code=pcre_compile(pattern, option_bits[0], &errptr, &erroffset, source_charset.pcre_tables); if(!code) throw Exception(0, - regexp.mid(erroffset, regexp.size()), + ®exp.mid(erroffset, regexp.size()), "regular expression syntax error - %s", errptr); - int info_substrings=pcre_info(code, 0, 0); - if(info_substrings<0) { + int subpatterns=pcre_info(code, 0, 0); + if(subpatterns<0) { pcre_free(code); throw Exception(0, - aorigin, + ®exp, "pcre_info error (%d)", - info_substrings); + subpatterns); } - CharPtr subject=cstr(); - int length=size(); + const char* subject=cstr(); + size_t subject_size=strlen(subject); const int ovecsize=(1/*match*/+MAX_MATCH_GROUPS)*3; int ovector[ovecsize]; // create table - TablePtr result(new Table(string_match_table_template)); + Table* table=new Table(string_match_table_template); int exec_option_bits=0; int prestart=0; @@ -435,50 +297,51 @@ TablePtr String::match(Charset& source_c int postfinish=size(); while(true) { int exec_substrings=pcre_exec(code, 0, - subject.get(), length, prestart, + subject, subject_size, prestart, exec_option_bits, ovector, ovecsize); if(exec_substrings==PCRE_ERROR_NOMATCH) { pcre_free(code); - row_action(result, ArrayStringPtr(0)/*last time, no raw*/, 0, 0, poststart, postfinish, info); - if(option_bits[1]) - return result; // global=true+result - else - return TablePtr(0);// not global=no result + row_action(table, 0/*last time, no raw*/, 0, 0, poststart, postfinish, info); + if(global || subpatterns) + return table; // global or with subpatterns=true+result + else { + just_matched=false; return 0; // not global=no result + } } if(exec_substrings<0) { pcre_free(code); throw Exception(0, - aorigin, + ®exp, "regular expression execute error (%d)", exec_substrings); } int prefinish=ovector[0]; poststart=ovector[1]; - object_ptr row(new ArrayString); + ArrayString* row=new ArrayString; if(need_pre_post_match) { - *row+=mid(0, prefinish); // .prematch column value - *row+=mid(prefinish, poststart); // .match - *row+=mid(poststart, postfinish); // .postmatch + *row+=&mid(0, prefinish); // .prematch column value + *row+=&mid(prefinish, poststart); // .match + *row+=&mid(poststart, postfinish); // .postmatch } else { - *row+=StringPtr(0); // .prematch column value - *row+=StringPtr(0); // .match - *row+=StringPtr(0); // .postmatch + *row+=0; // .prematch column value + *row+=0; // .match + *row+=0; // .postmatch } for(int i=1; i(kind)); // never a=b=0; // calm, compiler break; // never } - STRING_FOREACH_FRAGMENT( - char *new_cstr=new(pool) char[fragment->size]; - char *dest=new_cstr; - const char* end=fragment->ptr+fragment->size; - for(const char* current=fragment->ptr; currentAPPEND(new_cstr, fragment->size, - fragment->lang, - fragment->origin.file, fragment->origin.line); - ); + *dest++=(char)c; + } + result.body=new_cstr; + result.fragments.append(fragments, 0, fragments.count()); return result; } -/// @test if in some piece were found no dict words, append it, not it's duplicate -StringPtr String::replace(Pool& pool, const Dictionary& dict) const { - StringPtr result(new String()); - CharPtr lcstr(cstr()); - const char* current=lcstr.get(); - - STRING_FOREACH_FRAGMENT( -IFNDEF_NO_STRING_ORIGIN( - const char* joined_origin_file=fragment->origin.file; - const size_t joined_origin_line=fragment->origin.line; -); - String_UL joined_lang=fragment->lang; - const char* joined_ptr=current; - // calc size - size_t joined_size=0; - STRING_PREPARED_FOREACH_FRAGMENT( - if(fragment->lang==joined_lang) - joined_size+=fragment->size; - else - break; // before non-ours - ); - current+=joined_size; - - // pointers are after joined piece - // & one step back, see STRING_PREPARED_FOREACH_FRAGMENT - --fragment; - - char *new_cstr=new(pool) char[(size_t)ceil(joined_size*dict.max_ratio())]; - char *dest=new_cstr; - while(joined_size) { - // there is a fragment where first column starts 'joined_ptr' - if(Table::element_type row=dict.first_that_starts(joined_ptr, joined_size)) { - // get a=>b values - StringPtr a=row->get(0); - StringPtr b=row->get(1); - // skip 'a' in 'joined_ptr' && reduce work size - joined_ptr+=a->size(); joined_size-=a->size(); - // write 'b' to 'dest' && skip 'b' in 'dest' - b->store_to(dest, String::UL_AS_IS); dest+=b->size(); - } else { - // write a char to b && reduce work size - *dest++=*joined_ptr++; joined_size--; +const String& String::replace(const Dictionary& dict) const { + String& result=*new String(); + const char* old_cstr=cstr(); + const char* prematch_start=old_cstr; + + for(const char* current=old_cstr; *current; ) { + if(Table::element_type row=dict.first_that_starts(current)) { + // prematch + if(size_t prematch_size=current-prematch_start) { + result.body=CORD_cat_char_star(result.body, prematch_start, prematch_size); + result.fragments.append(fragments, prematch_start-old_cstr, prematch_size); } - } - result->APPEND(new_cstr, dest-new_cstr, joined_lang, - joined_origin_file, joined_origin_line); - ); + // match - return result; -} + const String* a=row->get(0); + // skip 'a' in 'current'; move prematch_start + current+=a->size(); prematch_start=current; + + if(row->count()>1) { // are there any b? + const String* b=row->get(1); + if(size_t b_size=b->size()) { + result.body=CORD_cat(result.body, b->cstr_to_cord()); + result.fragments.append(b->fragments, 0, b->fragments.count()); + } + } + } else // simply advance + current++; + } -StringPtr String::join_chains(Pool& pool) const { - StringPtr result(new String()); - - char *pooled_buf=new(pool) char[cstr_bufsize()]; - store_to(pooled_buf, String::UL_AS_IS); - const char* current=pooled_buf; - - STRING_FOREACH_FRAGMENT( -IFNDEF_NO_STRING_ORIGIN( - const char* joined_origin_file=fragment->origin.file; - const size_t joined_origin_line=fragment->origin.line; -); - String_UL joined_lang=fragment->lang; - const char* joined_ptr=current; - // calc size - size_t joined_size=0; - STRING_PREPARED_FOREACH_FRAGMENT( - if(fragment->lang==joined_lang) - joined_size+=fragment->size; - else - break; // before non-ours - ); - current+=joined_size; - - // pointers are after joined piece - // & one step back, see STRING_PREPARED_FOREACH_FRAGMENT - --fragment; - - result->APPEND(joined_ptr, joined_size, joined_lang, - joined_origin_file, joined_origin_line); - ); + // postmatch + if(size_t prematch_size=current-prematch_start) { + result.body=CORD_cat_char_star(result.body, prematch_start, prematch_size); + result.fragments.append(fragments, prematch_start-old_cstr, prematch_size); + } return result; } -double String::as_double() { +double String::as_double() const { double result; - char buf[MAX_STRING]; - if(size()>MAX_STRING-1) - throw Exception("number.format", - StringPtr(this), - "invalid number too long a string (%u>%u)", size(), MAX_STRING-1); - char *eol=store_to(buf, String::UL_AS_IS); *eol=0; - const char* cstr=buf; - - while(*cstr && isspace(*cstr)) - cstr++; - if(!*cstr) + const char *str=cstr(); + + while(*str && isspace(*str)) + str++; + if(!*str) return 0; char *error_pos; // 0xABC - if(cstr[0]=='0') - if(cstr[1]=='x' || cstr[1]=='X') - result=(double)(unsigned long)strtol(cstr, &error_pos, 0); + if(str[0]=='0') + if(str[1]=='x' || str[1]=='X') + result=(double)(unsigned long)strtol(str, &error_pos, 0); else - result=(double)strtod(cstr+1/*skip leading 0*/, &error_pos); + result=(double)strtod(str+1/*skip leading 0*/, &error_pos); else - result=(double)strtod(cstr, &error_pos); + result=(double)strtod(str, &error_pos); while(char c=*error_pos++) if(!isspace(c)) throw Exception("number.format", - StringPtr(this), + this, "invalid number (double)"); return result; } -int String::as_int() { +int String::as_int() const { int result; - char buf[MAX_STRING]; - if(size()>MAX_STRING-1) - throw Exception("number.format", - StringPtr(this), - "invalid number too long a string (%u>%u)", size(), MAX_STRING-1); - char *eol=store_to(buf, String::UL_AS_IS); *eol=0; - const char* cstr=buf; - - while(*cstr && isspace(*cstr)) - cstr++; - if(!*cstr) + const char *str=cstr(); + + while(*str && isspace(*str)) + str++; + if(!*str) return 0; char *error_pos; // 0xABC - if(cstr[0]=='0') - if(cstr[1]=='x' || cstr[1]=='X') - result=(int)(unsigned long)strtol(cstr, &error_pos, 0); + if(str[0]=='0') + if(str[1]=='x' || str[1]=='X') + result=(int)(unsigned long)strtol(str, &error_pos, 0); else - result=(int)strtol(cstr+1/*skip leading 0*/, &error_pos, 0); + result=(int)strtol(str+1/*skip leading 0*/, &error_pos, 0); else - result=(int)strtol(cstr, &error_pos, 0); + result=(int)strtol(str, &error_pos, 0); while(char c=*error_pos++) if(!isspace(c)) throw Exception("number.format", - StringPtr(this), + this, "invalid number (int)"); return result; @@ -701,52 +500,76 @@ inline uint uchars2uint(uchar *bytes) { | bytes[0]; } -void String::serialize(Pool& pool, size_t prolog_size, char *& buf, size_t& buf_size) const { - buf_size= +static int CORD_batched_iter_fn_append(const char* s, void* client_data) { + char*& cur=*static_cast(client_data); + + size_t size=strlen(s); + memcpy(cur, s, size); cur+=size; + return 0; +}; +String::C String::serialize(size_t prolog_size) const { + size_t buf_size= prolog_size - +fused*(sizeof(String_UL)+sizeof(size_t)) + +fragments.count()*(sizeof(Untaint_lang)+sizeof(size_t)) +size(); - buf=new(pool) char[buf_size]; - char *cur=buf+prolog_size; + C result(new(PointerFreeGC) char[buf_size], buf_size); + + // 1: prolog + char *cur=result.str+prolog_size; + - STRING_FOREACH_FRAGMENT( + // 2: fragments.count + size_t fragments_count=fragments.count(); + memcpy(cur, &fragments_count, sizeof(fragments_count)); cur+=sizeof(fragments_count); + + // 3: lang info + for(Array_iterator i(fragments); i.has_next(); ) { + const Fragment& fragment=i.next(); // lang - memcpy(cur, &fragment->lang, sizeof(fragment->lang)); - cur+=sizeof(fragment->lang); + memcpy(cur, &fragment.lang, sizeof(fragment.lang)); cur+=sizeof(fragment.lang); // size - // bug on some sparc platform [you can't work with integers on odd pointers] - // forces us to use byte array instead - uchar bytes[4]; - uint2uchars(fragment->size, bytes); - memcpy(cur, &bytes, sizeof(bytes)); cur+=sizeof(bytes); - // bytes - memcpy(cur, fragment->ptr, fragment->size); - cur+=fragment->size; - ); + memcpy(cur, &fragment.size, sizeof(fragment.size)); cur+=sizeof(fragment.size); + } + + // 4: letters + CORD_iter5(body, 0, 0, CORD_batched_iter_fn_append, &cur); + + return result; } bool String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char* file) { if(buf_size<=prolog_size) return false; - - char *cur=(char *)buf+prolog_size; buf_size-=prolog_size; - while(buf_size) { - if(sizeof(String_UL)+sizeof(size_t)>buf_size) // lang+size - return false; + // 1: prolog + const char* cur=(const char* )buf+prolog_size; - String_UL lang=*(String_UL *)(cur); - size_t size=uchars2uint((uchar *)cur); + // 2: fragments.count + if(buf_size(cur); cur+=sizeof(size_t); + buf_size-=sizeof(size_t); - size_t piece_size=sizeof(String_UL)+sizeof(size_t)+size; - if(piece_size>buf_size) // buffer overrun, can be on incomplete cache files + // 3: lang info + size_t total_size=0; + for(size_t f=0; f(cur); cur+=sizeof(Untaint_lang); + size_t fragment_size=*reinterpret_cast(cur); cur+=sizeof(size_t); + fragments+=Fragment(lang, fragment_size); + total_size+=fragment_size; - cur+=piece_size; buf_size-=piece_size; } + + // 4: letters + if(buf_size!=total_size) + return false; + + body=CORD_cat_char_star(CORD_EMPTY, cur, buf_size); + return true; }