--- parser3/src/main/pa_string.C 2002/02/22 12:20:27 1.145 +++ parser3/src/main/pa_string.C 2002/08/07 13:44:00 1.166 @@ -3,10 +3,10 @@ Copyright (c) 2001, 2002 ArtLebedev Group (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) - - $Id: pa_string.C,v 1.145 2002/02/22 12:20:27 paf Exp $ */ +static const char* IDENT_STRING_C="$Date: 2002/08/07 13:44:00 $"; + #include "pcre.h" #include "pa_pool.h" @@ -27,11 +27,22 @@ ulong string_piece_appends=0; #endif +String& String::OnPool(Pool& apool, const char *local_src, size_t src_size, bool tainted) { + if(local_src && *local_src) { + if(src_size==0) + src_size=strlen(local_src); + + char *pooled_src=(char *)apool.malloc(src_size); + memcpy(pooled_src, local_src, src_size); + return *new(apool) String(apool, pooled_src, src_size, tainted); + } else + return *new(apool) String(apool); +} String::String(Pool& apool, const char *src, size_t src_size, bool tainted) : Pooled(apool) { - last_chunk=&head; - head.count=CR_PREALLOCATED_COUNT; - append_here=head.rows; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; if(src) if(tainted) @@ -42,9 +53,9 @@ String::String(Pool& apool, const char * String::String(const String& src) : Pooled(src.pool()) { - last_chunk=&head; - head.count=CR_PREALLOCATED_COUNT; - append_here=head.rows; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; append(src, UL_UNSPECIFIED); } @@ -70,8 +81,12 @@ void String::expand() { if(new_chunk_count>max_integral(Chunk::count_type)) new_chunk_count=max_integral(Chunk::count_type); - Chunk *new_chunk=static_cast( - malloc(sizeof(Chunk::count_type)+sizeof(Chunk::Row)*new_chunk_count+sizeof(Chunk *), 10)); + Chunk *new_chunk=static_cast(malloc( + sizeof(Chunk)// count+interpadding(?)+rows[CR_PREALLOCATED_COUNT]+tailpadding(??) + -sizeof(Chunk::rows_type) // PREALLOCATED rows + +sizeof(Chunk::Row)*new_chunk_count // neaded rows + +sizeof(Chunk *) // link size + , 10)); new_chunk->rows[new_chunk->count=new_chunk_count].link=0; last_chunk->rows[last_chunk->count].link=new_chunk; @@ -81,7 +96,7 @@ void String::expand() { String& String::real_append(STRING_APPEND_PARAMS) { if(!last_chunk) // growth stopped [we're appended as string to somebody] - throw Exception(0, 0, + throw Exception(0, this, "string growth stopped (append cstr)"); @@ -131,11 +146,11 @@ String& String::real_append(STRING_APPEN char String::first_char() const { if(is_empty()) - throw Exception(0, 0, + throw Exception(0, this, "getting first char of empty string"); - return *head.rows[0].item.ptr; + return *head.chunk.rows[0].item.ptr; } uint String::hash_code() const { @@ -153,8 +168,8 @@ int String::cmp(int& partial, const Stri size_t a_size=size(); this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; - const Chunk *b_chunk=&src.head; + const Chunk *a_chunk=&head.chunk; + const Chunk *b_chunk=&src.head.chunk; const Chunk::Row *a_row=a_chunk->rows; const Chunk::Row *b_row=b_chunk->rows; size_t a_offset=this_offset; @@ -243,7 +258,7 @@ int String::cmp(int& partial, const char size_t b_size=src_size?src_size:b_ptr?strlen(b_ptr):0; this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; + const Chunk *a_chunk=&head.chunk; const Chunk::Row *a_row=a_chunk->rows; size_t a_offset=this_offset; size_t b_offset=0; @@ -314,22 +329,22 @@ const Origin& String::origin() const { return empty_origin; } - // determining origin by last appended piece - // because first one frequently constant. + // determining origin by first piece or last appended piece + // because any of them can be constant=without origin: // ex: ^load[/file] "document_root" + "/file" // when last peice is constant, // ex: parser_root_auto_path{dynamic} / auto.p{const} // using first piece - Origin& last_origin=append_here[-1].item.origin; - return last_origin.file ? last_origin : head.rows[0].item.origin; + Origin& first_origin=head.chunk.rows[0].item.origin; + return first_origin.file ? first_origin : append_here[-1].item.origin; } #endif String& String::mid(size_t start, size_t finish) const { String& result=*NEW String(pool()); - start=max(size_t(0), start); - finish=min(size(), finish); + start=min(start, size()); + finish=max(finish, start); if(start==finish) return result; @@ -437,33 +452,39 @@ void String::split(Array& result, } } -static void regex_options(char *options, int *result){ +static void regex_options(const String *options, int *result, bool& need_pre_post_match){ struct Regex_option { - char key; + const char *keyL; + const char *keyU; int clear, set; int *result; + bool *flag; } regex_option[]={ - {'i', 0, PCRE_CASELESS, result}, // a=A - {'s', 0, PCRE_DOTALL, result}, // \n\n$ [default] - {'x', 0, PCRE_EXTENDED, result}, // whitespace in regex ignored - {'m', PCRE_DOTALL, PCRE_MULTILINE, result}, // ^aaa\n$^bbb\n$ - {'g', 0, true, result+1}, // many rows - {0}, + {"i", "I", 0, PCRE_CASELESS, result}, // a=A + {"s", "S", 0, PCRE_DOTALL, result}, // \n\n$ [default] + {"x", "U", 0, PCRE_EXTENDED, result}, // whitespace in regex ignored + {"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result}, // ^aaa\n$^bbb\n$ + {"g", "G", 0, true, result+1}, // many rows + {"'", 0, 0, 0, 0, &need_pre_post_match}, + {0} }; result[0]=PCRE_EXTRA | PCRE_DOTALL; result[1]=0; if(options) - for(Regex_option *o=regex_option; o->key; o++) - if( - strchr(options, o->key) || - strchr(options, toupper(o->key))) { - *(o->result)&=~o->clear; - *(o->result)|=o->set; + for(Regex_option *o=regex_option; o->keyL; o++) + if(options->pos(o->keyL)>=0 + || (o->keyU && options->pos(o->keyU)>=0)) { + if(o->flag) + *o->flag=true; + else { // result + *o->result &= ~o->clear; + *o->result |= o->set; + } } } -/// @todo maybe need speedup: some option to remove pre/match/post string generation +/// @todo make replacement Table stacked bool String::match( const String *aorigin, const String& regexp, @@ -473,13 +494,15 @@ bool String::match( bool *was_global) const { if(regexp.is_empty()) - throw Exception(0, 0, + throw Exception(0, aorigin, "regexp is empty"); + const char *pattern=regexp.cstr(); const char *errptr; int erroffset; - int option_bits[2]; regex_options(options?options->cstr():0, option_bits); + bool need_pre_post_match=false; + int option_bits[2]; regex_options(options, option_bits, need_pre_post_match); if(was_global) *was_global=option_bits[1]!=0; pcre *code=pcre_compile(pattern, option_bits[0], @@ -487,77 +510,70 @@ bool String::match( pool().get_source_charset().pcre_tables); if(!code) - throw Exception(0, 0, + throw Exception(0, ®exp.mid(erroffset, regexp.size()), "regular expression syntax error - %s", errptr); int info_substrings=pcre_info(code, 0, 0); if(info_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "pcre_info error (%d)", info_substrings); } - int startoffset=0; const char *subject=cstr(); int length=strlen(subject); - int ovecsize; - int *ovector=(int *)malloc(sizeof(int)* - (ovecsize=(1/*match*/+info_substrings)*3), 11); - - { // create table - Array& columns=*NEW Array(pool()); - columns+=string_pre_match_name; - columns+=string_match_name; - columns+=string_post_match_name; - for(int i=1; i<=info_substrings; i++) { - char *column=(char *)malloc(MAX_NUMBER); - snprintf(column, MAX_NUMBER, "%d", i); - columns+=NEW String(pool(), column); // .i column name - } - *table=NEW Table(pool(), aorigin, &columns); - } + const int ovecsize=(1/*match*/+MAX_STRING_MATCH_TABLE_COLUMNS)*3; + int ovector[ovecsize]; + + // create table + *table=NEW Table(pool(), *string_match_table_template); int exec_option_bits=0; + int prestart=0; + int poststart=0; + int postfinish=size(); while(true) { int exec_substrings=pcre_exec(code, 0, - subject, length, startoffset, + subject, length, prestart, exec_option_bits, ovector, ovecsize); if(exec_substrings==PCRE_ERROR_NOMATCH) { pcre_free(code); - row_action(**table, 0/*last time, no row*/, 0, 0, info); + row_action(**table, 0/*last time, no row*/, 0, 0, poststart, postfinish, info); return option_bits[1]!=0; // global=true+table, not global=false } if(exec_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "regular expression execute error (%d)", exec_substrings); } + int prefinish=ovector[0]; + poststart=ovector[1]; Array& row=*NEW Array(pool()); - row+=&mid(0, ovector[0]); // .prematch column value - row+=&mid(ovector[0], ovector[1]); // .match - row+=&mid(ovector[1], size()); // .postmatch + row+=need_pre_post_match?&mid(0, prefinish):0; // .prematch column value + row+=need_pre_post_match?&mid(prefinish, poststart):0; // .match + row+=need_pre_post_match?&mid(poststart, postfinish):0; // .postmatch for(int i=1; i(kind)); // never @@ -611,76 +627,11 @@ String& String::change_case(Pool& pool, return result; } -void String::join_chain(Pool& pool, - const Chunk*& achunk, const Chunk::Row*& arow, uint& acountdown, - uchar& joined_lang, const char *& joined_ptr, size_t& joined_size) const { - joined_lang=arow->item.lang; - - // calc size - joined_size=0; - { - const Chunk* chunk=achunk; - const Chunk::Row* row=arow; - uint countdown=acountdown; - STRING_PREPARED_FOREACH_ROW(*this, - if(row->item.lang==joined_lang) - joined_size+=row->item.size; - else - break; - ); - } - - // if one row, return simply itself - if(joined_size==arow->item.size) { - joined_ptr=arow->item.ptr; - } else { - // join adjacent rows - char *ptr=(char *)pool.malloc(joined_size,13); - joined_ptr=ptr; - - const Chunk* chunk=achunk; - const Chunk::Row* row=arow; - uint countdown=acountdown; - STRING_PREPARED_FOREACH_ROW(*this, - if(row->item.lang==joined_lang) { - memcpy(ptr, row->item.ptr, row->item.size); ptr+=row->item.size; - } else { - --row; ++countdown; // step back from not-ours - break; - } - ); - - // set pointers after joined piece - achunk=chunk; arow=row; acountdown=countdown; - // & one step back, see String::reconstruct - --row; ++countdown; - } -} - -String& String::reconstruct(Pool& pool) const { - //_asm int 3; +/// @test if in some piece were found no dict words, append it, not it's duplicate +String& String::replace(Pool& pool, Dictionary& dict) const { +// return reconstruct(pool).replace_in_reconstructed(pool, dict); String& result=*new(pool) String(pool); - STRING_FOREACH_ROW( - uchar joined_lang; - const char *joined_ptr; - size_t joined_size; -IFNDEF_NO_STRING_ORIGIN( - const char *joined_origin_file=row->item.origin.file; - const size_t joined_origin_line=row->item.origin.line; -); - join_chain(pool, chunk, row, countdown, - joined_lang, joined_ptr, joined_size); - result.APPEND(joined_ptr, joined_size, joined_lang, - joined_origin_file, joined_origin_line); - ); - - return result; -}; - -String& String::replace_in_reconstructed(Pool& pool, Dictionary& dict) const { - //_asm int 3; - String& result=*new(pool) String(pool); STRING_FOREACH_ROW( const char *src=row->item.ptr; size_t src_size=row->item.size; @@ -702,28 +653,63 @@ String& String::replace_in_reconstructed } } - result.APPEND(new_cstr, dest-new_cstr, - row->item.lang, + result.APPEND(new_cstr, dest-new_cstr, row->item.lang, row->item.origin.file, row->item.origin.line); ); return result; } -String& String::replace(Pool& pool, Dictionary& dict) const { - return reconstruct(pool).replace_in_reconstructed(pool, dict); +String& String::join_chains(Pool& pool, char** acstr) const { + char *lcstr=cstr(); + const char *current=lcstr; + + String& result=*new(pool) String(pool); + STRING_FOREACH_ROW( +IFNDEF_NO_STRING_ORIGIN( + const char *joined_origin_file=row->item.origin.file; + const size_t joined_origin_line=row->item.origin.line; +); + uchar joined_lang=row->item.lang; + const char *joined_ptr=current; + // calc size + size_t joined_size=0; + STRING_PREPARED_FOREACH_ROW(*this, + if(row->item.lang==joined_lang) + joined_size+=row->item.size; + else + break; // before non-ours + ); + current+=joined_size; + + // pointers are after joined piece + // & one step back, see STRING_FOREACH_ROW + --row; ++countdown; + + result.APPEND(joined_ptr, joined_size, joined_lang, + joined_origin_file, joined_origin_line); + ); + + if(acstr) + *acstr=lcstr; + return result; } double String::as_double() const { double result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + int size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else cstr=this->cstr(); + while(*cstr && isspace(*cstr)) + cstr++; + if(!*cstr) + return 0; + char *error_pos; // 0xABC if(cstr[0]=='0') @@ -734,10 +720,11 @@ double String::as_double() const { else result=(double)strtod(cstr, &error_pos); - if(*error_pos/*not EOS*/) - throw Exception(0, 0, - this, - "invalid number (double)"); + while(char c=*error_pos++) + if(!isspace(c)) + throw Exception("number.format", + this, + "invalid number (double)"); return result; } @@ -745,13 +732,18 @@ int String::as_int() const { int result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + size_t size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else cstr=this->cstr(); + while(*cstr && isspace(*cstr)) + cstr++; + if(!*cstr) + return 0; + char *error_pos; // 0xABC if(cstr[0]=='0') @@ -762,10 +754,11 @@ int String::as_int() const { else result=(int)strtol(cstr, &error_pos, 0); - if(*error_pos/*not EOS*/) - throw Exception(0, 0, - this, - "invalid number (int)"); + while(char c=*error_pos++) + if(!isspace(c)) + throw Exception("number.format", + this, + "invalid number (int)"); return result; } @@ -804,26 +797,32 @@ void String::serialize(size_t prolog_siz cur+=row->item.size; ); } -void String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { +bool String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { if(buf_size<=prolog_size) - return; + return false; char *cur=(char *)buf+prolog_size; buf_size-=prolog_size; while(buf_size) { - uchar lang=*(uchar *)(cur); - + if(sizeof(uchar)+sizeof(ushort)>buf_size) // lang+size + return false; + + uchar lang=*(uchar *)(cur); ushort size=uchars2ushort( *(uchar*)(cur+sizeof(uchar)*1), *(uchar*)(cur+sizeof(uchar)*2) ); + size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; + if(piece_size>buf_size) // buffer overrun, can be on incomplete cache files + return false; + const char *ptr=(const char*)(cur+sizeof(uchar)*3); APPEND(ptr, size, lang, file, 0); - size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; cur+=piece_size; buf_size-=piece_size; } + return true; }