--- parser3/src/main/pa_string.C 2002/02/20 12:35:29 1.142 +++ parser3/src/main/pa_string.C 2002/06/26 10:35:16 1.162 @@ -4,7 +4,7 @@ Copyright (c) 2001, 2002 ArtLebedev Group (http://www.artlebedev.com) Author: Alexandr Petrosian (http://paf.design.ru) - $Id: pa_string.C,v 1.142 2002/02/20 12:35:29 paf Exp $ + $Id: pa_string.C,v 1.162 2002/06/26 10:35:16 paf Exp $ */ #include "pcre.h" @@ -27,12 +27,22 @@ ulong string_piece_appends=0; #endif +String& String::OnPool(Pool& apool, const char *local_src, size_t src_size, bool tainted) { + if(local_src && *local_src) { + if(src_size==0) + src_size=strlen(local_src); + + char *pooled_src=(char *)apool.malloc(src_size); + memcpy(pooled_src, local_src, src_size); + return *new(apool) String(apool, pooled_src, src_size, tainted); + } else + return *new(apool) String(apool); +} String::String(Pool& apool, const char *src, size_t src_size, bool tainted) : Pooled(apool) { - last_chunk=&head; - head.count=CR_PREALLOCATED_COUNT; - append_here=head.rows; - initial_head_link=0; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; if(src) if(tainted) @@ -43,10 +53,9 @@ String::String(Pool& apool, const char * String::String(const String& src) : Pooled(src.pool()) { - last_chunk=&head; - head.count=CR_PREALLOCATED_COUNT; - append_here=head.rows; - initial_head_link=0; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; append(src, UL_UNSPECIFIED); } @@ -56,7 +65,6 @@ size_t String::size() const { STRING_FOREACH_ROW( result+=row->item.size; ); -break2: return result; } @@ -66,16 +74,19 @@ uint String::used_rows() const { STRING_FOREACH_ROW( result++; ); -break2: return result; } void String::expand() { - Chunk::count_type new_chunk_count=last_chunk->count+CR_GROW_COUNT; + uint new_chunk_count=last_chunk->count+CR_GROW_COUNT; if(new_chunk_count>max_integral(Chunk::count_type)) new_chunk_count=max_integral(Chunk::count_type); - Chunk *new_chunk=static_cast( - malloc(sizeof(Chunk::count_type)+sizeof(Chunk::Row)*new_chunk_count+sizeof(Chunk *), 10)); + Chunk *new_chunk=static_cast(malloc( + sizeof(Chunk)// count+interpadding(?)+rows[CR_PREALLOCATED_COUNT]+tailpadding(??) + -sizeof(Chunk::rows_type) // PREALLOCATED rows + +sizeof(Chunk::Row)*new_chunk_count // neaded rows + +sizeof(Chunk *) // link size + , 10)); new_chunk->rows[new_chunk->count=new_chunk_count].link=0; last_chunk->rows[last_chunk->count].link=new_chunk; @@ -85,7 +96,7 @@ void String::expand() { String& String::real_append(STRING_APPEND_PARAMS) { if(!last_chunk) // growth stopped [we're appended as string to somebody] - throw Exception(0, 0, + throw Exception(0, this, "string growth stopped (append cstr)"); @@ -135,11 +146,11 @@ String& String::real_append(STRING_APPEN char String::first_char() const { if(is_empty()) - throw Exception(0, 0, + throw Exception(0, this, "getting first char of empty string"); - return *head.rows[0].item.ptr; + return *head.chunk.rows[0].item.ptr; } uint String::hash_code() const { @@ -147,7 +158,6 @@ uint String::hash_code() const { STRING_FOREACH_ROW( result=Hash::generic_code(result, row->item.ptr, row->item.size); ); -break2: return result; } @@ -158,8 +168,8 @@ int String::cmp(int& partial, const Stri size_t a_size=size(); this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; - const Chunk *b_chunk=&src.head; + const Chunk *a_chunk=&head.chunk; + const Chunk *b_chunk=&src.head.chunk; const Chunk::Row *a_row=a_chunk->rows; const Chunk::Row *b_row=b_chunk->rows; size_t a_offset=this_offset; @@ -248,7 +258,7 @@ int String::cmp(int& partial, const char size_t b_size=src_size?src_size:b_ptr?strlen(b_ptr):0; this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; + const Chunk *a_chunk=&head.chunk; const Chunk::Row *a_row=a_chunk->rows; size_t a_offset=this_offset; size_t b_offset=0; @@ -319,14 +329,14 @@ const Origin& String::origin() const { return empty_origin; } - // determining origin by last appended piece - // because first one frequently constant. + // determining origin by first piece or last appended piece + // because any of them can be constant=without origin: // ex: ^load[/file] "document_root" + "/file" // when last peice is constant, // ex: parser_root_auto_path{dynamic} / auto.p{const} // using first piece - Origin& last_origin=append_here[-1].item.origin; - return last_origin.file ? last_origin : head.rows[0].item.origin; + Origin& first_origin=head.chunk.rows[0].item.origin; + return first_origin.file ? first_origin : append_here[-1].item.origin; } #endif @@ -442,33 +452,39 @@ void String::split(Array& result, } } -static void regex_options(char *options, int *result){ +static void regex_options(const String *options, int *result, bool& need_pre_post_match){ struct Regex_option { - char key; + const char *keyL; + const char *keyU; int clear, set; int *result; + bool *flag; } regex_option[]={ - {'i', 0, PCRE_CASELESS, result}, // a=A - {'s', 0, PCRE_DOTALL, result}, // \n\n$ [default] - {'x', 0, PCRE_EXTENDED, result}, // whitespace in regex ignored - {'m', PCRE_DOTALL, PCRE_MULTILINE, result}, // ^aaa\n$^bbb\n$ - {'g', 0, true, result+1}, // many rows - {0}, + {"i", "I", 0, PCRE_CASELESS, result}, // a=A + {"s", "S", 0, PCRE_DOTALL, result}, // \n\n$ [default] + {"x", "U", 0, PCRE_EXTENDED, result}, // whitespace in regex ignored + {"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result}, // ^aaa\n$^bbb\n$ + {"g", "G", 0, true, result+1}, // many rows + {"'", 0, 0, 0, 0, &need_pre_post_match}, + {0} }; result[0]=PCRE_EXTRA | PCRE_DOTALL; result[1]=0; if(options) - for(Regex_option *o=regex_option; o->key; o++) - if( - strchr(options, o->key) || - strchr(options, toupper(o->key))) { - *(o->result)&=~o->clear; - *(o->result)|=o->set; + for(Regex_option *o=regex_option; o->keyL; o++) + if(options->pos(o->keyL)>=0 + || (o->keyU && options->pos(o->keyU)>=0)) { + if(o->flag) + *o->flag=true; + else { // result + *o->result &= ~o->clear; + *o->result |= o->set; + } } } -/// @todo maybe need speedup: some option to remove pre/match/post string generation +/// @todo make replacement Table stacked bool String::match( const String *aorigin, const String& regexp, @@ -478,13 +494,15 @@ bool String::match( bool *was_global) const { if(regexp.is_empty()) - throw Exception(0, 0, + throw Exception(0, aorigin, "regexp is empty"); + const char *pattern=regexp.cstr(); const char *errptr; int erroffset; - int option_bits[2]; regex_options(options?options->cstr():0, option_bits); + bool need_pre_post_match=false; + int option_bits[2]; regex_options(options, option_bits, need_pre_post_match); if(was_global) *was_global=option_bits[1]!=0; pcre *code=pcre_compile(pattern, option_bits[0], @@ -492,77 +510,70 @@ bool String::match( pool().get_source_charset().pcre_tables); if(!code) - throw Exception(0, 0, + throw Exception(0, ®exp.mid(erroffset, regexp.size()), "regular expression syntax error - %s", errptr); int info_substrings=pcre_info(code, 0, 0); if(info_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "pcre_info error (%d)", info_substrings); } - int startoffset=0; const char *subject=cstr(); int length=strlen(subject); - int ovecsize; - int *ovector=(int *)malloc(sizeof(int)* - (ovecsize=(1/*match*/+info_substrings)*3), 11); - - { // create table - Array& columns=*NEW Array(pool()); - columns+=string_pre_match_name; - columns+=string_match_name; - columns+=string_post_match_name; - for(int i=1; i<=info_substrings; i++) { - char *column=(char *)malloc(MAX_NUMBER); - snprintf(column, MAX_NUMBER, "%d", i); - columns+=NEW String(pool(), column); // .i column name - } - *table=NEW Table(pool(), aorigin, &columns); - } + const int ovecsize=(1/*match*/+MAX_STRING_MATCH_TABLE_COLUMNS)*3; + int ovector[ovecsize]; + + // create table + *table=NEW Table(pool(), *string_match_table_template); int exec_option_bits=0; + int prestart=0; + int poststart=0; + int postfinish=size(); while(true) { int exec_substrings=pcre_exec(code, 0, - subject, length, startoffset, + subject, length, prestart, exec_option_bits, ovector, ovecsize); if(exec_substrings==PCRE_ERROR_NOMATCH) { pcre_free(code); - row_action(**table, 0/*last time, no row*/, 0, 0, info); + row_action(**table, 0/*last time, no row*/, 0, 0, poststart, postfinish, info); return option_bits[1]!=0; // global=true+table, not global=false } if(exec_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "regular expression execute error (%d)", exec_substrings); } + int prefinish=ovector[0]; + poststart=ovector[1]; Array& row=*NEW Array(pool()); - row+=&mid(0, ovector[0]); // .prematch column value - row+=&mid(ovector[0], ovector[1]); // .match - row+=&mid(ovector[1], size()); // .postmatch + row+=need_pre_post_match?&mid(0, prefinish):0; // .prematch column value + row+=need_pre_post_match?&mid(prefinish, poststart):0; // .match + row+=need_pre_post_match?&mid(poststart, postfinish):0; // .postmatch for(int i=1; i(kind)); // never @@ -596,144 +607,31 @@ String& String::change_case(Pool& pool, break; // never } - const Chunk *chunk=&head; - do { - const Chunk::Row *row=chunk->rows; - for(Chunk::count_type i=0; icount; i++, row++) { - if(row==append_here) - goto break2; - - char *new_cstr=(char *)pool.malloc(row->item.size, 12); - char *dest=new_cstr; - const char *src=row->item.ptr; - for(int size=row->item.size; size--; src++) { - unsigned char c=a[(unsigned char)*src]; - if(b) - c=b[c]; + STRING_FOREACH_ROW( + char *new_cstr=(char *)pool.malloc(row->item.size, 12); + char *dest=new_cstr; + const char *src=row->item.ptr; + for(int size=row->item.size; size--; src++) { + unsigned char c=a[(unsigned char)*src]; + if(b) + c=b[c]; - *dest++=(char)c; - } - - result.APPEND(new_cstr, row->item.size, - row->item.lang, - row->item.origin.file, row->item.origin.line); + *dest++=(char)c; } - chunk=row->link; - } while(chunk); -break2: + + result.APPEND(new_cstr, row->item.size, + row->item.lang, + row->item.origin.file, row->item.origin.line); + ); return result; } -void String::join_chain(Pool& pool, - uint& ai, const Chunk*& achunk, const Chunk::Row*& arow, - uchar& joined_lang, const char *& joined_ptr, size_t& joined_size) const { - joined_lang=arow->item.lang; - - // calc size - joined_size=0; - { - uint start_i=ai; - const Chunk::Row *start_row=arow; - const Chunk *chunk=achunk; - do { - const Chunk::Row *row=start_row; - for(uint i=start_i; icount; i++, row++) { - if(row==append_here) - goto break21; - - if(row->item.lang==joined_lang) - joined_size+=row->item.size; - else - goto break21; - } - if(chunk=row->link) { - start_i=0; - start_row=chunk->rows; - } else - break; - } while(true); -break21:; - } - - // if one row, return simply itself - if(joined_size==arow->item.size) { - joined_ptr=arow->item.ptr; - ai++; arow++; - if(ai==achunk->count) { - if(achunk=arow->link) { - ai=0; - arow=achunk->rows; - } - } - } else { - // join adjacent rows - char *ptr=(char *)pool.malloc(joined_size,13); - joined_ptr=ptr; - uint start_i=ai; - const Chunk::Row *start_row=arow; - const Chunk *chunk=achunk; - uint i; - const Chunk::Row *row; - do { - row=start_row; - for(i=start_i; icount; i++, row++) { - if(row==append_here) - goto break22; - - if(row->item.lang==joined_lang) { - memcpy(ptr, row->item.ptr, row->item.size); - ptr+=row->item.size; - } else - goto break22; - } - if(chunk=row->link) { - start_i=0; - start_row=chunk->rows; - } else - break; - } while(true); -break22:; - - // return joined rows - ai=i; - arow=row; - achunk=chunk; - } -} - -String& String::reconstruct(Pool& pool) const { - //_asm int 3; +/// @test if in some piece were found no dict words, append it, not it's duplicate +String& String::replace(Pool& pool, Dictionary& dict) const { +// return reconstruct(pool).replace_in_reconstructed(pool, dict); String& result=*new(pool) String(pool); - const Chunk *chunk=&head; - const Chunk::Row *row=chunk->rows; - for(uint i=0; icount; ) { - if(row==append_here) - break; - - uchar joined_lang; - const char *joined_ptr; - size_t joined_size; -#ifndef NO_STRING_ORIGIN - const char *joined_origin_file=row->item.origin.file; - const size_t joined_origin_line=row->item.origin.line; -#endif - join_chain(pool, i, chunk, row, - joined_lang, joined_ptr, joined_size); - - result.APPEND(joined_ptr, joined_size, joined_lang, - joined_origin_file, joined_origin_line); - if(!chunk) - break; - } - - return result; -}; - -String& String::replace_in_reconstructed(Pool& pool, Dictionary& dict) const { - //_asm int 3; - String& result=*new(pool) String(pool); STRING_FOREACH_ROW( const char *src=row->item.ptr; size_t src_size=row->item.size; @@ -755,29 +653,63 @@ String& String::replace_in_reconstructed } } - result.APPEND(new_cstr, dest-new_cstr, - row->item.lang, + result.APPEND(new_cstr, dest-new_cstr, row->item.lang, row->item.origin.file, row->item.origin.line); ); -break2: return result; } -String& String::replace(Pool& pool, Dictionary& dict) const { - return reconstruct(pool).replace_in_reconstructed(pool, dict); +String& String::join_chains(Pool& pool, char** acstr) const { + char *lcstr=cstr(); + const char *current=lcstr; + + String& result=*new(pool) String(pool); + STRING_FOREACH_ROW( +IFNDEF_NO_STRING_ORIGIN( + const char *joined_origin_file=row->item.origin.file; + const size_t joined_origin_line=row->item.origin.line; +); + uchar joined_lang=row->item.lang; + const char *joined_ptr=current; + // calc size + size_t joined_size=0; + STRING_PREPARED_FOREACH_ROW(*this, + if(row->item.lang==joined_lang) + joined_size+=row->item.size; + else + break; // before non-ours + ); + current+=joined_size; + + // pointers are after joined piece + // & one step back, see STRING_FOREACH_ROW + --row; ++countdown; + + result.APPEND(joined_ptr, joined_size, joined_lang, + joined_origin_file, joined_origin_line); + ); + + if(acstr) + *acstr=lcstr; + return result; } double String::as_double() const { double result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + int size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else cstr=this->cstr(); + while(*cstr && isspace(*cstr)) + cstr++; + if(!*cstr) + return 0; + char *error_pos; // 0xABC if(cstr[0]=='0') @@ -788,10 +720,11 @@ double String::as_double() const { else result=(double)strtod(cstr, &error_pos); - if(*error_pos/*not EOS*/) - throw Exception(0, 0, - this, - "invalid number (double)"); + while(char c=*error_pos++) + if(!isspace(c)) + throw Exception("number.format", + this, + "invalid number (double)"); return result; } @@ -799,13 +732,18 @@ int String::as_int() const { int result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + int size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else cstr=this->cstr(); + while(*cstr && isspace(*cstr)) + cstr++; + if(!*cstr) + return 0; + char *error_pos; // 0xABC if(cstr[0]=='0') @@ -816,10 +754,11 @@ int String::as_int() const { else result=(int)strtol(cstr, &error_pos, 0); - if(*error_pos/*not EOS*/) - throw Exception(0, 0, - this, - "invalid number (int)"); + while(char c=*error_pos++) + if(!isspace(c)) + throw Exception("number.format", + this, + "invalid number (int)"); return result; } @@ -857,29 +796,33 @@ void String::serialize(size_t prolog_siz memcpy(cur, row->item.ptr, row->item.size); cur+=row->item.size; ); -break2: - ; } -void String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { +bool String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { if(buf_size<=prolog_size) - return; + return false; char *cur=(char *)buf+prolog_size; buf_size-=prolog_size; while(buf_size) { - uchar lang=*(uchar *)(cur); - + if(sizeof(uchar)+sizeof(ushort)>buf_size) // lang+size + return false; + + uchar lang=*(uchar *)(cur); ushort size=uchars2ushort( *(uchar*)(cur+sizeof(uchar)*1), *(uchar*)(cur+sizeof(uchar)*2) ); + size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; + if(piece_size>buf_size) // buffer overrun, can be on incomplete cache files + return false; + const char *ptr=(const char*)(cur+sizeof(uchar)*3); APPEND(ptr, size, lang, file, 0); - size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; cur+=piece_size; buf_size-=piece_size; } + return true; }