--- parser3/src/main/pa_string.C 2001/12/25 09:00:28 1.136 +++ parser3/src/main/pa_string.C 2002/04/10 08:53:55 1.151 @@ -1,10 +1,10 @@ /** @file Parser: string class. @see untasize_t.C. - Copyright (c) 2001 ArtLebedev Group (http://www.artlebedev.com) - Author: Alexander Petrosyan (http://paf.design.ru) + Copyright (c) 2001, 2002 ArtLebedev Group (http://www.artlebedev.com) + Author: Alexandr Petrosian (http://paf.design.ru) - $Id: pa_string.C,v 1.136 2001/12/25 09:00:28 paf Exp $ + $Id: pa_string.C,v 1.151 2002/04/10 08:53:55 paf Exp $ */ #include "pcre.h" @@ -20,13 +20,18 @@ #include "pa_dictionary.h" #include "pa_charset.h" +#define DEBUG_STRING_APPENDS_VS_EXPANDS + + +#ifdef DEBUG_STRING_APPENDS_VS_EXPANDS +ulong string_piece_appends=0; +#endif + String::String(Pool& apool, const char *src, size_t src_size, bool tainted) : Pooled(apool) { - last_chunk=&head; - head.count=CR_PREALLOCATED_COUNT; - append_here=head.rows; - head_link=0; - link_row=&head.rows[head.count]; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; if(src) if(tainted) @@ -37,60 +42,11 @@ String::String(Pool& apool, const char * String::String(const String& src) : Pooled(src.pool()) { - head.count=CR_PREALLOCATED_COUNT; - - uint src_used_rows=src.used_rows(); - if(src_used_rows<=head.count) { - // all new rows fit size_to preallocated area - last_chunk=&head; - uint curr_chunk_rows=head.count; - memcpy(head.rows, src.head.rows, sizeof(Chunk::Row)*src_used_rows); - append_here=&head.rows[src_used_rows]; - link_row=&head.rows[curr_chunk_rows]; - } else { - // warning: - // heavily relies on the fact - // "preallocated area is the same for all strings" - // - // info: - // allocating only enough mem to fit src string rows - // next append would allocate a new chunk - // - // new rows don't fit size_to preallocated area: splitting size_to two chunks - // preallocated chunk src to constructing head - memcpy(head.rows, src.head.rows, sizeof(Chunk::Row)*head.count); - // remaining rows size_to new_chunk - uint curr_chunk_rows=src_used_rows-head.count; - last_chunk=static_cast( - malloc(sizeof(uint)+sizeof(Chunk::Row)*curr_chunk_rows+sizeof(Chunk *), 9)); - last_chunk->count=curr_chunk_rows; - head_link=last_chunk; - append_here=link_row=&last_chunk->rows[last_chunk->count]; - - Chunk *old_chunk=src.head_link; - Chunk::Row *new_rows=last_chunk->rows; - uint rows_left_to_copy=last_chunk->count; - while(true) { - uint old_count=old_chunk->count; - Chunk *next_chunk=old_chunk->rows[old_count].link; - if(next_chunk) { - // not last source chunk - // taking it all - memcpy(new_rows, old_chunk->rows, sizeof(Chunk::Row)*old_count); - new_rows+=old_count; - rows_left_to_copy-=old_count; + last_chunk=&head.chunk; + head.chunk.count=CR_PREALLOCATED_COUNT; + append_here=head.chunk.rows; - old_chunk=next_chunk; - } else { - // the last source chunk - // taking only those rows of chunk that _left_to_copy - memcpy(new_rows, old_chunk->rows, sizeof(Chunk::Row)*rows_left_to_copy); - break; - } - } - } - link_row->link=0; - src_used_rows; + append(src, UL_UNSPECIFIED); } size_t String::size() const { @@ -98,7 +54,6 @@ size_t String::size() const { STRING_FOREACH_ROW( result+=row->item.size; ); -break2: return result; } @@ -108,24 +63,33 @@ uint String::used_rows() const { STRING_FOREACH_ROW( result++; ); -break2: return result; } void String::expand() { uint new_chunk_count=last_chunk->count+CR_GROW_COUNT; - if(new_chunk_count>MAX_USHORT) - new_chunk_count=MAX_USHORT; + if(new_chunk_count>max_integral(Chunk::count_type)) + new_chunk_count=max_integral(Chunk::count_type); - last_chunk=static_cast( - malloc(sizeof(uint)+sizeof(Chunk::Row)*new_chunk_count+sizeof(Chunk *), 10)); - last_chunk->count=new_chunk_count; - link_row->link=last_chunk; + Chunk *new_chunk=static_cast( + malloc( + sizeof(Chunk)// count+interpadding(?)+rows[CR_PREALLOCATED_COUNT]+tailpadding(??) + -sizeof(Chunk::rows_type) // PREALLOCATED rows + +sizeof(Chunk::Row)*new_chunk_count // neaded rows + +sizeof(Chunk *) // link size + , 10)); + new_chunk->rows[new_chunk->count=new_chunk_count].link=0; + last_chunk->rows[last_chunk->count].link=new_chunk; + + last_chunk=new_chunk; append_here=last_chunk->rows; - link_row=&last_chunk->rows[last_chunk->count]; - link_row->link=0; } String& String::real_append(STRING_APPEND_PARAMS) { + if(!last_chunk) // growth stopped [we're appended as string to somebody] + throw Exception(0, + this, + "string growth stopped (append cstr)"); + if(!src) return *this; if(!size) @@ -133,12 +97,17 @@ String& String::real_append(STRING_APPEN if(!size) return *this; - while(size>MAX_USHORT) { +#ifdef DEBUG_STRING_APPENDS_VS_EXPANDS + string_piece_appends++; +#endif + + // manually unrolled to avoid extra check + while(size>max_integral(Chunk::Row::item_size_type)) { if(chunk_is_full()) expand(); append_here->item.ptr=src; - append_here->item.size=MAX_USHORT; + append_here->item.size=max_integral(Chunk::Row::item_size_type); append_here->item.lang=lang; #ifndef NO_STRING_ORIGIN append_here->item.origin.file=file; @@ -146,8 +115,8 @@ String& String::real_append(STRING_APPEN #endif append_here++; - src+=MAX_USHORT; - size-=MAX_USHORT; + src+=max_integral(Chunk::Row::item_size_type); + size-=max_integral(Chunk::Row::item_size_type); } if(chunk_is_full()) @@ -166,12 +135,12 @@ String& String::real_append(STRING_APPEN } char String::first_char() const { - if(!used_rows()) - throw Exception(0, 0, + if(is_empty()) + throw Exception(0, this, "getting first char of empty string"); - return *head.rows[0].item.ptr; + return *head.chunk.rows[0].item.ptr; } uint String::hash_code() const { @@ -179,7 +148,6 @@ uint String::hash_code() const { STRING_FOREACH_ROW( result=Hash::generic_code(result, row->item.ptr, row->item.size); ); -break2: return result; } @@ -190,8 +158,8 @@ int String::cmp(int& partial, const Stri size_t a_size=size(); this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; - const Chunk *b_chunk=&src.head; + const Chunk *a_chunk=&head.chunk; + const Chunk *b_chunk=&src.head.chunk; const Chunk::Row *a_row=a_chunk->rows; const Chunk::Row *b_row=b_chunk->rows; size_t a_offset=this_offset; @@ -204,7 +172,7 @@ int String::cmp(int& partial, const Stri size_t pos=0; bool a_break=a_size==0; - bool b_break=src.size()==0; + bool b_break=src.is_empty(); if(!(a_break || b_break)) while(true) { if(pos+a_row->item.size > this_offset) { if(lang!=UL_UNSPECIFIED && a_row->item.lang>lang) @@ -280,7 +248,7 @@ int String::cmp(int& partial, const char size_t b_size=src_size?src_size:b_ptr?strlen(b_ptr):0; this_offset=min(this_offset, a_size-1); - const Chunk *a_chunk=&head; + const Chunk *a_chunk=&head.chunk; const Chunk::Row *a_row=a_chunk->rows; size_t a_offset=this_offset; size_t b_offset=0; @@ -346,26 +314,26 @@ int String::cmp(int& partial, const char #ifndef NO_STRING_ORIGIN const Origin& String::origin() const { - if(!used_rows()) { + if(is_empty()) { static const Origin empty_origin={"empty string"}; return empty_origin; } - // determining origin by last appended piece - // because first one frequently constant. + // determining origin by first piece or last appended piece + // because any of them can be constant=without origin: // ex: ^load[/file] "document_root" + "/file" // when last peice is constant, // ex: parser_root_auto_path{dynamic} / auto.p{const} // using first piece - Origin& last_origin=append_here[-1].item.origin; - return last_origin.file ? last_origin : head.rows[0].item.origin; + Origin& first_origin=head.chunk.rows[0].item.origin; + return first_origin.file ? first_origin : append_here[-1].item.origin; } #endif String& String::mid(size_t start, size_t finish) const { String& result=*NEW String(pool()); - start=max(0, start); + start=max(size_t(0), start); finish=min(size(), finish); if(start==finish) return result; @@ -374,7 +342,7 @@ String& String::mid(size_t start, size_t STRING_FOREACH_ROW( size_t item_finish=pos+row->item.size; if(item_finish > start) { // started now or already? - bool started=result.size()==0; // started now? + bool started=result.is_empty(); // started now? bool finished=finish <= item_finish; // finished now? size_t offset=started?start-pos:0; size_t size=finished?finish-pos:row->item.size; @@ -452,7 +420,7 @@ void String::split(Array& result, size_t* pos_after_ref, const String& delim, Untaint_lang lang, int limit) const { - if(delim.size()) { + if(!delim.is_empty()) { size_t pos_after=pos_after_ref?*pos_after_ref:0; int pos_before; // while we have 'delim'... @@ -509,8 +477,8 @@ bool String::match( Row_action row_action, void *info, bool *was_global) const { - if(!regexp.size()) - throw Exception(0, 0, + if(regexp.is_empty()) + throw Exception(0, aorigin, "regexp is empty"); const char *pattern=regexp.cstr(); @@ -524,14 +492,14 @@ bool String::match( pool().get_source_charset().pcre_tables); if(!code) - throw Exception(0, 0, + throw Exception(0, ®exp.mid(erroffset, regexp.size()), "regular expression syntax error - %s", errptr); int info_substrings=pcre_info(code, 0, 0); if(info_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "pcre_info error (%d)", info_substrings); @@ -571,7 +539,7 @@ bool String::match( if(exec_substrings<0) { pcre_free(code); - throw Exception(0, 0, + throw Exception(0, aorigin, "regular expression execute error (%d)", exec_substrings); @@ -620,7 +588,7 @@ String& String::change_case(Pool& pool, b=0; break; default: - throw Exception(0, 0, + throw Exception(0, this, "unknown change case kind #%d", static_cast(kind)); // never @@ -628,147 +596,88 @@ String& String::change_case(Pool& pool, break; // never } - const Chunk *chunk=&head; - do { - const Chunk::Row *row=chunk->rows; - for(uint i=0; icount; i++, row++) { - if(row==append_here) - goto break2; - - char *new_cstr=(char *)pool.malloc(row->item.size, 12); - char *dest=new_cstr; - const char *src=row->item.ptr; - for(int size=row->item.size; size--; src++) { - unsigned char c=a[(unsigned char)*src]; - if(b) - c=b[c]; + STRING_FOREACH_ROW( + char *new_cstr=(char *)pool.malloc(row->item.size, 12); + char *dest=new_cstr; + const char *src=row->item.ptr; + for(int size=row->item.size; size--; src++) { + unsigned char c=a[(unsigned char)*src]; + if(b) + c=b[c]; - *dest++=(char)c; - } - - result.APPEND(new_cstr, row->item.size, - row->item.lang, - row->item.origin.file, row->item.origin.line); + *dest++=(char)c; } - chunk=row->link; - } while(chunk); -break2: + + result.APPEND(new_cstr, row->item.size, + row->item.lang, + row->item.origin.file, row->item.origin.line); + ); return result; } void String::join_chain(Pool& pool, - uint& ai, const Chunk*& achunk, const Chunk::Row*& arow, + const Chunk*& achunk, const Chunk::Row*& arow, uint& acountdown, uchar& joined_lang, const char *& joined_ptr, size_t& joined_size) const { joined_lang=arow->item.lang; // calc size joined_size=0; { - uint start_i=ai; - const Chunk::Row *start_row=arow; - const Chunk *chunk=achunk; - do { - const Chunk::Row *row=start_row; - for(uint i=start_i; icount; i++, row++) { - if(row==append_here) - goto break21; - - if(row->item.lang==joined_lang) - joined_size+=row->item.size; - else - goto break21; - } - if(chunk=row->link) { - start_i=0; - start_row=chunk->rows; - } else + const Chunk* chunk=achunk; + const Chunk::Row* row=arow; + uint countdown=acountdown; + STRING_PREPARED_FOREACH_ROW(*this, + if(row->item.lang==joined_lang) + joined_size+=row->item.size; + else break; - } while(true); -break21:; + ); } // if one row, return simply itself if(joined_size==arow->item.size) { joined_ptr=arow->item.ptr; - ai++; arow++; - if(ai==achunk->count) { - if(achunk=arow->link) { - ai=0; - arow=achunk->rows; - } - } } else { // join adjacent rows - char *ptr=(char *)pool.malloc(joined_size,13); + char *ptr=(char *)pool.malloc(joined_size,13); joined_ptr=ptr; - uint start_i=ai; - const Chunk::Row *start_row=arow; - const Chunk *chunk=achunk; - uint i; - const Chunk::Row *row; - do { - row=start_row; - for(i=start_i; icount; i++, row++) { - if(row==append_here) - goto break22; - - if(row->item.lang==joined_lang) { - memcpy(ptr, row->item.ptr, row->item.size); - ptr+=row->item.size; - } else - goto break22; - } - if(chunk=row->link) { - start_i=0; - start_row=chunk->rows; + + const Chunk* chunk=achunk; + const Chunk::Row* row=arow; + uint countdown=acountdown; + STRING_PREPARED_FOREACH_ROW(*this, + if(row->item.lang==joined_lang) { + memcpy(ptr, row->item.ptr, row->item.size); ptr+=row->item.size; } else - break; - } while(true); -break22:; + break; // before non-ours + ); - // return joined rows - ai=i; - arow=row; - achunk=chunk; + // set pointers after joined piece + achunk=chunk; arow=row; acountdown=countdown; + // & one step back, see String::reconstruct + --arow; ++acountdown; } } -String& String::reconstruct(Pool& pool) const { - //_asm int 3; +/// @test if in some piece were found no dict words, append it, not it's duplicate +String& String::replace(Pool& pool, Dictionary& dict) const { +// return reconstruct(pool).replace_in_reconstructed(pool, dict); String& result=*new(pool) String(pool); - const Chunk *chunk=&head; - const Chunk::Row *row=chunk->rows; - for(uint i=0; icount; ) { - if(row==append_here) - break; + STRING_FOREACH_ROW( uchar joined_lang; const char *joined_ptr; size_t joined_size; -#ifndef NO_STRING_ORIGIN +IFNDEF_NO_STRING_ORIGIN( const char *joined_origin_file=row->item.origin.file; const size_t joined_origin_line=row->item.origin.line; -#endif - join_chain(pool, i, chunk, row, +); + join_chain(pool, chunk, row, countdown, joined_lang, joined_ptr, joined_size); - - result.APPEND(joined_ptr, joined_size, joined_lang, - joined_origin_file, joined_origin_line); - - if(!chunk) - break; - } - - return result; -}; - -String& String::replace_in_reconstructed(Pool& pool, Dictionary& dict) const { - //_asm int 3; - String& result=*new(pool) String(pool); - STRING_FOREACH_ROW( - const char *src=row->item.ptr; - size_t src_size=row->item.size; + + const char *src=joined_ptr; + size_t src_size=joined_size; char *new_cstr=(char *)pool.malloc((size_t)ceil(src_size*dict.max_ratio()), 14); char *dest=new_cstr; while(src_size) { @@ -787,25 +696,19 @@ String& String::replace_in_reconstructed } } - result.APPEND(new_cstr, dest-new_cstr, - row->item.lang, - row->item.origin.file, row->item.origin.line); + result.APPEND(new_cstr, dest-new_cstr, joined_lang, + joined_origin_file, joined_origin_line); ); -break2: return result; } -String& String::replace(Pool& pool, Dictionary& dict) const { - return reconstruct(pool).replace_in_reconstructed(pool, dict); -} - double String::as_double() const { double result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + int size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else @@ -821,7 +724,7 @@ double String::as_double() const { result=(double)strtod(cstr, &error_pos); if(*error_pos/*not EOS*/) - throw Exception(0, 0, + throw Exception("number.format", this, "invalid number (double)"); @@ -831,9 +734,9 @@ int String::as_int() const { int result; const char *cstr; char buf[MAX_NUMBER]; - if(head.rows+1==append_here) { - int size=min(head.rows[0].item.size, MAX_NUMBER-1); - memcpy(buf, head.rows[0].item.ptr, size); + if(head.chunk.rows+1==append_here) { + int size=min(head.chunk.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.chunk.rows[0].item.ptr, size); buf[size]=0; cstr=buf; } else @@ -849,7 +752,7 @@ int String::as_int() const { result=(int)strtol(cstr, &error_pos, 0); if(*error_pos/*not EOS*/) - throw Exception(0, 0, + throw Exception("number.format", this, "invalid number (int)"); @@ -889,29 +792,33 @@ void String::serialize(size_t prolog_siz memcpy(cur, row->item.ptr, row->item.size); cur+=row->item.size; ); -break2: - ; } -void String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { +bool String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { if(buf_size<=prolog_size) - return; + return false; char *cur=(char *)buf+prolog_size; buf_size-=prolog_size; while(buf_size) { - uchar lang=*(uchar *)(cur); - + if(sizeof(uchar)+sizeof(ushort)>buf_size) // lang+size + return false; + + uchar lang=*(uchar *)(cur); ushort size=uchars2ushort( *(uchar*)(cur+sizeof(uchar)*1), *(uchar*)(cur+sizeof(uchar)*2) ); + size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; + if(piece_size>buf_size) // buffer overrun, can be on incomplete cache files + return false; + const char *ptr=(const char*)(cur+sizeof(uchar)*3); APPEND(ptr, size, lang, file, 0); - size_t piece_size=sizeof(uchar)+sizeof(ushort)+size; cur+=piece_size; buf_size-=piece_size; } + return true; }