--- parser3/src/main/pa_string.C 2001/08/01 12:08:40 1.97 +++ parser3/src/main/pa_string.C 2001/11/16 13:58:28 1.121 @@ -2,10 +2,10 @@ Parser: string class. @see untasize_t.C. Copyright (c) 2001 ArtLebedev Group (http://www.artlebedev.com) + Author: Alexander Petrosyan (http://paf.design.ru) - Author: Alexander Petrosyan (http://design.ru/paf) + $Id: pa_string.C,v 1.121 2001/11/16 13:58:28 paf Exp $ */ -static const char *RCSId="$Id: pa_string.C,v 1.97 2001/08/01 12:08:40 parser Exp $"; #include "pa_config_includes.h" @@ -20,16 +20,15 @@ static const char *RCSId="$Id: pa_string #include "pa_array.h" #include "pa_globals.h" #include "pa_table.h" +#include "pa_dictionary.h" String::String(Pool& apool, const char *src, size_t src_size, bool tainted) : - Pooled(apool), - forigins_mode(false) { + Pooled(apool) { last_chunk=&head; head.count=CR_PREALLOCATED_COUNT; append_here=head.rows; head.preallocated_link=0; link_row=&head.rows[head.count]; - fused_rows=fsize=0; if(src) if(tainted) @@ -39,14 +38,14 @@ String::String(Pool& apool, const char * } String::String(const String& src) : - Pooled(src.pool()), - forigins_mode(false) { + Pooled(src.pool()) { head.count=CR_PREALLOCATED_COUNT; - size_t src_used_rows=src.fused_rows; + uint src_used_rows=src.used_rows(); if(src_used_rows<=head.count) { // all new rows fit size_to preallocated area - size_t curr_chunk_rows=head.count; + last_chunk=&head; + uint curr_chunk_rows=head.count; memcpy(head.rows, src.head.rows, sizeof(Chunk::Row)*src_used_rows); append_here=&head.rows[src_used_rows]; link_row=&head.rows[curr_chunk_rows]; @@ -63,18 +62,18 @@ String::String(const String& src) : // preallocated chunk src to constructing head memcpy(head.rows, src.head.rows, sizeof(Chunk::Row)*head.count); // remaining rows size_to new_chunk - size_t curr_chunk_rows=src_used_rows-head.count; - Chunk *new_chunk=static_cast( - malloc(sizeof(size_t)+sizeof(Chunk::Row)*curr_chunk_rows+sizeof(Chunk *))); - new_chunk->count=curr_chunk_rows; - head.preallocated_link=new_chunk; - append_here=link_row=&new_chunk->rows[new_chunk->count]; + uint curr_chunk_rows=src_used_rows-head.count; + last_chunk=static_cast( + malloc(sizeof(uint)+sizeof(Chunk::Row)*curr_chunk_rows+sizeof(Chunk *), 9)); + last_chunk->count=curr_chunk_rows; + head.preallocated_link=last_chunk; + append_here=link_row=&last_chunk->rows[last_chunk->count]; Chunk *old_chunk=src.head.preallocated_link; - Chunk::Row *new_rows=new_chunk->rows; - size_t rows_left_to_copy=new_chunk->count; + Chunk::Row *new_rows=last_chunk->rows; + uint rows_left_to_copy=last_chunk->count; while(true) { - size_t old_count=old_chunk->count; + uint old_count=old_chunk->count; Chunk *next_chunk=old_chunk->rows[old_count].link; if(next_chunk) { // not last source chunk @@ -93,14 +92,49 @@ String::String(const String& src) : } } link_row->link=0; - fused_rows=src_used_rows; - fsize=src.fsize; + src_used_rows; } +size_t String::size() const { + size_t result=0; + const Chunk *chunk=&head; + do { + const Chunk::Row *row=chunk->rows; + for(uint i=0; icount; i++, row++) { + if(row==append_here) + goto break2; + + result+=row->item.size; + } + chunk=row->link; + } while(chunk); + +break2: + return result; +} + +/// @todo not very optimal +uint String::used_rows() const { + uint result=0; + const Chunk *chunk=&head; + do { + const Chunk::Row *row=chunk->rows; + for(uint i=0; icount; i++, row++) { + if(row==append_here) + goto break2; + + result++; + } + chunk=row->link; + } while(chunk); + +break2: + return result; +} void String::expand() { - size_t new_chunk_count=last_chunk->count+CR_GROW_COUNT; + uint new_chunk_count=last_chunk->count+CR_GROW_COUNT; last_chunk=static_cast( - malloc(sizeof(size_t)+sizeof(Chunk::Row)*new_chunk_count+sizeof(Chunk *))); + malloc(sizeof(uint)+sizeof(Chunk::Row)*new_chunk_count+sizeof(Chunk *), 10)); last_chunk->count=new_chunk_count; link_row->link=last_chunk; append_here=last_chunk->rows; @@ -112,12 +146,12 @@ String& String::append(const String& src const Chunk *chunk=&src.head; do { const Chunk::Row *row=chunk->rows; - for(size_t i=0; icount; i++, row++) { + for(uint i=0; icount; i++, row++) { if(row==src.append_here) goto break2; APPEND(row->item.ptr, row->item.size, - (lang!=UL_PASS_APPENDED && (row->item.lang==UL_TAINTED || forced))?lang:row->item.lang, + (lang!=UL_PASS_APPENDED && (row->item.lang==UL_TAINTED || forced))?lang:(Untaint_lang)row->item.lang, row->item.origin.file, row->item.origin.line); } chunk=row->link; @@ -138,20 +172,20 @@ String& String::real_append(STRING_APPEN expand(); append_here->item.ptr=src; - fsize+=append_here->item.size=size; + append_here->item.size=size; append_here->item.lang=lang; #ifndef NO_STRING_ORIGIN append_here->item.origin.file=file; append_here->item.origin.line=line; #endif - append_here++; fused_rows++; + append_here++; return *this; } char String::first_char() const { - if(!fused_rows) - THROW(0, 0, + if(!used_rows()) + throw Exception(0, 0, this, "getting first char of empty string"); @@ -164,7 +198,7 @@ uint String::hash_code() const { const Chunk *chunk=&head; do { const Chunk::Row *row=chunk->rows; - for(size_t i=0; icount; i++) { + for(uint i=0; icount; i++) { if(row==append_here) goto break2; @@ -191,9 +225,9 @@ int String::cmp(int& partial, const Stri size_t b_offset=0; Chunk::Row *a_end=append_here; Chunk::Row *b_end=src.append_here; - size_t a_countdown=a_chunk->count; - size_t b_countdown=b_chunk->count; - size_t result; + uint a_countdown=a_chunk->count; + uint b_countdown=b_chunk->count; + int result; size_t pos=0; bool a_break=size()==0; @@ -277,7 +311,7 @@ int String::cmp(int& partial, const char size_t a_offset=this_offset; size_t b_offset=0; Chunk::Row *a_end=append_here; - size_t a_countdown=a_chunk->count; + uint a_countdown=a_chunk->count; size_t pos=0; bool a_break=size()==0; @@ -292,20 +326,20 @@ int String::cmp(int& partial, const char (b_size-b_offset); if(size_diff==0) { // a has same size as b - if(size_t result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, + if(int result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, a_row->item.size-a_offset)!=0) return result; pos+=a_row->item.size; a_row++; a_countdown--; a_offset=0; b_break=true; } else if (size_diff>0) { // a longer - if(size_t result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, + if(int result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, b_size-b_offset)!=0) return result; a_offset+=b_size-b_offset; b_break=true; } else { // b longer - if(size_t result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, + if(int result=memcmp(a_row->item.ptr+a_offset, b_ptr+b_offset, a_row->item.size-a_offset)!=0) return result; b_offset+=a_row->item.size-a_offset; @@ -338,7 +372,7 @@ int String::cmp(int& partial, const char #ifndef NO_STRING_ORIGIN const Origin& String::origin() const { - if(!fused_rows) { + if(!used_rows()) { static const Origin empty_origin={"empty string"}; return empty_origin; } @@ -355,18 +389,18 @@ const Origin& String::origin() const { #endif String& String::mid(size_t start, size_t finish) const { + String& result=*NEW String(pool()); + start=max(0, start); finish=min(size(), finish); if(start==finish) - return *empty_string; - - String& result=*NEW String(pool()); + return result; size_t pos=0; const Chunk *chunk=&head; do { const Chunk::Row *row=chunk->rows; - for(size_t i=0; icount; pos+=row->item.size, i++, row++) { + for(uint i=0; icount; pos+=row->item.size, i++, row++) { if(row==append_here) goto break2; @@ -378,7 +412,7 @@ String& String::mid(size_t start, size_t size_t size=finished?finish-pos:row->item.size; result.APPEND( row->item.ptr+offset, size-offset, - row->item.lang, + (Untaint_lang)row->item.lang, row->item.origin.file, row->item.origin.line); if(finished) goto break2; @@ -393,7 +427,7 @@ break2: } int String::pos(const String& substr, - size_t result, Untaint_lang lang) const { + int result, Untaint_lang lang) const { for(; resultcstr():0, option_bits); @@ -520,25 +554,25 @@ bool String::match(const unsigned char * pcre_tables); if(!code) - THROW(0, 0, + throw Exception(0, 0, ®exp.mid(erroffset, regexp.size()), "regular expression syntax error - %s", errptr); int info_substrings=pcre_info(code, 0, 0); if(info_substrings<0) { - (*pcre_free)(code); - THROW(0, 0, + pcre_free(code); + throw Exception(0, 0, aorigin, "pcre_info error (%d)", info_substrings); } int startoffset=0; - const char *subject=cstr(UL_AS_IS); + const char *subject=cstr(); int length=strlen(subject); int ovecsize; int *ovector=(int *)malloc(sizeof(int)* - (ovecsize=(1/*match*/+info_substrings)*3)); + (ovecsize=(1/*match*/+info_substrings)*3), 11); { // create table Array& columns=*NEW Array(pool()); @@ -560,14 +594,14 @@ bool String::match(const unsigned char * exec_option_bits, ovector, ovecsize); if(exec_substrings==PCRE_ERROR_NOMATCH) { - (*pcre_free)(code); - (*row_action)(**table, 0/*last time, no row*/, 0, 0, info); + pcre_free(code); + row_action(**table, 0/*last time, no row*/, 0, 0, info); return option_bits[1]!=0; // global=true+table, not global=false } if(exec_substrings<0) { - (*pcre_free)(code); - THROW(0, 0, + pcre_free(code); + throw Exception(0, 0, aorigin, "regular expression execute error (%d)", exec_substrings); @@ -583,13 +617,14 @@ bool String::match(const unsigned char * row+=&mid(ovector[i*2+0], ovector[i*2+1]); // .i column value } - (*row_action)(**table, &row, startoffset, ovector[0], info); + row_action(**table, &row, startoffset, ovector[0], info); - if(!option_bits[1] || !(startoffset=ovector[1])) { // not global | going to hang - (*pcre_free)(code); - (*row_action)(**table, 0/*last time, no row*/, 0, 0, info); + if(!option_bits[1] || startoffset==ovector[1]) { // not global | going to hang + pcre_free(code); + row_action(**table, 0/*last time, no row*/, 0, 0, info); return true; } + startoffset=ovector[1]; /* if(option_bits[0] & PCRE_MULTILINE) @@ -614,7 +649,7 @@ String& String::change_case(Pool& pool, b=0; break; default: - PTHROW(0, 0, + throw Exception(0, 0, this, "unknown change case kind #%d", static_cast(kind)); // never @@ -625,11 +660,11 @@ String& String::change_case(Pool& pool, const Chunk *chunk=&head; do { const Chunk::Row *row=chunk->rows; - for(size_t i=0; icount; i++, row++) { + for(uint i=0; icount; i++, row++) { if(row==append_here) goto break2; - char *new_cstr=(char *)pool.malloc(row->item.size); + char *new_cstr=(char *)pool.malloc(row->item.size, 12); char *dest=new_cstr; const char *src=row->item.ptr; for(int size=row->item.size; size--; src++) { @@ -641,7 +676,7 @@ String& String::change_case(Pool& pool, } result.APPEND(new_cstr, row->item.size, - row->item.lang, + (Untaint_lang)row->item.lang, row->item.origin.file, row->item.origin.line); } chunk=row->link; @@ -651,18 +686,175 @@ break2: return result; } +void String::join_chain(Pool& pool, + uint& ai, const Chunk*& achunk, const Chunk::Row*& arow, + Untaint_lang& joined_lang, const char *& joined_ptr, size_t& joined_size) const { + joined_lang=(Untaint_lang)arow->item.lang; + + // calc size + joined_size=0; + { + uint start_i=ai; + const Chunk::Row *start_row=arow; + const Chunk *chunk=achunk; + do { + const Chunk::Row *row=start_row; + for(uint i=start_i; icount; i++, row++) { + if(row==append_here) + goto break21; + + if(row->item.lang==joined_lang) + joined_size+=row->item.size; + else + break; + } + if(chunk=row->link) { + start_i=0; + start_row=chunk->rows; + } else + break; + } while(true); +break21:; + } + + // if one row, return simply itself + if(joined_size==arow->item.size) { + joined_ptr=arow->item.ptr; + ai++; arow++; + if(ai==achunk->count) + achunk=arow->link; + } else { + // join adjacent rows + char *ptr=(char *)pool.malloc(joined_size,13); + joined_ptr=ptr; + uint start_i=ai; + const Chunk::Row *start_row=arow; + const Chunk *chunk=achunk; + uint i; + const Chunk::Row *row; + do { + row=start_row; + for(i=start_i; icount; i++, row++) { + if(row==append_here) + goto break22; + + if(row->item.lang==joined_lang) { + memcpy(ptr, row->item.ptr, row->item.size); + ptr+=row->item.size; + } else + break; + } + if(chunk=row->link) { + start_i=0; + start_row=chunk->rows; + } else + break; + } while(true); +break22:; + + // return joined rows + ai=i; + arow=row; + achunk=chunk; + } +} + +String& String::reconstruct(Pool& pool) const { + //_asm int 3; + String& result=*new(pool) String(pool); + const Chunk *chunk=&head; + do { + const Chunk::Row *row=chunk->rows; + for(uint i=0; icount; ) { + if(row==append_here) + goto break2; + + Untaint_lang joined_lang; + const char *joined_ptr; + size_t joined_size; + join_chain(pool, i, chunk, row, + joined_lang, joined_ptr, joined_size); + + result.APPEND(joined_ptr, joined_size, + joined_lang, + row->item.origin.file, row->item.origin.line); + if(!chunk) + goto break2; + } + } while(true); +break2: + + return result; +}; + +String& String::replace_in_reconstructed(Pool& pool, Dictionary& dict) const { + //_asm int 3; + String& result=*new(pool) String(pool); + const Chunk *chunk=&head; + do { + const Chunk::Row *row=chunk->rows; + for(uint i=0; icount; i++, row++) { + if(row==append_here) + goto break2; + + const char *src=row->item.ptr; + size_t src_size=row->item.size; + char *new_cstr=(char *)pool.malloc((size_t)ceil(src_size*dict.max_ratio()), 14); + char *dest=new_cstr; + while(src_size) { + // there is a row where first column starts 'src' + if(Table::Item *item=dict.first_that_starts(src, src_size)) { + // get a=>b values + const String& a=*static_cast(item)->get_string(0); + const String& b=*static_cast(item)->get_string(1); + // skip 'a' in 'src' && reduce work size + src+=a.size(); src_size-=a.size(); + // write 'b' to 'dest' && skip 'b' in 'dest' + b.store_to(dest); dest+=b.size(); + } else { + // write a char to b && reduce work size + *dest++=*src++; src_size--; + } + } + + result.APPEND(new_cstr, dest-new_cstr, + (Untaint_lang)row->item.lang, + row->item.origin.file, row->item.origin.line); + } + chunk=row->link; + } while(chunk); + +break2: + return result; +} + +String& String::replace(Pool& pool, Dictionary& dict) const { + return reconstruct(pool).replace_in_reconstructed(pool, dict); +} + double String::as_double() const { double result; - const char *cstr=this->cstr(); - char *error_pos=0; + const char *cstr; + char buf[MAX_NUMBER]; + if(head.rows+1==append_here) { + int size=min(head.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.rows[0].item.ptr, size); + buf[size]=0; + cstr=buf; + } else + cstr=this->cstr(); + char *error_pos; // 0xABC - if(cstr[0]=='0' && (cstr[1]=='x' || cstr[1]=='X')) - result=(double)(unsigned long)strtol(cstr, &error_pos, 0); + if(cstr[0]=='0') + if(cstr[1]=='x' || cstr[1]=='X') + result=(double)(unsigned long)strtol(cstr, &error_pos, 0); + else + result=(double)strtod(cstr+1/*skip leading 0*/, &error_pos); else - result=strtod(cstr, &error_pos); + result=(double)strtod(cstr, &error_pos); - if(error_pos && *error_pos) - THROW(0, 0, + if(*error_pos/*not EOS*/) + throw Exception(0, 0, this, "invalid number (double)"); @@ -670,18 +862,95 @@ double String::as_double() const { } int String::as_int() const { int result; - const char *cstr=this->cstr(); - char *error_pos=0; + const char *cstr; + char buf[MAX_NUMBER]; + if(head.rows+1==append_here) { + int size=min(head.rows[0].item.size, MAX_NUMBER-1); + memcpy(buf, head.rows[0].item.ptr, size); + buf[size]=0; + cstr=buf; + } else + cstr=this->cstr(); + char *error_pos; // 0xABC - if(cstr[0]=='0' && (cstr[1]=='x' || cstr[1]=='X')) - result=(int)(unsigned long)strtol(cstr, &error_pos, 0); + if(cstr[0]=='0') + if(cstr[1]=='x' || cstr[1]=='X') + result=(int)(unsigned long)strtol(cstr, &error_pos, 0); + else + result=(int)strtol(cstr+1/*skip leading 0*/, &error_pos, 0); else result=(int)strtol(cstr, &error_pos, 0); - if(error_pos && *error_pos) - THROW(0, 0, + if(*error_pos/*not EOS*/) + throw Exception(0, 0, this, "invalid number (int)"); return result; } + +/* @todo maybe network order worth spending some effort? + don't bothering myself with network byte order, + am not planning to be able to move resulting file across platforms + for now +*/ +void String::serialize(size_t prolog_size, void *& buf, size_t& buf_size) const { + buf_size= + prolog_size + +used_rows()*(sizeof(Untaint_lang)+sizeof(size_t)) + +size(); + buf=malloc(buf_size,15); + char *cur=(char *)buf+prolog_size; + + const Chunk *chunk=&head; + do { + const Chunk::Row *row=chunk->rows; + for(uint i=0; icount; i++) { + if(row==append_here) + goto break2; + + // lang + memcpy(cur, &row->item.lang, sizeof(Untaint_lang)); + cur+=sizeof(Untaint_lang); + // size + memcpy(cur, &row->item.size, sizeof(size_t)); + cur+=sizeof(size_t); + // bytes + memcpy(cur, row->item.ptr, row->item.size); + cur+=row->item.size; + + row++; + } + chunk=row->link; + } while(chunk); +break2: + ; +} + +/* @todo maybe network order worth spending some effort? + don't bothering myself with network byte order, + am not planning to be able to move resulting file across platforms + for now +*/ +#ifndef DOXYGEN +struct Serialized_piece { + String::Untaint_lang lang; + size_t size; + char ptr[1]; +}; +#endif + +void String::deserialize(size_t prolog_size, void *buf, size_t buf_size, const char *file) { + char *cur=((char *)buf)+prolog_size; + buf_size-=prolog_size; + + while(buf_size) { + Serialized_piece& p=*(Serialized_piece *)cur; + APPEND(p.ptr, p.size, p.lang, file, 0); + + size_t piece_size=sizeof(p.lang)+sizeof(p.size)+p.size; + cur+=piece_size; + buf_size-=piece_size; + } +} +