parser3/src/main/pa_string.C - diff

Return to pa_string.C CVS log

Up to [parser3project] / parser3 / src / main

Diff for /parser3/src/main/pa_string.C between versions 1.172.2.21.2.8 and 1.172.2.21.2.29

version 1.172.2.21.2.8, 2003/03/20 13:09:54	version 1.172.2.21.2.29, 2003/03/25 10:21:08
Line 1	Line 1
/** @file	/** @file
Parser: string class. @see untasize_t.C.	Parser: string class. @see untalength_t.C.

Copyright (c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)	Copyright (c) 2001-2003 ArtLebedev Group (http://www.artlebedev.com)
Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)	Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)
Line 38 Table string_match_table_template(new St	Line 38 Table string_match_table_template(new St

// String::ArrayFragment methods	// String::ArrayFragment methods

String::ArrayFragment& String::ArrayFragment::append(const ArrayFragment& src,	void String::ArrayFragment::append_positions(const ArrayFragment& src,
size_t substr_start, size_t substr_end) {	size_t substr_begin, size_t substr_end) {
if(substr_start==substr_end)	if(substr_begin==substr_end)
return *this;	return;

size_t fragment_start=0;	size_t fragment_begin=0;
size_t fragment_end;	size_t fragment_end;
for(Array_iterator<element_type> i(src); ; fragment_start=fragment_end) {	for(Array_iterator<element_type> i(src); ; fragment_begin=fragment_end) {
const element_type& fragment=i.next();	const element_type& fragment=i.next();
fragment_end=fragment_start+fragment.size;	fragment_end=fragment_begin+fragment.length;

	if(substr_begin<fragment_begin) // not reached fragments which may include 'substr'?
	continue;

if(substr_start<fragment_start) // not reached fragments which may include 'substr'?	if(substr_begin>fragment_end) // not reached fragments which may include 'substr'?
continue;	continue;

// found first fragment including piece of 'substr'	// found first fragment including piece of 'substr'
size_t piece_end=min(substr_end, fragment_end);	size_t piece_end=min(substr_end, fragment_end);
*this+=Fragment(fragment.lang, piece_end-substr_start);	*this+=Fragment(fragment.lang, piece_end-substr_begin);

while(substr_end>fragment_end) { // are there more fragments including pieces of 'substr'?	while(substr_end>fragment_end) { // are there more fragments including pieces of 'substr'?
	fragment_begin=fragment_end;

const element_type& fragment=i.next();	const element_type& fragment=i.next();
fragment_end=fragment_start+fragment.size;	fragment_end=fragment_begin+fragment.length;

if(substr_end>fragment_end) // are there still more?	if(substr_end>fragment_end) // are there still more?
simple_append(Fragment(fragment.lang, fragment.size)); // appending whole fragment	append(Fragment(fragment.lang, fragment.length)); // appending whole fragment
else { // no, it was last	else { // no, it was last
simple_append(Fragment(fragment.lang, substr_end-fragment_start));	append(Fragment(fragment.lang, substr_end-fragment_begin));
goto break2;	return;
}	}
}	}

break;	break;
}	}
break2:

return *this;
}	}
/*
void String::ArrayFragment::mid(ArrayFragment& result, size_t substr_start, size_t substr_end) {	// StringBody methods

	StringBody StringBody::Format(int value) {
	char local[MAX_NUMBER];
	size_t length=snprintf(local, MAX_NUMBER, "%d", value);
	return StringBody(pa_strdup(local, length), length);
	}

	static int CORD_batched_iter_fn_generic_hash_code(char c, void * client_data) {
	uint& result=static_cast<uint>(client_data);
	generic_hash_code(result, c);
	return 0;
	}
	static int CORD_batched_iter_fn_generic_hash_code(const char* s, void * client_data) {
	uint& result=static_cast<uint>(client_data);
	generic_hash_code(result, s);
	return 0;
	};
	uint StringBody::hash_code() const {
	uint result=0;
	CORD_iter5(body, 0,
	CORD_batched_iter_fn_generic_hash_code,
	CORD_batched_iter_fn_generic_hash_code, &result);
	return result;
}	}
*/

// String methods	// String methods

String::String(const char* src, size_t src_size, bool tainted): body(CORD_EMPTY) {	String::String(const char* cstr, size_t helper_length, bool tainted): body(CORD_EMPTY) {
if(src)	append(cstr, helper_length, tainted?L_TAINTED:L_CLEAN);
if(tainted)
APPEND_TAINTED(src, src_size);
else
APPEND_CLEAN(src, src_size);
}	}

String::String(const String& src): body(src.body), fragments(src.fragments) {}	String::String(const String& src): body(src.body), fragments(src.fragments) {}

String& String::append(const char* src, size_t size, Untaint_lang lang) {	String& String::append(const char* str, size_t length, Language lang) {
if(!src)	if(!str)
return *this;	return *this;
if(!size)	size_t use_length=length?length:strlen(str);
size=strlen(src);	if(!use_length)
if(!size)
return *this;	return *this;

body=CORD_cat_char_star(body, src, size);	body.append(str, use_length);
fragments+=Fragment(lang, size);	fragments+=Fragment(lang, use_length);

return *this;	return *this;
}	}

static int CORD_batched_iter_fn_generic_hash_code(const char* s, void * client_data) {	/// @todo check in doc: whether it documents NOW bad situation "abc".mid(-1, 3) =were?="ab"
uint& result=static_cast<uint>(client_data);	String& String::mid(size_t substr_begin, size_t substr_end) const {
generic_hash_code(result, s);
return 0;
};
uint String::hash_code() const {
uint result=0;
CORD_iter5(body, 0, 0, CORD_batched_iter_fn_generic_hash_code, &result);
return result;
}

const String& String::mid(size_t substr_start, size_t substr_end) const {
String& result=*new String;	String& result=*new String;

substr_start=min(substr_start, size());	size_t self_length=length();
substr_end=max(substr_start, substr_end);	substr_begin=max(min(substr_begin, self_length), (size_t)0);
if(substr_start==substr_end)	substr_end=min(max(substr_end, substr_begin), self_length);
	if(substr_begin==substr_end)
return result;	return result;

// first: letters themselves	// first: letters themselves
result.body=CORD_substr(body, substr_start, substr_end-substr_start);	result.body=body.mid(substr_begin, substr_end-substr_begin);

// next: their langs	// next: their langs
result.fragments.append(fragments, substr_start, substr_end);	result.fragments.append_positions(fragments, substr_begin, substr_end);

// SAPI::log(pool(), "piece of '%s' from %d to %d is '%s'",	// SAPI::log("piece of '%s' from %d to %d is '%s'",
//cstr(), substr_start, substr_end, result.cstr());	//cstr(), substr_begin, substr_end, result.cstr());
return result;	return result;
}	}

size_t String::pos(CORD substr,	size_t String::pos(const StringBody substr,
size_t this_offset, Untaint_lang lang) const {	size_t this_offset, Language lang) const {
// first: letters themselves	// first: letters themselves
size_t substr_start=CORD_str(body, this_offset, substr);	size_t substr_begin=body.pos(substr, this_offset);
if(substr_start==CORD_NOT_FOUND)	if(substr_begin==CORD_NOT_FOUND)
return STRING_NOT_FOUND;	return STRING_NOT_FOUND;

// next: check the lang when specified	// next: check the lang when specified

if(lang==UL_UNSPECIFIED) // ignore lang?	if(lang==L_UNSPECIFIED) // ignore lang?
return substr_start;	return substr_begin;

// substr must be in one fragment, and fragments' lang must = lang	// substr must be in one fragment, and fragments' lang must = lang
size_t substr_end=substr_start+CORD_len(substr);	size_t substr_end=substr_begin+substr.length();
size_t fragment_start=0;	size_t fragment_begin=0;
size_t fragment_end;	size_t fragment_end;
for(Array_iterator<ArrayFragment::element_type> i(fragments); i.has_next(); fragment_start=fragment_end) {	for(Array_iterator<ArrayFragment::element_type> i(fragments); i.has_next(); fragment_begin=fragment_end) {
const Fragment& fragment=i.next();	const Fragment& fragment=i.next();
fragment_end=fragment_start+fragment.size;	fragment_end=fragment_begin+fragment.length;

if(substr_start<fragment_start) // not reached fragments which may include 'result'?	if(substr_begin<fragment_begin) // not reached fragments which may include 'result'?
	continue;
	if(substr_begin>=fragment_end) // begin of substr OUT of current fragment?
continue;	continue;

if(substr_end>fragment_end) // end of substr OUT of current fragment?	if(substr_end>fragment_end) // end of substr OUT of current fragment?
break;	throw Exception(0, // (*) see below
	this,
	"searching for '%s' starting from %ud problem: found begin in one fragment, but end in another",
	substr.cstr(), this_offset);

	if(fragment.lang<=lang)
	return substr_begin;
	else { // bad lang...
	/// WARNING: this possibly skips assert (*), but it's fast
	substr_begin=body.pos(substr, fragment_end/...search AFTER for more/);
	if(substr_begin==CORD_NOT_FOUND)
	return STRING_NOT_FOUND;

return fragment.lang==lang?substr_end:STRING_NOT_FOUND;	size_t substr_end=substr_begin+substr.length();
	// and continuing with next fragment
	}
}	}

return STRING_NOT_FOUND;	return STRING_NOT_FOUND;
}	}

size_t String::pos(const String& substr,	size_t String::pos(const String& substr,
size_t this_offset, Untaint_lang lang) const {	size_t this_offset, Language lang) const {
return pos(substr.body, this_offset, lang);	return pos(substr.body, this_offset, lang);
}	}

void String::split(ArrayString& result,	void String::split(ArrayString& result,
size_t& pos_after,	size_t& pos_after,
const char* delim,	const char* delim,
Untaint_lang lang, int limit) {	Language lang, int limit) const {
size_t self_size=size();	size_t self_length=length();
if(size_t delim_size=strlen(delim)) {	if(size_t delim_length=strlen(delim)) {
int pos_before;	int pos_before;
// while we have 'delim'...	// while we have 'delim'...
for(; (pos_before=pos(delim, pos_after, lang))>=0 && limit; limit--) {	for(; (pos_before=pos(delim, pos_after, lang))>=0 && limit; limit--) {
result+=&mid(pos_after, pos_before);	result+=&mid(pos_after, pos_before);
pos_after=pos_before+delim_size;	pos_after=pos_before+delim_length;
}	}
// last piece	// last piece
if(pos_after<self_size && limit) {	if(pos_after<self_length && limit) {
result+=&mid(pos_after, self_size);	result+=&mid(pos_after, self_length);
pos_after=self_size;	pos_after=self_length;
}	}
} else { // empty delim	} else { // empty delim
result+=this;	result+=this;
pos_after+=self_size;	pos_after+=self_length;
}	}
}	}

void String::split(ArrayString& result,	void String::split(ArrayString& result,
size_t& pos_after,	size_t& pos_after,
const String& delim, Untaint_lang lang,	const String& delim, Language lang,
int limit) const {	int limit) const {
if(delim) {	if(!delim.is_empty()) {
int pos_before;	int pos_before;
// while we have 'delim'...	// while we have 'delim'...
for(; (pos_before=pos(delim, pos_after, lang))>=0 && limit; limit--) {	for(; (pos_before=pos(delim, pos_after, lang))>=0 && limit; limit--) {
result+=&mid(pos_after, pos_before);	result+=&mid(pos_after, pos_before);
pos_after=pos_before+delim.size();	pos_after=pos_before+delim.length();
}	}
// last piece	// last piece
if(pos_after<size() && limit) {	if(pos_after<length() && limit) {
result+=&mid(pos_after, size());	result+=&mid(pos_after, length());
pos_after=size();	pos_after=length();
}	}
} else { // empty delim	} else { // empty delim
result+=this;	result+=this;
pos_after+=size();	pos_after+=length();
}	}
}	}

static void regex_options(const String& options, int *result, bool& need_pre_post_match){	static void regex_options(const String* options, int *result, bool& need_pre_post_match){
struct Regex_option {	struct Regex_option {
const char* keyL;	const char* keyL;
const char* keyU;	const char* keyU;
Line 236 static void regex_options(const String&	Line 261 static void regex_options(const String&
result[0]=PCRE_EXTRA \| PCRE_DOTALL \| PCRE_DOLLAR_ENDONLY;	result[0]=PCRE_EXTRA \| PCRE_DOTALL \| PCRE_DOLLAR_ENDONLY;
result[1]=0;	result[1]=0;

if(options)	if(options && !options->is_empty())
for(Regex_option *o=regex_option; o->keyL; o++)	for(Regex_option *o=regex_option; o->keyL; o++)
if(options.pos(o->keyL)>=0	if(options->pos(o->keyL)>=0
\|\| (o->keyU && options.pos(o->keyU)>=0)) {	\|\| (o->keyU && options->pos(o->keyU)>=0)) {
if(o->flag)	if(o->flag)
*o->flag=true;	*o->flag=true;
else { // result	else { // result
Line 251 static void regex_options(const String&	Line 276 static void regex_options(const String&

Table* String::match(Charset& source_charset,	Table* String::match(Charset& source_charset,
const String& regexp,	const String& regexp,
const String& options,	const String* options,
Row_action row_action, void *info,	Row_action row_action, void *info,
bool& just_matched) const {	bool& just_matched) const {
if(!regexp)	if(regexp.is_empty())
throw Exception(0,	throw Exception(0,
0,	0,
"regexp is empty");	"regexp is empty");
Line 263 Table* String::match(Charset& source_cha	Line 288 Table* String::match(Charset& source_cha
const char* errptr;	const char* errptr;
int erroffset;	int erroffset;
bool need_pre_post_match=false;	bool need_pre_post_match=false;
int option_bits[2]; regex_options(options, option_bits, need_pre_post_match);	int option_bits[2]={0}; regex_options(options, option_bits, need_pre_post_match);
bool global=option_bits[1]!=0;	bool global=option_bits[1]!=0;
pcre *code=pcre_compile(pattern, option_bits[0],	pcre *code=pcre_compile(pattern, option_bits[0],
&errptr, &erroffset,	&errptr, &erroffset,
Line 271 Table* String::match(Charset& source_cha	Line 296 Table* String::match(Charset& source_cha

if(!code)	if(!code)
throw Exception(0,	throw Exception(0,
&regexp.mid(erroffset, regexp.size()),	&regexp.mid(erroffset, regexp.length()),
"regular expression syntax error - %s", errptr);	"regular expression syntax error - %s", errptr);

int subpatterns=pcre_info(code, 0, 0);	int subpatterns=pcre_info(code, 0, 0);
Line 284 Table* String::match(Charset& source_cha	Line 309 Table* String::match(Charset& source_cha
}	}

const char* subject=cstr();	const char* subject=cstr();
size_t subject_size=strlen(subject);	size_t subject_length=strlen(subject);
const int ovecsize=(1/match/+MAX_MATCH_GROUPS)*3;	const int oveclength=(1/match/+MAX_MATCH_GROUPS)*3;
int ovector[ovecsize];	int ovector[oveclength];

// create table	// create table
Table* table=new Table(string_match_table_template);	Table& table=*new Table(string_match_table_template);

int exec_option_bits=0;	int exec_option_bits=0;
int prestart=0;	int prestart=0;
int poststart=0;	int poststart=0;
int postfinish=size();	int postfinish=length();
while(true) {	while(true) {
int exec_substrings=pcre_exec(code, 0,	int exec_substrings=pcre_exec(code, 0,
subject, subject_size, prestart,	subject, subject_length, prestart,
exec_option_bits, ovector, ovecsize);	exec_option_bits, ovector, oveclength);

if(exec_substrings==PCRE_ERROR_NOMATCH) {	if(exec_substrings==PCRE_ERROR_NOMATCH) {
pcre_free(code);	pcre_free(code);
row_action(table, 0/last time, no raw/, 0, 0, poststart, postfinish, info);	row_action(table, 0/last time, no raw/, 0, 0, poststart, postfinish, info);
if(global \|\| subpatterns)	if(global \|\| subpatterns)
return table; // global or with subpatterns=true+result	return &table; // global or with subpatterns=true+result
else {	else {
just_matched=false; return 0; // not global=no result	just_matched=false; return 0; // not global=no result
}	}
Line 341 Table* String::match(Charset& source_cha	Line 366 Table* String::match(Charset& source_cha
if(!global \|\| prestart==poststart) { // not global \| going to hang	if(!global \|\| prestart==poststart) { // not global \| going to hang
pcre_free(code);	pcre_free(code);
row_action(table, 0/last time, no row/, 0, 0, poststart, postfinish, info);	row_action(table, 0/last time, no row/, 0, 0, poststart, postfinish, info);
return table;	return &table;
}	}
prestart=poststart;	prestart=poststart;

Line 352 Table* String::match(Charset& source_cha	Line 377 Table* String::match(Charset& source_cha
}	}
}	}

String& String::change_case(Charset& source_charset, Change_case_kind kind) {	String& String::change_case(Charset& source_charset, Change_case_kind kind) const {
String& result=*new String();	String& result=*new String();

const unsigned char *tables=source_charset.pcre_tables;	const unsigned char *tables=source_charset.pcre_tables;
Line 388 String& String::change_case(Charset& sou	Line 413 String& String::change_case(Charset& sou
*dest++=(char)c;	*dest++=(char)c;
}	}
result.body=new_cstr;	result.body=new_cstr;
result.fragments.append(fragments, 0, fragments.count());	result.fragments.append(fragments);

return result;	return result;
}	}
Line 396 String& String::change_case(Charset& sou	Line 421 String& String::change_case(Charset& sou
const String& String::replace(const Dictionary& dict) const {	const String& String::replace(const Dictionary& dict) const {
String& result=*new String();	String& result=*new String();
const char* old_cstr=cstr();	const char* old_cstr=cstr();
const char* prematch_start=old_cstr;	const char* prematch_begin=old_cstr;

for(const char* current=old_cstr; *current; ) {	for(const char* current=old_cstr; *current; ) {
if(Table::element_type row=dict.first_that_starts(current)) {	if(Table::element_type row=dict.first_that_begins(current)) {
// prematch	// prematch
if(size_t prematch_size=current-prematch_start) {	if(size_t prematch_length=current-prematch_begin) {
result.body=CORD_cat_char_star(result.body, prematch_start, prematch_size);	result.body.append(prematch_begin, prematch_length);
result.fragments.append(fragments, prematch_start-old_cstr, prematch_size);	result.fragments.append_positions(fragments, prematch_begin-old_cstr, current-old_cstr);
}	}

// match	// match

const String* a=row->get(0);	const String* a=row->get(0);
// skip 'a' in 'current'; move prematch_start	// skip 'a' in 'current'; move prematch_begin
current+=a->size(); prematch_start=current;	current+=a->length(); prematch_begin=current;

if(row->count()>1) { // are there any b?	if(row->count()>1) { // are there any b?
const String* b=row->get(1);	const String* b=row->get(1);
if(size_t b_size=b->size()) {	if(size_t b_length=b->length()) {
result.body=CORD_cat(result.body, b->cstr_to_cord());	result.body.append(b->cstr_to_string_body());
result.fragments.append(b->fragments, 0, b->fragments.count());	result.fragments.append(b->fragments);
}	}
}	}
} else // simply advance	} else // simply advance
Line 424 const String& String::replace(const Dict	Line 449 const String& String::replace(const Dict
}	}

// postmatch	// postmatch
if(size_t prematch_size=current-prematch_start) {	if(size_t postmatch_length=current-prematch_begin) {
result.body=CORD_cat_char_star(result.body, prematch_start, prematch_size);	result.body.append(prematch_begin, postmatch_length);
result.fragments.append(fragments, prematch_start-old_cstr, prematch_size);	result.fragments.append_positions(fragments, prematch_begin-old_cstr, current-old_cstr);
}	}

return result;	return result;
Line 500 inline uint uchars2uint(uchar *bytes) {	Line 525 inline uint uchars2uint(uchar *bytes) {
\| bytes[0];	\| bytes[0];
}	}

static int CORD_batched_iter_fn_append(const char* s, void* client_data) {	static int serialize_body_piece(const char* s, char** cur) {
char& cur=static_cast<char**>(client_data);	size_t length=strlen(s);
	memcpy(cur, s, length); cur+=length;
size_t size=strlen(s);
memcpy(cur, s, size); cur+=size;
return 0;	return 0;
};	};
String::C String::serialize(size_t prolog_size) const {	String::C String::serialize(size_t prolog_length) const {
size_t buf_size=	size_t buf_length=
prolog_size	prolog_length
+fragments.count()*(sizeof(Untaint_lang)+sizeof(size_t))	+fragments.count()*(sizeof(Language)+sizeof(size_t))
+size();	+length();
C result(new(PointerFreeGC) char[buf_size], buf_size);	C result(new(PointerFreeGC) char[buf_length], buf_length);

// 1: prolog	// 1: prolog
char *cur=result.str+prolog_size;	char *cur=result.str+prolog_length;


// 2: fragments.count	// 2: fragments.count
Line 527 String::C String::serialize(size_t prolo	Line 550 String::C String::serialize(size_t prolo
const Fragment& fragment=i.next();	const Fragment& fragment=i.next();
// lang	// lang
memcpy(cur, &fragment.lang, sizeof(fragment.lang)); cur+=sizeof(fragment.lang);	memcpy(cur, &fragment.lang, sizeof(fragment.lang)); cur+=sizeof(fragment.lang);
// size	// length
memcpy(cur, &fragment.size, sizeof(fragment.size)); cur+=sizeof(fragment.size);	memcpy(cur, &fragment.length, sizeof(fragment.length)); cur+=sizeof(fragment.length);
}	}

// 4: letters	// 4: letters
CORD_iter5(body, 0, 0, CORD_batched_iter_fn_append, &cur);	body.for_each(serialize_body_piece, &cur);

return result;	return result;
}	}
bool String::deserialize(size_t prolog_size, void buf, size_t buf_size, const char file) {	bool String::deserialize(size_t prolog_length, void *buf, size_t buf_length) {
if(buf_size<=prolog_size)	if(buf_length<=prolog_length)
return false;	return false;
buf_size-=prolog_size;	buf_length-=prolog_length;

// 1: prolog	// 1: prolog
const char* cur=(const char* )buf+prolog_size;	const char* cur=(const char* )buf+prolog_length;

// 2: fragments.count	// 2: fragments.count
if(buf_size<sizeof(size_t)) // fragments.count don't fit?	if(buf_length<sizeof(size_t)) // fragments.count don't fit?
return false;	return false;
size_t fragments_count=reinterpret_cast<const size_t>(cur); cur+=sizeof(size_t);	size_t fragments_count=reinterpret_cast<const size_t>(cur); cur+=sizeof(size_t);
buf_size-=sizeof(size_t);	buf_length-=sizeof(size_t);

// 3: lang info	// 3: lang info
size_t total_size=0;	size_t total_length=0;
for(size_t f=0; f<fragments_count; f++) {	for(size_t f=0; f<fragments_count; f++) {
size_t piece_size=sizeof(Untaint_lang)+sizeof(size_t);	size_t piece_length=sizeof(Language)+sizeof(size_t);
if(buf_size<piece_size) // lang+size	if(buf_length<piece_length) // lang+length
return false;	return false;

Untaint_lang lang=reinterpret_cast<const Untaint_lang >(cur); cur+=sizeof(Untaint_lang);	Language lang=reinterpret_cast<const Language >(cur); cur+=sizeof(Language);
size_t fragment_size=reinterpret_cast<const size_t>(cur); cur+=sizeof(size_t);	size_t fragment_length=reinterpret_cast<const size_t>(cur); cur+=sizeof(size_t);
fragments+=Fragment(lang, fragment_size);	fragments+=Fragment(lang, fragment_length);
total_size+=fragment_size;	total_length+=fragment_length;

buf_size-=piece_size;	buf_length-=piece_length;
}	}

// 4: letters	// 4: letters
if(buf_size!=total_size)	if(buf_length!=total_length)
return false;	return false;

body=CORD_cat_char_star(CORD_EMPTY, cur, buf_size);	body=CORD_cat_char_star(CORD_EMPTY, cur, buf_length);

return true;	return true;
}	}

E-mail:

Removed from v.1.172.2.21.2.8
changed lines
	Added in v.1.172.2.21.2.29