--- parser3/src/main/pa_string.C	2003/09/24 07:13:52	1.175.2.3
+++ parser3/src/main/pa_string.C	2003/11/20 17:46:02	1.190
@@ -5,7 +5,7 @@
 	Author: Alexandr Petrosian <paf@design.ru> (http://paf.design.ru)
 */
 
-static const char* IDENT_STRING_C="$Date: 2003/09/24 07:13:52 $";
+static const char * const IDENT_STRING_C="$Date: 2003/11/20 17:46:02 $";
 
 #include "pcre.h"
 
@@ -15,6 +15,8 @@ static const char* IDENT_STRING_C="$Date
 #include "pa_dictionary.h"
 #include "pa_charset.h"
 
+const String String::Empty;
+
 // cord lib extension
 
 #ifndef DOXYGEN
@@ -32,15 +34,28 @@ static int CORD_range_contains_chr_great
     if (c > d -> target) return(1);
     return(0);
 }
-int CORD_range_contains_chr_greater_then(CORD x, size_t i, size_t n, int not_c)
+int CORD_range_contains_chr_greater_then(CORD x, size_t i, size_t n, int c)
 {
     chr_data d;
 
-    d.countdown = i+n;
-    d.target = not_c;
+    d.countdown = n;
+    d.target = c;
     return(CORD_block_iter(x, i, CORD_range_contains_chr_greater_then_proc, &d) == 1/*alternatives: 0 normally ended, 2=struck 'n'*/);
 }
 
+static int CORD_block_count_proc(char /*c*/, size_t /*size*/, void* client_data)
+{
+    int* result=(int*)client_data;
+    (*result)++;
+    return(0); // 0=continue
+}
+size_t CORD_block_count(CORD x)
+{
+	size_t result=0;
+	CORD_block_iter(x, 0, CORD_block_count_proc, &result);
+    return result;
+}
+
 // helpers
 
 /// String::match uses this as replace & global search table columns
@@ -54,19 +69,19 @@ public:
 		*this+=new String("match");
 		*this+=new String("postmatch");
 		for(int i=0; i<MAX_MATCH_GROUPS; i++) {
-			*this+=new String(StringBody::Format(1+i), String::L_CLEAN);
+			*this+=new String(String::Body::Format(1+i), String::L_CLEAN);
 		}
 	}
 };
 
 Table string_match_table_template(new String_match_table_template_columns);
 
-// StringBody methods
+// String::Body methods
 
-StringBody StringBody::Format(int value) {
+String::Body String::Body::Format(int value) {
 	char local[MAX_NUMBER];
 	size_t length=snprintf(local, MAX_NUMBER, "%d", value);
-	return StringBody(pa_strdup(local, length), length);
+	return String::Body(pa_strdup(local, length), length);
 }
 
 static int CORD_batched_iter_fn_generic_hash_code(char c, void * client_data) {
@@ -79,7 +94,7 @@ static int CORD_batched_iter_fn_generic_
 	generic_hash_code(result, s);
 	return 0;
 };
-uint StringBody::hash_code() const {
+uint String::Body::hash_code() const {
 	uint result=0;
 	CORD_iter5(body, 0,
 		CORD_batched_iter_fn_generic_hash_code, 
@@ -138,13 +153,14 @@ String& String::mid(size_t substr_begin,
 	size_t self_length=length();
 	substr_begin=min(substr_begin, self_length);
 	substr_end=min(max(substr_end, substr_begin), self_length);
-	if(substr_begin==substr_end)
+	size_t substr_length=substr_end-substr_begin;
+	if(!substr_length)
 		return result;
 
 	// first: their langs
-	result.langs.append(result.body, langs, substr_begin, substr_end);
+	result.langs.append(result.body, langs, substr_begin, substr_length);
 	// next: letters themselves
-	result.body=body.mid(substr_begin, substr_end-substr_begin);
+	result.body=body.mid(substr_begin, substr_length);
 
 //	SAPI::log("piece of '%s' from %d to %d is '%s'",
 		//cstr(), substr_begin, substr_end, result.cstr());
@@ -152,12 +168,19 @@ String& String::mid(size_t substr_begin,
 	return result;
 }
 
-size_t String::pos(const StringBody substr, size_t this_offset, Language lang) const {
-	size_t substr_begin=body.pos(substr, this_offset);
-	if(substr_begin==CORD_NOT_FOUND || !langs.check_lang(lang, substr_begin, substr.length()))
-		return STRING_NOT_FOUND;
+size_t String::pos(const String::Body substr, size_t this_offset, Language lang) const {
+	size_t substr_length=substr.length();
+	while(true) {
+		size_t substr_begin=body.pos(substr, this_offset);
+		
+		if(substr_begin==CORD_NOT_FOUND)
+			return STRING_NOT_FOUND;
 
-	return substr_begin;
+		if(langs.check_lang(lang, substr_begin, substr_length))
+			return substr_begin;
+
+		this_offset=substr_begin+substr_length;
+	}
 }
 
 size_t String::pos(const String& substr, 
@@ -171,7 +194,7 @@ void String::split(ArrayString& result,
 		   Language lang, int limit) const {
 	size_t self_length=length();
 	if(size_t delim_length=strlen(delim)) {
-		int pos_before;
+		size_t pos_before;
 		// while we have 'delim'...
 		for(; (pos_before=pos(delim, pos_after, lang))!=STRING_NOT_FOUND && limit; limit--) {
 			result+=&mid(pos_after, pos_before);
@@ -193,7 +216,7 @@ void String::split(ArrayString& result,
 		   const String& delim, Language lang, 
 		   int limit) const {
 	if(!delim.is_empty()) {
-		int pos_before;
+		size_t pos_before;
 		// while we have 'delim'...
 		for(; (pos_before=pos(delim, pos_after, lang))!=STRING_NOT_FOUND && limit; limit--) {
 			result+=&mid(pos_after, pos_before);
@@ -218,13 +241,13 @@ static void regex_options(const String*
 		int *result;
 		bool *flag;
     } regex_option[]={
-		{"i", "I", 0, PCRE_CASELESS, result}, // a=A
-		{"s", "S", 0, PCRE_DOTALL, result}, // \n\n$ [default]
-		{"x", "U", 0, PCRE_EXTENDED, result}, // whitespace in regex ignored
-		{"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result}, // ^aaa\n$^bbb\n$
-		{"g", "G", 0, true, result+1}, // many rows
+		{"i", "I", 0, PCRE_CASELESS, result, 0}, // a=A
+		{"s", "S", 0, PCRE_DOTALL, result, 0}, // \n\n$ [default]
+		{"x", "U", 0, PCRE_EXTENDED, result, 0}, // whitespace in regex ignored
+		{"m", "M", PCRE_DOTALL, PCRE_MULTILINE, result, 0}, // ^aaa\n$^bbb\n$
+		{"g", "G", 0, 1, result+1, 0}, // many rows
 		{"'", 0, 0, 0, 0, &need_pre_post_match},
-		{0}
+		{0, 0, 0, 0, 0, 0}
     };
 	result[0]=PCRE_EXTRA | PCRE_DOTALL | PCRE_DOLLAR_ENDONLY;
 	result[1]=0;
@@ -320,9 +343,9 @@ Table* String::match(Charset& source_cha
 			*row+=&mid(prefinish, poststart); // .match
 			*row+=&mid(poststart, postfinish); // .postmatch
 		} else {
-			*row+=0; // .prematch column value
-			*row+=0; // .match
-			*row+=0; // .postmatch
+			*row+=&Empty; // .prematch column value
+			*row+=&Empty; // .match
+			*row+=&Empty; // .postmatch
 		}
 		
 		for(int i=1; i<exec_substrings; i++) {
@@ -351,37 +374,49 @@ String& String::change_case(Charset& sou
 	if(is_empty())
 		return result;
 
-	const unsigned char *tables=source_charset.pcre_tables;
-
-	const unsigned char *a;
-	const unsigned char *b;
-	switch(kind) {
-	case CC_UPPER:
-		a=tables+lcc_offset;
-		b=tables+fcc_offset;
-		break;
-	case CC_LOWER:
-		a=tables+lcc_offset;
-		b=0;
-		break;
-	default:
-		throw Exception(0, 
-			this, 
-			"unknown change case kind #%d", 
-				static_cast<int>(kind)); // never
-		a=b=0; // calm, compiler
-		break; // never
-	}	
-
 	char* new_cstr=cstrm();
 	char *dest=new_cstr;
-	unsigned char index;
-	for(const char* current=new_cstr; index=(unsigned char)*current; current++) {
-		unsigned char c=a[index];
-		if(b)
-			c=b[c];
+	if(source_charset.isUTF8()) {
+		switch(kind) {
+		case CC_UPPER:
+			change_case_UTF8((const XMLByte*)new_cstr, (XMLByte*)new_cstr, UTF8CaseToUpper);
+			break;
+		case CC_LOWER:
+			change_case_UTF8((const XMLByte*)new_cstr, (XMLByte*)new_cstr, UTF8CaseToLower);
+			break;
+		default:
+			assert(!"unknown change case kind");
+			break; // never
+		}	
+		
+	} else {
+		const unsigned char *tables=source_charset.pcre_tables;
 
-		*dest++=(char)c;
+		const unsigned char *a;
+		const unsigned char *b;
+		switch(kind) {
+		case CC_UPPER:
+			a=tables+lcc_offset;
+			b=tables+fcc_offset;
+			break;
+		case CC_LOWER:
+			a=tables+lcc_offset;
+			b=0;
+			break;
+		default:
+			assert(!"unknown change case kind");
+			a=b=0; // calm, compiler
+			break; // never
+		}	
+
+		unsigned char index;
+		for(const char* current=new_cstr; (index=(unsigned char)*current); current++) {
+			unsigned char c=a[index];
+			if(b)
+				c=b[c];
+
+			*dest++=(char)c;
+		}
 	}
 	result.langs=langs;
 	result.body=new_cstr;
@@ -396,30 +431,26 @@ const String& String::replace(const Dict
 
 	const char* current=old_cstr;
 	while(*current) {
-		if(Table::element_type row=dict.first_that_begins(current)) {
+		if(Dictionary::Subst subst=dict.first_that_begins(current)) {
 			// prematch
 			if(size_t prematch_length=current-prematch_begin) {
-				result.langs.append(result.body, langs, prematch_begin-old_cstr, current-old_cstr);
+				result.langs.append(result.body, langs, prematch_begin-old_cstr, prematch_length);
 				result.body.append_strdup_know_length(prematch_begin, prematch_length);
 			}
 
 			// match
-
-			const String* a=row->get(0);
 			// skip 'a' in 'current'; move prematch_begin
-			current+=a->length(); prematch_begin=current;
+			current+=subst.from_length; prematch_begin=current;
 
-			if(row->count()>1) { // are there any b?
-				const String* b=row->get(1);
+			if(const String* b=subst.to) // are there any b?
 				result<<*b;
-			}
 		} else // simply advance
 			current++; 
 	}
 
 	// postmatch
 	if(size_t postmatch_length=current-prematch_begin) {
-		result.langs.append(result.body, langs, prematch_begin-old_cstr, current-old_cstr);
+		result.langs.append(result.body, langs, prematch_begin-old_cstr, postmatch_length);
 		result.body.append_strdup_know_length(prematch_begin, postmatch_length);
 	}
 
@@ -482,63 +513,51 @@ int String::as_int() const {
 	return result;
 }
 
-inline void uint2uchars(uint word, uchar *bytes) {
-	bytes[0]=word&0xFF;
-	bytes[1]=(word>>8)&0xFF;
-	bytes[2]=(word>>16)&0xFF;
-	bytes[3]=(word>>24)&0xFF;
-}
-inline uint uchars2uint(uchar *bytes) {
-	return bytes[3]<<24
-		| bytes[2]<<16
-		| bytes[1]<<8
-		| bytes[0];
-}
-
+static int serialize_body_char(char c, char** cur) {
+	*((*cur)++)=c;
+	return 0; // 0=continue
+};
 static int serialize_body_piece(const char* s, char** cur) {
 	size_t length=strlen(s);
 	memcpy(*cur, s, length);  *cur+=length;
-	return 0;
+	return 0; // 0=continue
 };
+static int serialize_lang_piece(char alang, size_t asize, char** cur) {
+	// lang
+	memcpy(*cur, &alang, sizeof(alang));  *cur+=sizeof(alang);
+	// length
+	memcpy(*cur, &asize, sizeof(asize));  *cur+=sizeof(asize);
+
+	return 0; // 0=continue
+}
 String::Cm String::serialize(size_t prolog_length) const {
-#if TODO
-	//_asm int 3;
+	size_t fragments_count=langs.count();
 	size_t buf_length=
-		prolog_length
-		+sizeof(size_t)
-		+langs.count()*(sizeof(Language)+sizeof(size_t))
-		+length();
+		prolog_length //1
+		+sizeof(size_t) //2
+		+fragments_count*(sizeof(char)+sizeof(size_t)) //3
+		+body.length() //4
+		+1; // for zero terminator used in deserialize
 	String::Cm result(new(PointerFreeGC) char[buf_length], buf_length);
 
 	// 1: prolog
 	char *cur=result.str+prolog_length;
-
-
 	// 2: langs.count
-	size_t fragments_count=langs.count();
 	memcpy(cur, &fragments_count, sizeof(fragments_count));  cur+=sizeof(fragments_count);
-
 	// 3: lang info
-	for(Array_iterator<ArrayFragment::element_type> i(langs); i.has_next(); ) {
-		const Fragment fragment=i.next();
-		// lang
-		memcpy(cur, &fragment.lang, sizeof(fragment.lang));  cur+=sizeof(fragment.lang);
-		// length
-		memcpy(cur, &fragment.length, sizeof(fragment.length));  cur+=sizeof(fragment.length);
-	}
-
+	langs.for_each(body, serialize_lang_piece, &cur);
 	// 4: letters
-	body.for_each(serialize_body_piece, &cur);
+	body.for_each(serialize_body_char, serialize_body_piece, &cur);
+	// 5: zero terminator
+	*cur=0;
 
 	return result;
-#endif
-	return String::Cm(0, 0);
 }
 bool String::deserialize(size_t prolog_length, void *buf, size_t buf_length) {
-#if TODO
 	if(buf_length<=prolog_length)
 		return false;
 	buf_length-=prolog_length;
+	buf_length-=1; // 5: zero terminator
 
 	// 1: prolog
 	const char* cur=(const char* )buf+prolog_length;
@@ -553,13 +572,13 @@ bool String::deserialize(size_t prolog_l
 		// 3: lang info
 		size_t total_length=0;
 		for(size_t f=0; f<fragments_count; f++) {
-			size_t piece_length=sizeof(Language)+sizeof(size_t);
+			size_t piece_length=sizeof(char)+sizeof(size_t);
 			if(buf_length<piece_length) // lang+length
 				return false;
 
-			Language lang=*reinterpret_cast<const Language *>(cur);  cur+=sizeof(Language);
+			Language lang=*reinterpret_cast<const Language *>(cur);  cur+=sizeof(char);
 			size_t fragment_length=*reinterpret_cast<const size_t*>(cur);  cur+=sizeof(size_t);
-			langs+=Fragment(lang, fragment_length);
+			langs.append(total_length, lang, fragment_length);
 			total_length+=fragment_length;
 
 			buf_length-=piece_length;
@@ -569,11 +588,35 @@ bool String::deserialize(size_t prolog_l
 		if(buf_length!=total_length)
 			return false;
 
-		body=StringBody(cur, buf_length);
+		// serialize wrote extra zero byte there, we can rely on that
+		body=String::Body(cur, buf_length);
 	}
 
 	ASSERT_STRING_INVARIANT(*this);
 	return true;
-#endif
-	return false;
+}
+
+const char* String::Body::v() const {
+	return CORD_to_const_char_star(body);
+}
+const char* String::Languages::v() const {
+	if(opt.is_not_just_lang)
+		return CORD_to_const_char_star(langs);
+	else
+		return (const char*)&langs;
+}
+const char* String::v() const {
+#define LIMIT_VIEW 20
+	char* buf=(char*)malloc(MAX_STRING);
+	const char*body_view=body.v();
+	const char*langs_view=langs.v();
+	snprintf(buf, MAX_STRING, 
+		"%d:%.*s%s}   "
+		"{%d:%s",
+		langs.count(), LIMIT_VIEW, langs_view, strlen(langs_view)>LIMIT_VIEW?"...":"",
+		strlen(body_view), body_view
+	);
+
+	return buf;
+#undef LIMIT_VIEW
 }