--- parser3/src/classes/file.C	2016/07/21 18:30:10	1.243
+++ parser3/src/classes/file.C	2017/02/06 16:17:12	1.261
@@ -25,7 +25,7 @@
 #include "pa_vregex.h"
 #include "pa_version.h"
 
-volatile const char * IDENT_FILE_C="$Id: file.C,v 1.243 2016/07/21 18:30:10 moko Exp $";
+volatile const char * IDENT_FILE_C="$Id: file.C,v 1.261 2017/02/06 16:17:12 moko Exp $";
 
 // defines
 
@@ -118,6 +118,7 @@ static const char* suexec_safe_env_lst[]
 
 // statics
 
+static const String::Body size_name("size");
 static const String::Body adate_name("adate");
 static const String::Body mdate_name("mdate");
 static const String::Body cdate_name("cdate");
@@ -133,7 +134,7 @@ static void _save(Request& r, MethodPara
 		if(HashStringValue* options=params.as_hash(2)){
 			int valid_options=0;
 			if(Value* vcharset_name=options->get(PA_CHARSET_NAME)){
-				asked_charset=&::charsets.get(vcharset_name->as_string().change_case(r.charsets.source(), String::CC_UPPER));
+				asked_charset=&pa_charsets.get(vcharset_name->as_string());
 				valid_options++;
 			}
 			if(valid_options != options->count())
@@ -153,11 +154,11 @@ static void _delete(Request& r, MethodPa
 		if(HashStringValue* options=params.as_hash(1)){
 			int valid_options=0;
 			if(Value* vkeep_empty_dirs=options->get(KEEP_EMPTY_DIRS_NAME)){
-				keep_empty_dirs=r.process_to_value(*vkeep_empty_dirs).as_bool();
+				keep_empty_dirs=r.process(*vkeep_empty_dirs).as_bool();
 				valid_options++;
 			}
 			if(Value* vsuppress_exception=options->get(SUPPRESS_EXCEPTION_NAME)){
-				fail_on_problem=r.process_to_value(*vsuppress_exception).as_bool();
+				fail_on_problem=r.process(*vsuppress_exception).as_bool();
 				valid_options++;
 			}
 			if(valid_options != options->count())
@@ -177,7 +178,7 @@ static void _move(Request& r, MethodPara
 		if(HashStringValue* options=params.as_hash(2)){
 			int valid_options=0;
 			if(Value* vkeep_empty_dirs=options->get(KEEP_EMPTY_DIRS_NAME)){
-				keep_empty_dirs=r.process_to_value(*vkeep_empty_dirs).as_bool();
+				keep_empty_dirs=r.process(*vkeep_empty_dirs).as_bool();
 				valid_options++;
 			}
 			if(valid_options != options->count())
@@ -191,7 +192,7 @@ static void _move(Request& r, MethodPara
 		keep_empty_dirs);
 }
 
-static void copy_process_source(struct stat& , int from_file, const String&, void *context) {
+static void copy_process_source(struct stat&, int from_file, const String&, void *context) {
 	int& to_file=*static_cast<int *>(context);
 
 	int nCount=0;
@@ -262,10 +263,10 @@ static void _load(Request& r, MethodPara
 	if(options){
 		options=new HashStringValue(*options);
 		if(Value *voffset=(Value *)options->get(sql_offset_name)){
-			offset=r.process_to_value(*voffset).as_int();
+			offset=r.process(*voffset).as_int();
 		}
 		if(Value *vlimit=(Value *)options->get(sql_limit_name)){
-			limit=r.process_to_value(*vlimit).as_int();
+			limit=r.process(*vlimit).as_int();
 		}
 		// no check on options count here, see file_read
 	}
@@ -285,7 +286,7 @@ static void _load(Request& r, MethodPara
 	if(file.headers){
 		file.headers->for_each<HashStringValue*>(_load_pass_param, &self.fields());
 	} else {
-		size_t size;
+		uint64_t size;
 		time_t atime, mtime, ctime;
 
 		file_stat(lfile_name, size, atime, mtime, ctime);
@@ -318,7 +319,8 @@ static void _create(Request& r, MethodPa
 	}
 
 	VString* vcontent_type=0;
-	Charset* asked_charset=0;
+	Charset* to_charset=0;
+	Charset* from_charset=0;
 	if(params.count()>options_index)
 		if(HashStringValue* options=params.as_hash(options_index)) {
 			int valid_options=0;
@@ -333,8 +335,18 @@ static void _create(Request& r, MethodPa
 					valid_options++;
 				}
 			}
+			if(Value* vcharset_name=options->get("to-charset")) {
+				to_charset=&pa_charsets.get(vcharset_name->as_string());
+				valid_options++;
+			}
+			if(Value* vcharset_name=options->get("from-charset")) {
+				from_charset=&pa_charsets.get(vcharset_name->as_string());
+				valid_options++;
+			}
 			if(Value* vcharset_name=options->get(PA_CHARSET_NAME)) {
-				asked_charset=&::charsets.get(vcharset_name->as_string().change_case(r.charsets.source(), String::CC_UPPER));
+				if(to_charset)
+					throw Exception(PARSER_RUNTIME, 0, "charset option can not be used with to-charset");
+				to_charset=&pa_charsets.get(vcharset_name->as_string());
 				valid_options++;
 			}
 			if(Value* value=options->get(CONTENT_TYPE_NAME)) {
@@ -351,30 +363,38 @@ static void _create(Request& r, MethodPa
 
 	if(const String* content_str=vcontent.get_string()){
 		String::Body body=content_str->cstr_to_string_body_untaint(String::L_AS_IS, r.connection(false), &r.charsets); // explode content, honor tainting changes
-		if(asked_charset && is_text)
-			body=Charset::transcode(body, r.charsets.source(), *asked_charset);
 		self.set(true/*tainted*/, is_text, body.cstrm(), body.length(), file_name, vcontent_type, &r);
 	} else {
-		if(asked_charset)
-			throw Exception(PARSER_RUNTIME, 0, "charset option can not be used with file-content");
-		self.set(*vcontent.as_vfile(String::L_AS_IS), mode != 0, is_text, file_name, vcontent_type, &r);
+		VFile& fcontent=*vcontent.as_vfile(String::L_AS_IS); // can't be null
+		if(mode){
+			self.set(fcontent, &is_text, file_name, vcontent_type, &r);
+			if(is_text && !fcontent.is_text_mode())
+				from_charset=self.detect_binary_charset(from_charset);
+		} else {
+			self.set(fcontent, 0, file_name, vcontent_type, &r);
+			is_text=fcontent.is_text_mode();
+		}
 	}
 
+	if(to_charset || from_charset)
+		if(is_text)
+			self.transcode(from_charset ? *from_charset : r.charsets.source(), to_charset ? *to_charset : r.charsets.source());
+		else
+			throw Exception(PARSER_RUNTIME, 0, "charset options can not be used with binary content");
 }
 
 static void _stat(Request& r, MethodParams& params) {
 	const String& lfile_name=params.as_string(0, FILE_NAME_MUST_NOT_BE_CODE);
 
-	size_t size;
+	uint64_t size;
 	time_t atime, mtime, ctime;
-	file_stat(r.absolute(lfile_name),
-		size,
-		atime, mtime, ctime);
+	file_stat(r.absolute(lfile_name), size, atime, mtime, ctime);
 	
 	VFile& self=GET_SELF(r, VFile);
 
-	self.set_binary(true/*tainted*/, 0/*no bytes*/, size, &lfile_name, 0, &r);
+	self.set_binary(true/*tainted*/, 0 /*no bytes*/, 0 /*fake size*/, &lfile_name, 0, &r);
 	HashStringValue& ff=self.fields();
+	ff.put(size_name, new VDouble((double)size) /*real size*/);
 	ff.put(adate_name, new VDate((pa_time_t)atime));
 	ff.put(mdate_name, new VDate((pa_time_t)mtime));
 	ff.put(cdate_name, new VDate((pa_time_t)ctime));
@@ -485,14 +505,14 @@ static void _exec_cgi(Request& r, Method
 	env.put("SCRIPT_NAME", script_name);
 
 	// environment & stdin from param
-	String *in=new String();
+	bool in_is_text_mode=true;
+	String::C in;
 	Charset *charset=0; // default script works raw_in 'source' charset = no transcoding needed
 	if(param_index < params.count()) {
 		if(HashStringValue* user_env=params.as_hash(param_index++, "env")) {
 			// $.charset  [previewing to handle URI pieces]
 			if(Value* vcharset=user_env->get(CHARSET_EXEC_PARAM_NAME))
-				charset=&charsets.get(vcharset->as_string()
-					.change_case(r.charsets.source(), String::CC_UPPER));
+				charset=&pa_charsets.get(vcharset->as_string());
 
 			// $.others
 			Append_env_pair_info info={&r.charsets, &env, 0};
@@ -506,14 +526,13 @@ static void _exec_cgi(Request& r, Method
 			if(info.vstdin) {
 				if(const String* sstdin=info.vstdin->get_string()) {
 					// untaint stdin
-					in = new String(sstdin->cstr_to_string_body_untaint(String::L_AS_IS, r.connection(false), &r.charsets), String::L_AS_IS);
+					in = String::C(sstdin->cstr_to_string_body_untaint(String::L_AS_IS, r.connection(false), &r.charsets));
 				} else
-					if(VFile* vfile=static_cast<VFile *>(info.vstdin->as("file")))
-						in->append_know_length((const char* )vfile->value_ptr(), vfile->value_size(), String::L_TAINTED);
-					else
-						throw Exception(PARSER_RUNTIME,
-							0,
-							STDIN_EXEC_PARAM_NAME " parameter must be string or file");
+					if(VFile* vfile=static_cast<VFile *>(info.vstdin->as("file"))){
+						in = String::C((const char* )vfile->value_ptr(), vfile->value_size());
+						in_is_text_mode = vfile->is_text_mode();
+					} else
+						throw Exception(PARSER_RUNTIME, 0, STDIN_EXEC_PARAM_NAME " parameter must be string or file");
 			}
 		}
 	}
@@ -532,13 +551,10 @@ static void _exec_cgi(Request& r, Method
 				} else {
 					Table* table=param.get_table();
 					if(table){
-						for(size_t i=0; i<table->count(); i++) {
-							append_to_argv(r, argv, table->get(i)->get(0));
-						}
+						for(size_t j=0; j<table->count(); j++)
+							append_to_argv(r, argv, table->get(j)->get(0));
 					} else {
-						throw Exception(PARSER_RUNTIME,
-							0,
-							"param must be string or table");
+						throw Exception(PARSER_RUNTIME, 0, "param must be string or table");
 					}
 				}
 			}
@@ -549,14 +565,15 @@ static void _exec_cgi(Request& r, Method
 	if(charset) {
 		Charset::transcode(env, r.charsets.source(), *charset);
 		Charset::transcode(argv, r.charsets.source(), *charset);
-		in=&Charset::transcode(*in, r.charsets.source(), *charset);
+		if(in_is_text_mode)
+			in=Charset::transcode(in, r.charsets.source(), *charset);
 	}
 	// @todo 
 	// ifdef WIN32 do  OEM->ANSI transcode on some(.cmd?) programs to 
 	// match silent conversion in OS
 
 	// exec!
-	PA_exec_result execution=pa_exec(false/*forced_allow*/, script_name, &env, argv, *in);
+	PA_exec_result execution=pa_exec(false/*forced_allow*/, script_name, &env, argv, in);
 
 	File_read_result *file_out=&execution.out;
 	String *real_err=&execution.err;
@@ -635,8 +652,7 @@ static void _exec_cgi(Request& r, Method
 		// $fields << header
 		if(header) {
 			ArrayString rows;
-			size_t pos_after=0;
-			header->split(rows, pos_after, eol_marker);
+			header->split(rows, 0, eol_marker);
 			Pass_cgi_header_attribute_info info={0, 0, 0};
 			info.charset=&r.charsets.source();
 			info.fields=&self.fields();
@@ -646,7 +662,7 @@ static void _exec_cgi(Request& r, Method
 		}
 	} else { // ^file::exec
 		// $body
-		self.set(false/*not tainted*/, is_text, file_out->str, file_out->length);
+		self.set(false/*not tainted*/, is_text, file_out->str ? file_out->str : pa_strdup("") /*to distinguish from stat-ed file*/, file_out->length);
 	}
 
 	// $status
@@ -677,7 +693,7 @@ static void _list(Request& r, MethodPara
 			if(HashStringValue* options=voption.get_hash()) {
 				int valid_options=0;
 				if(Value* vstat=options->get("stat")) {
-					stat=r.process_to_value(*vstat).as_bool();
+					stat=r.process(*vstat).as_bool();
 					valid_options++;
 				}
 				if(Value* value=options->get("filter")) {
@@ -714,10 +730,10 @@ static void _list(Request& r, MethodPara
 	int ovector[ovector_size];
 
 	LOAD_DIR(absolute_path_cstr, 
-		const char* file_name_cstr=ffblk.ff_name;
+		const char* file_name_cstr=ffblk.name();
 		size_t file_name_size=strlen(file_name_cstr);
 
-		if(!vregex || vregex->exec(ffblk.ff_name, file_name_size, ovector, ovector_size)>=0) {
+		if(!vregex || vregex->exec(file_name_cstr, file_name_size, ovector, ovector_size)>=0) {
 			Table::element_type row(new ArrayString);
 			*row+=new String(pa_strdup(file_name_cstr, file_name_size), String::L_TAINTED);
 			*row+=new String(String::Body::Format(ffblk.is_dir(stat) ? 1 : 0), String::L_CLEAN);
@@ -732,7 +748,7 @@ static void _list(Request& r, MethodPara
 	);
 
 	// write out result
-	r.write_no_lang(*new VTable(&table));
+	r.write(*new VTable(&table));
 }
 
 #ifndef DOXYGEN
@@ -745,7 +761,7 @@ struct Lock_execute_body_info {
 static void lock_execute_body(int , void *ainfo) {
 	Lock_execute_body_info& info=*static_cast<Lock_execute_body_info *>(ainfo);
 	// execute body
-	info.r->write_assign_lang(info.r->process(*info.body_code));
+	info.r->write(info.r->process(*info.body_code));
 }
 
 static void _lock(Request& r, MethodParams& params) {
@@ -785,7 +801,7 @@ static void _find(Request& r, MethodPara
 
 	// easy way
 	if(file_exist(r.absolute(*file_spec))) {
-		r.write_assign_lang(*file_spec);
+		r.write(*file_spec);
 		return;
 	}
 
@@ -803,7 +819,7 @@ static void _find(Request& r, MethodPara
 		test_name << dirname.mid(0, slash+1);
 		test_name << basename;
 		if(file_exist(r.absolute(test_name))) {
-			r.write_assign_lang(test_name);
+			r.write(test_name);
 			return;
 		}
 		rpos=slash;
@@ -811,7 +827,7 @@ static void _find(Request& r, MethodPara
 
 	// no way, not found
 	if(not_found_code)
-		r.write_pass_lang(r.process(*not_found_code));
+		r.write(r.process(*not_found_code));
 }
 
 static void _dirname(Request& r, MethodParams& params) {
@@ -829,22 +845,22 @@ static void _dirname(Request& r, MethodP
 	// file    > .
 
 	if(file_spec.is_empty()) {
-		r.write_assign_lang(String("."));
+		r.write(String("."));
 		return;
 	}
 
 	size_t p;
 	size_t slash;
 	if((p=file_spec.rskipchars("/\\"))==STRING_NOT_FOUND)
-		r.write_assign_lang(String("/"));
+		r.write(String("/"));
 	else {
 		if((slash=file_spec.strrpbrk("/\\", 0, p))!=STRING_NOT_FOUND) {
 			if((p=file_spec.rskipchars("/\\", 0, slash))==STRING_NOT_FOUND)
 				p=slash;
-			r.write_assign_lang(file_spec.mid(0, p+1));
+			r.write(file_spec.mid(0, p+1));
 			return;
 		}
-		r.write_assign_lang(String("."));
+		r.write(String("."));
 	}
 }
 
@@ -863,15 +879,15 @@ static void _basename(Request& r, Method
 	// file    > file
 
 	if(file_spec.is_empty()) {
-		r.write_assign_lang(String("."));
+		r.write(String("."));
 		return;
 	}
 
 	size_t p=file_spec.rskipchars("/\\");
 	if(p==STRING_NOT_FOUND)
-		r.write_assign_lang(String("/"));
+		r.write(String("/"));
 	else
-		r.write_assign_lang(file_spec.mid(afterlastslash(file_spec, p), p+1));
+		r.write(file_spec.mid(afterlastslash(file_spec, p), p+1));
 }
 
 static void _justname(Request& r, MethodParams& params) {
@@ -881,7 +897,7 @@ static void _justname(Request& r, Method
 	// /a/b.c  > b
 	size_t pos=afterlastslash(file_spec);
 	size_t dotpos=file_spec.strrpbrk(".", pos);
-	r.write_assign_lang(file_spec.mid(pos, dotpos!=STRING_NOT_FOUND?dotpos:file_spec.length()));
+	r.write(file_spec.mid(pos, dotpos!=STRING_NOT_FOUND?dotpos:file_spec.length()));
 }
 
 static void _justext(Request& r, MethodParams& params) {
@@ -891,7 +907,7 @@ static void _justext(Request& r, MethodP
 	size_t pos=afterlastslash(file_spec);
 	size_t dotpos=file_spec.strrpbrk(".", pos);
 	if(dotpos!=STRING_NOT_FOUND)
-		r.write_assign_lang(file_spec.mid(dotpos+1, file_spec.length()));
+		r.write(file_spec.mid(dotpos+1, file_spec.length()));
 }
 
 static void _fullpath(Request& r, MethodParams& params) {
@@ -911,14 +927,14 @@ static void _fullpath(Request& r, Method
 		}
 		result=&full_disk_path.mid(document_root_length,  full_disk_path.length());
 	}
-	r.write_assign_lang(*result);
+	r.write(*result);
 }
 
 static void _sql_string(Request& r, MethodParams&) {
 	VFile& self=GET_SELF(r, VFile);
 
 	const char *quoted=r.connection()->quote(self.value_ptr(), self.value_size());
-	r.write_assign_lang(*new String(quoted));
+	r.write(*new String(quoted));
 }
 
 #ifndef DOXYGEN
@@ -977,9 +993,8 @@ public:
 static void _sql(Request& r, MethodParams& params) {
 	Value& statement=params.as_junction(0, "statement must be code");
 
-	Temp_lang temp_lang(r, String::L_SQL);
 	const String& statement_string=r.process_to_string(statement);
-	const char* statement_cstr=statement_string.untaint_cstr(r.flang, r.connection());
+	const char* statement_cstr=statement_string.untaint_cstr(String::L_SQL, r.connection());
 
 	File_sql_event_handlers handlers(statement_string, statement_cstr);
 
@@ -999,11 +1014,11 @@ static void _sql(Request& r, MethodParam
 			}
 			if(Value* vlimit=options->get(sql_limit_name)) {
 				valid_options++;
-				limit=(ulong)r.process_to_value(*vlimit).as_double();
+				limit=(ulong)r.process(*vlimit).as_double();
 			}
 			if(Value* voffset=options->get(sql_offset_name)) {
 				valid_options++;
-				offset=(ulong)r.process_to_value(*voffset).as_double();
+				offset=(ulong)r.process(*voffset).as_double();
 			}
 			if(valid_options!=options->count())
 				throw Exception(PARSER_RUNTIME, 0, CALLED_WITH_INVALID_OPTION);
@@ -1059,7 +1074,7 @@ static void _base64(Request& r, MethodPa
 							valid_options++;
 						}
 						if(Value* vstrict=options->get(BASE64_STRICT_OPTION_NAME)) {
-							strict=r.process_to_value(*vstrict).as_bool();
+							strict=r.process(*vstrict).as_bool();
 							valid_options++;
 						}
 						if(valid_options!=options->count())
@@ -1079,13 +1094,13 @@ static void _base64(Request& r, MethodPa
 		} else {
 			// encode: ^f.base64[]
 			const char* encoded=pa_base64_encode(self.value_ptr(), self.value_size());
-			r.write_assign_lang(*new String(encoded, String::L_TAINTED/*once ?param=base64(something) was needed**/));
+			r.write(*new String(encoded, String::L_TAINTED/*once ?param=base64(something) was needed**/));
 		}
 	} else {
 		// encode: ^file:base64[filespec]
 		const String& file_spec=params.as_string(0, FILE_NAME_MUST_BE_STRING);
 		const char* encoded=pa_base64_encode(r.absolute(file_spec));
-		r.write_assign_lang(*new String(encoded, String::L_TAINTED/*once ?param=base64(something) was needed*/));
+		r.write(*new String(encoded, String::L_TAINTED/*once ?param=base64(something) was needed*/));
 	}
 }
 
@@ -1104,7 +1119,7 @@ static void _crc32(Request& r, MethodPar
 		VFile& self=GET_SELF(r, VFile);
 		crc32=pa_crc32(self.value_ptr(), self.value_size());
 	}
-	r.write_no_lang(*new VInt(crc32));
+	r.write(*new VInt(crc32));
 }
 
 
@@ -1161,7 +1176,7 @@ static void _md5(Request& r, MethodParam
 		md5=pa_md5(self.value_ptr(), self.value_size());
 
 	}
-	r.write_no_lang(*new String(md5));
+	r.write(*new String(md5));
 }
 
 // constructor