segmentunicodeandascii.d

bool isascii(dchar c){
	return c <dchar(127);
}
auto processtext(dstring s){
	import std.sumtype;
	struct range{
		dstring s;
		bool isunicode;
		long i,j;
		alias sumtype=SumType!(string,dchar);
		sumtype front(){
			if(isunicode){
				return sumtype(s[i..j].to!dchar);
			} else {
				return sumtype(s[i..j].to!string);
		}}
		void popFront(){
			i=j;
			if(empty){return;}
			if( ! isunicode){
				isunicode=true;
				j=i+1;
				//j=s[i..$].countUntil!(a=>!a.isascii);
			} else {//if(isunicode)
				if(s[j].isascii){
					j=s[i..$].countUntil!(a=>!a.isascii)+i;
					isunicode=false;
				} else {
					j++;
			}}
		}
		auto pop(){
			popFront;
			return this;
		}
		bool empty(){
			return j>=s.length;
		}
	}
	bool isascii=s.length!=0?s[0].isascii:false;
	return range(s,!isascii).pop;
}
unittest{
	dstring foo="Lorem ipsum dolor sit amet, 你好, consectetur adipiscing elit. 🌍 Morbi euismod, こんにちは, quam at tincidunt pulvinar, tortor metus bibendum mauris, eget vulputate lacus elit sit amet elit. Duis ullamcorper ligula at lectus bibendum, 🐶🐱 neque consectetur dapibus. Nullam eget augue ac metus maximus volutpat. 😃";
	import std.stdio;
	foo.processtext.each!writeln;
}