Streaming string tokenizer, made specifically for Node.js, but could be used in the browser too. Note fully tested yet!

StreamingTokenizer.js

function StreamingTokenizer(delimiter, callback) {
	this.delimiter = delimiter;
	this.callback = callback;
	this.pendingText = '';
	this.open = true;
}

StreamingTokenizer.prototype = {
	add: function(text) {
		if (!this.open) {
			throw 'StreamingTokenizer closed';
		}
		this.pendingText += text;
		var lines = this.pendingText.split(this.delimiter);
		if (lines.length == 0) {
			return;
		}
		for (var i = 0; i < lines.length - 1; ++i) {
			this.callback(lines[i]);
		}
		this.pendingText = lines[lines.length - 1];
	},
	close: function() {
		if (this.pendingText.length > 0) {
			this.callback(this.pendingText);
		}
		this.pendingText = '';
		this.open = false;
	}
};