node,js asyncronous JSON.stringify stringifyToStream ----- Intended for saving JSON of a large object out to a file without using up memory or choking out other activity on the node ----- Caveat: do not modify the object as it is being written out

stringifyToStream.js

// Copyright 2013 Paul Brewer
// License: You may copy this file under the same terms as the MIT License
// located at http://opensource.org/licenses/MIT
// This file is provided AS IS WITH NO WARRANTY OF ANY KIND. All use is at your own risk.
//
// Status: July 13, 2013, Paul Brewer:  First version. Simple test case passed.  It needs to be more extensively tested.
//   It lacks escaping for odd characters in strings, though a workaround is possible as described in the comments.
// 
// July 14, 2013, Paul Brewer:  JSON.stringify() used to stringify basic
// types such as string, number, null/undefined.  This provides string
// escaping correctly, which is complex and important.
//
// .toJSON detected and called for objects like Date
//
// added indicator function for misc punctuation
// plan is now a list of items to stringify, so functions are used
// for punctuation since functions are not otherwise permitted in json
// Testing for particular punctuation is attempted with === for efficiency
// instead of evaluating the function. If an unknown function appears
// in plan, it came from the object directly, and so we ignore it.

function leftSquareBracket(){ return '[' }
function rightSquareBracket(){ return ']' }
function leftCurlyBracket(){ return '{' }
function rightCurlyBracket(){ return '}' }
function comma(){ return ',' }
function quote(){ return '"' }
function quoteColon(){ return '":' }
function colon(){ return ':' }


function stringifyToStream(obj, stream, onsuccess, onfail){
    var i,l, plan=[];

    function nextChunk(){
	var cursor,str='';
	try {
	    if (plan.length === 0) return onsuccess();
	    cursor = plan.shift();
	    if (typeof cursor === 'undefined'){ 
		str = 'null' 
	    } else if (typeof cursor === 'object'){
		if (cursor === null){ 
		    str='null' 
		} else if (typeof cursor.toJSON === 'function'){
		    str = '"'+cursor.toJSON()+'"';
		} else { 
		    return stringifyToStream(cursor, stream, nextChunk, onfail)
		}
	    } else if (typeof cursor === 'function'){
		if (cursor === comma){
		    str = ",\n"; 
		} else if (cursor === colon){
		    str = ':';
		} else if (cursor === leftSquareBracket){
		    str = '[';
		} else if (cursor === rightSquareBracket){
		    str = ']';
		} else if (cursor === leftCurlyBracket){
		    str = '{';
		} else if (cursor === rightCurlyBracket){
		    str = '}';
		} else if (cursor === quote){
		    str = '"'; 
		} else if (cursor === quoteColon){
		    str = '":';
		} else return nextChunk(); // ignore unknown funcs in plan
	    } else str = JSON.stringify(cursor); // not a function,object,array
	    return stream.write(str, nextChunk);
	} catch(e){ 
	    return onfail(e); 
	}
    }

    if (typeof obj !== 'object'){
	return stream.write(JSON.stringify(obj), onsuccess);
    }

    if (typeof obj.toJSON === 'function'){ 
	return stream.write(obj.toJSON(), onsuccess);
    }

    if (Object.prototype.toString.call(obj)==='[object Array]'){
	plan.push(leftSquareBracket);
	if (obj.length > 0){ 
	    for(i=0,l=obj.length;i<l;++i){
		plan.push(obj[i], comma);
	    }
	    plan.pop(); // extra comma
	}
	plan.push(rightSquareBracket);
    } else {
	plan.push(leftCurlyBracket);
	l = 0;
	for (i in obj){
	    if (obj.hasOwnProperty(i)){
		plan.push(i,colon,obj[i],comma);
		++l;
	    }
	}
	if (l>0) plan.pop(); // remove last comma
	plan.push(rightCurlyBracket);
    }    

    return nextChunk();

}

exports.stringifyToStream = stringifyToStream;

test.js

var stringifyToStream = require('./stringifyToStream.js').stringifyToStream;

// simple test case

function test1(cb){
    testobj = {'A': [1,3,4,6,7,8,9],
	       'b': { 'hi': 'there', 'ok': 78, bs: [9,8,false,'ohno!']},
	       'c': [ {a: 34, c: 98, e:true }, {x:56, y:true, z:false}]
	      };
    
    // print out the first line with the synchronous JSON.stringify
    console.log("This is test1 -- the next output is the legacy JSON.stringify, which we take as correct:");
    var correct = JSON.stringify(testobj);
    console.log(correct);
    console.log("Check manually that it matches the following output, which is from async stringify to stream:");
    
    function test1Compare(){
	console.log();
	console.log("test #1 completed -- you must judge pass/fail manually");
    }
    
    function test1Error(e){
	console.log();
	console.log("test #1 internal error -- "+e);
    }
    
    // after 1 sec print a seond line using the asynchronous stringifyToStream 
    
    setTimeout(
	stringifyToStream.bind(null, 
			       testobj, 
			       process.stdout, 
			       test1Compare, 
			       test1Error), 
	1000);


}

test1();

TODO LIst for DrPaulBrewer/stingifyToStream.js -- 14 Jul 2013 Paul Brewer:

If someone wants to do any of this stuff, or something else, feel free to fork and go ahead.

THIS LIST IS FOR ME TOO. I'm not necessarily expecting help.

NOT DONE Write some larger tests, streaming to a disk file, for performance testing purposes.

DONE -- The code might be made a little more layered and DRYer by not doing stream.write at the top level but only inside nextChunk --

DONE but just go ahead and assume JSON.stringify exists -- Test for existing JSON.stringify and use that to stringify strings in nextChunk before they go out on stream.write.

DONE This would take care of escaping strings properly. Create some special objects that can be pushed into plan in the top level [] and {} loops that represent the comma, [. ]. {. } symbols and can be tested for with ===. These shouldn't be strings because we'd like to pass strings through legacy JSON.stringify to get escaping.

DONE Check if non-prototypical onJSON exists for Objects

Interesting discussion on asynchronous JSON.stringify feature request for mozilla browser (rejected),

https://bugzilla.mozilla.org/show_bug.cgi?id=832664