Get, Parse and Transform JSONStreams on the fly before delivering to the browser

node-express-jsonstream.js

// Get, Parse and Transform JSONStreams on the fly before delivering to the browser
// ref: http://nodestreams.com/

var express = require('express'),
    server = express();

var fs = require("fs"),
    JSONStream = require("JSONStream"),
    es = require('event-stream');

server.all(
  '/my/json/api*',
  function (req, res, next){

    // Stream output will be a string and you cannot pipe to express.json() so make sure the browser knows its JSON
    res.setHeader('Content-Type', 'application/json; charset=utf-8');

    // Source can be a File or from a Request
    var source = fs.createReadStream("data/huge_ammount_of.json"),
    
    /*
      JSONStream can parse the source in a number of formats, e.g.
  
      1) {...} newline separated JSON Objects
      2) [
           {...} array of JSON Objects
         ]
      3) {
           "some_props": "...",
           "rows": [
             {...}  nested array of JSON Objects in a wrapper
           ]
         }      
    */    
    parser = JSONStream.parse("rows.*"),

    // Convert a single JSON Object
    convert = es.mapSync(
      function(data) {
        console.log('data received:', data);
        
        // e.g. add a property
        data.converted = true;
        return data;
      }
    );

  
    // Read file
    source
      // Parse JSON
      .pipe(parser)
      // convert individual JSON Object
      .pipe(convert)
      // convert JSON to strings in place
      .pipe(JSONStream.stringify())
      // output Strings
      .pipe(res);

  }
);

server.listen('1337');