ndjson-to-file

README.md

Trying to save a big blob of JSON that crashes JSON.stringify. Spliting into smaller arrays and saving works, but I can't figure out how to use a stream.

index.js

var fs = require('fs')

//generate fake data
var data = [
  {foo: 1},
  {bar: 3},
]

var bigData = [
  data,
  data,
  data,
  data,
  data  
]

var count = 100100
for (var i = 0; i < count; i++){
  bigData.push(data)
}




//write out 10 blobs of json. works but isn't too pretty...
var maxItems = Math.floor(count/10)
var i = 0
while (i*maxItems < count){
  var dataSlice = JSON.stringify(bigData.slice(i*maxItems, (i + 1)*maxItems))
  fs.writeFileSync(i + '.json', dataSlice)
  i++
}





//this breaks - don't think i'm piping to the wstream correctly?
var ndjson = require('ndjson')
var serialize = ndjson.serialize()
var wstream = fs.createWriteStream('myfile.json');
serialize.on('data', function(line) {
  // line is a line of stringified JSON with a newline delimiter at the end
  // console.log(line)

})
serialize.write(bigData)
serialize.pipe(wstream)

serialize.end()







// this also breaks
var JSONStream = require('JSONStream');
var es = require('event-stream');
var fs = require('fs');

var obj = {};
for (var i = 0; i < 2000; i++) {
    obj['prop' + i] = 'value' + i;
}

var out = fs.createWriteStream(__dirname + '/out.json');

es.readable(function (count, next) {
    // for (var key in obj) {
    //     this.emit('data', [key, obj[key]]);
    // }

    var that = this
    bigData.forEach(function(d){
      that.emit('data', d)
    })
    that.emit('end');
    next();
}).pipe(JSONStream.stringify()).pipe(out);

did you succeed ? tried this one which works: https://www.bennadel.com/blog/3233-parsing-and-serializing-large-datasets-using-newline-delimited-json-in-node-js.htm