Created
April 30, 2015 00:37
-
-
Save cmawhorter/9ff5ebd388e423c8ce45 to your computer and use it in GitHub Desktop.
I was getting V8 crashes in nodejs 0.12.1 while trying to load large (not really. 500mb.) JSON files and so I needed this.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // This loads large-ish new line separated JSON files into memory. If you need truly large file support | |
| // and getting errors, you might want to google nodejs v8 heap limit or something. | |
| // I believe it's the --max-old-space-size option in particular. | |
| // Another script I wrote generated the JSON files in this | |
| // format, so this script matches the format. | |
| // File Lines: | |
| // 1. [ | |
| // 2. null | |
| // 3. , { some: 'object', data: 'here' } | |
| // ... | |
| // 9184753. , { some: 'object', data: 'here' } | |
| // 9184754. ] | |
| // 9184755. (empty line) | |
| var fs = require('fs'); | |
| var readline = require('readline'); | |
| module.exports = function(fileName, callback) { | |
| var lineCount = 0; | |
| var data = [null]; // null at data[0] is expected. you can use data.slice(1) or whatever | |
| readline.createInterface({ | |
| input: fs.createReadStream(fileName), | |
| terminal: false | |
| }).on('line', function(line) { | |
| console.log('Loaded line %s', ++lineCount); | |
| if (line && line[0] === ',') { | |
| data.push(JSON.parse(line.substr(1))); | |
| } | |
| }) | |
| .on('error', function(err) { | |
| callback(err); | |
| }) | |
| .on('close', function() { // should be end, but readline doesn't have an end event documented? | |
| callback(null, data); | |
| }); | |
| }; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment