Last active
May 26, 2022 15:17
-
-
Save mattmorgis/f2e71a83b2caaabd1ec61a35baa3999c to your computer and use it in GitHub Desktop.
Parse JSON with Async Iterator
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "async-iterator-with-json-stream", | |
"version": "0.1.0", | |
"description": "Use an async iterator or generator with streaming JSON.parse", | |
"main": "index.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"author": "Matt Morgis", | |
"license": "MIT", | |
"dependencies": { | |
"JSONStream": "^1.3.5", | |
"request": "^2.88.0" | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const request = require("request"); | |
const JSONStream = require("JSONStream"); | |
const stream = require("stream"); | |
// Needed to bind JSONStream with async iterator | |
const passThrough = new stream.PassThrough({ | |
objectMode: true | |
}); | |
const jsonStream = request | |
.get( | |
"https://s3.amazonaws.com/philadelphia-parking-violations-raw-data/parking_violations_2017.json" | |
) | |
.pipe(JSONStream.parse("rows.*")) | |
.pipe(passThrough); | |
async function run() { | |
let counter = 0; | |
try { | |
for await (const chunk of jsonStream) { | |
counter++; | |
console.log(chunk); | |
} | |
console.log(`CALLED ${counter} TIMES`); | |
console.log(" ### Done ###"); | |
const used = process.memoryUsage().heapUsed / 1024 / 1024; | |
console.log(`The script uses approximately ${used} MB`); | |
} catch (e) { | |
console.log(e.message); | |
process.exit(-1); | |
} | |
} | |
run(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Can you explain why this
PassThrough
was needed?How come the
JSONStream
outputs parsed JSONs (i.e. actual objects) and we still need to convert them via anobjectMode
stream?