Skip to content

Instantly share code, notes, and snippets.

@mattmorgis
Last active May 26, 2022 15:17
Show Gist options
  • Save mattmorgis/f2e71a83b2caaabd1ec61a35baa3999c to your computer and use it in GitHub Desktop.
Save mattmorgis/f2e71a83b2caaabd1ec61a35baa3999c to your computer and use it in GitHub Desktop.
Parse JSON with Async Iterator
{
"name": "async-iterator-with-json-stream",
"version": "0.1.0",
"description": "Use an async iterator or generator with streaming JSON.parse",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "Matt Morgis",
"license": "MIT",
"dependencies": {
"JSONStream": "^1.3.5",
"request": "^2.88.0"
}
}
const request = require("request");
const JSONStream = require("JSONStream");
const stream = require("stream");
// Needed to bind JSONStream with async iterator
const passThrough = new stream.PassThrough({
objectMode: true
});
const jsonStream = request
.get(
"https://s3.amazonaws.com/philadelphia-parking-violations-raw-data/parking_violations_2017.json"
)
.pipe(JSONStream.parse("rows.*"))
.pipe(passThrough);
async function run() {
let counter = 0;
try {
for await (const chunk of jsonStream) {
counter++;
console.log(chunk);
}
console.log(`CALLED ${counter} TIMES`);
console.log(" ### Done ###");
const used = process.memoryUsage().heapUsed / 1024 / 1024;
console.log(`The script uses approximately ${used} MB`);
} catch (e) {
console.log(e.message);
process.exit(-1);
}
}
run();
@johnib
Copy link

johnib commented May 26, 2022

Can you explain why this PassThrough was needed?
How come the JSONStream outputs parsed JSONs (i.e. actual objects) and we still need to convert them via an objectMode stream?

@mattmorgis
Copy link
Author

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment