Skip to content

Instantly share code, notes, and snippets.

@mattmorgis
Last active May 26, 2022 15:17
Show Gist options
  • Select an option

  • Save mattmorgis/f2e71a83b2caaabd1ec61a35baa3999c to your computer and use it in GitHub Desktop.

Select an option

Save mattmorgis/f2e71a83b2caaabd1ec61a35baa3999c to your computer and use it in GitHub Desktop.
Parse JSON with Async Iterator
{
"name": "async-iterator-with-json-stream",
"version": "0.1.0",
"description": "Use an async iterator or generator with streaming JSON.parse",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "Matt Morgis",
"license": "MIT",
"dependencies": {
"JSONStream": "^1.3.5",
"request": "^2.88.0"
}
}
const request = require("request");
const JSONStream = require("JSONStream");
const stream = require("stream");
// Needed to bind JSONStream with async iterator
const passThrough = new stream.PassThrough({
objectMode: true
});
const jsonStream = request
.get(
"https://s3.amazonaws.com/philadelphia-parking-violations-raw-data/parking_violations_2017.json"
)
.pipe(JSONStream.parse("rows.*"))
.pipe(passThrough);
async function run() {
let counter = 0;
try {
for await (const chunk of jsonStream) {
counter++;
console.log(chunk);
}
console.log(`CALLED ${counter} TIMES`);
console.log(" ### Done ###");
const used = process.memoryUsage().heapUsed / 1024 / 1024;
console.log(`The script uses approximately ${used} MB`);
} catch (e) {
console.log(e.message);
process.exit(-1);
}
}
run();
@mattmorgis
Copy link
Author

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment