Created
January 12, 2022 13:33
-
-
Save gabrieledarrigo/31aac513c2a778b848ffd7891660e0e2 to your computer and use it in GitHub Desktop.
Write JSON in a for loop, it causes a memory leak
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const path = require('path'); | |
const NUMBER_OF_ROWS = process.env.NUMBER_OF_ROWS || 100_000_000; | |
const JSON_OUTPUT = path.resolve(__dirname, `data_${Date.now()}.ndjson`); | |
const { info, error } = console; | |
(async () => { | |
function writeJSON() { | |
return new Promise((resolve, reject) => { | |
let index = 0; | |
const jsonWriter = fs.createWriteStream(JSON_OUTPUT, { flags: 'a' }); | |
jsonWriter.on('error', reject); | |
jsonWriter.on('close', () => { | |
info(`\nCorrectly wrote ${NUMBER_OF_ROWS} in ndjson format\n`); | |
resolve(); | |
}); | |
function write(data, next) { | |
const wrote = jsonWriter.write(data); | |
if (!wrote) { | |
jsonWriter.once('drain', next); | |
} else { | |
process.nextTick(next); | |
} | |
} | |
function run() { | |
if (index < NUMBER_OF_ROWS) { | |
index++; | |
write(`${JSON.stringify({ data: index })}\n`, run); | |
} else { | |
jsonWriter.end(); | |
} | |
} | |
info(`Writing ${JSON_OUTPUT} file\n`); | |
run(); | |
}); | |
} | |
await writeJSON(); | |
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const path = require('path'); | |
const NUMBER_OF_ROWS = process.env.NUMBER_OF_ROWS || 100_000_000; | |
const JSON_OUTPUT = path.resolve(__dirname, `data_${Date.now()}.ndjson`); | |
(async () => { | |
async function writeJSON() { | |
return new Promise((resolve, reject) => { | |
const jsonWriter = fs.createWriteStream(JSON_OUTPUT, { flags: 'a' }); | |
jsonWriter.on('error', reject); | |
jsonWriter.on('close', () => { | |
info(`\nCorrectly wrote ${NUMBER_OF_ROWS} in ndjson format\n`); | |
resolve(); | |
}); | |
info(`Writing ${JSON_OUTPUT} file\n`); | |
for (let i = 0; i < NUMBER_OF_ROWS; i += 1) { | |
(async () => { | |
jsonWriter.write(`${JSON.stringify({ data: i })}\n`); | |
})(); | |
} | |
jsonWriter.end(); | |
}); | |
} | |
await writeJSON(); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment