Skip to content

Instantly share code, notes, and snippets.

@stekhn
Created August 6, 2018 15:47
Show Gist options
  • Save stekhn/d3170b5a4b0d8b02f2d4805b24d98281 to your computer and use it in GitHub Desktop.
Save stekhn/d3170b5a4b0d8b02f2d4805b24d98281 to your computer and use it in GitHub Desktop.
Concurrently write one JSON file from multiple processes spawned by the Node.js "cluster" module. Uses a lockfile to prevent hickups.
const fs = require('fs');
const os = require('os');
const path = require('path');
const cluster = require('cluster');
const lockFile = require('lockfile');
const filePath = path.resolve(__dirname, 'result.json');
const lockPath = path.resolve(__dirname, 'result.json.lock');
const lockOptions = {
wait: 1000,
stale: 5000,
retries: 100,
retryWait: 1000
};
(function init() {
if (cluster.isMaster) {
// Create empty JSON file or overwrite old JSON file
fs.writeFileSync(filePath, JSON.stringify([]), 'utf8');
// Create new worker for each CPU core
for (var i = 0; i < os.cpus().length; i++) {
cluster.fork().on('error', (error) => { console.error(error); });
}
} else if (cluster.isWorker) {
writeToJson(cluster.worker.id);
}
})();
function writeToJson(workerId) {
lockFile.lock(lockPath, lockOptions, (error) => {
if (error) { console.error(error); }
// Open existing JSON file
const oldJson = JSON.parse(fs.readFileSync(filePath, 'utf8'));
// Merge object array with new data
const newJson = oldJson.concat([{ id: workerId }]);
console.log(newJson);
// Save new data to the existing JSON file
fs.writeFileSync(filePath, JSON.stringify(newJson), 'utf8');
lockFile.unlock(lockPath, (error) => {
if (error) { console.error(error); }
cluster.worker.kill();
process.exit(0);
});
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment