Last active
May 4, 2020 13:43
-
-
Save schellingb/1077d6976c6ee8494e0b2970d90c2df8 to your computer and use it in GitHub Desktop.
Run a maximum of X processes at the same time in a non-async node script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'), child_process = require('child_process'); | |
function RunSync(cmd, args) | |
{ | |
console.log('Running sync: ' + cmd + ' ' + args.join(' ')); | |
const proc = child_process.spawnSync(cmd, args, {stdio:[0,1,2]}); | |
if (proc.status === null) throw 'Error while starting ' + cmd + '. Executable not found at path or no access.'; | |
if (proc.status !== 0) throw 'Error while running ' + cmd + '. An error should have been printed above.'; | |
} | |
function WaitProcs(procs, maxProcs) | |
{ | |
for (;;) //clear finished processes and wait while there are more than maxProcs running at the same | |
{ | |
for (var i = procs.length; i--;) | |
{ | |
if (procs[i].IsRunning()) continue; | |
procs[i].AbortIfError(); | |
procs.splice(i, 1); | |
} | |
if (procs.length <= (maxProcs|0)) return; | |
child_process.spawnSync(process.execPath,['-e','setTimeout(function(){},100)']); //sleep 100 ms | |
} | |
} | |
function RunAsync(cmd, args, outPath, procs, maxProcs) | |
{ | |
WaitProcs(procs, maxProcs - 1); | |
console.log('Running async: ' + cmd + ' ' + args.join(' ')); | |
var proc = child_process.spawn(cmd, args, {stdio:[0,1,2]}); | |
if (proc.pid === undefined) throw 'Error while starting ' + cmd + '. Executable not found at path or no access.'; | |
procs.push({ | |
AbortIfError: () => {if (!fs.existsSync(outPath)) throw 'Error while running ' + cmd + '. An error should have been printed above.'}, | |
IsRunning: () => {try{return process.kill(proc.pid,0)}catch{}}, | |
}); | |
} | |
// fake program that waits for a while (arg 1) then writes a file (arg 2) | |
var myCmd = process.execPath; | |
var myArgs = [ '-e', "setTimeout(function(){require('fs').writeFileSync(process.argv[2], 1)},process.argv[1])" ]; | |
// spawn 10 processes that take 2 seconds async, except the last one which takes .5 seconds and is spawned synchronous | |
var maxProcs = 4; //run a maximum of 4 processes at the same time | |
var procs = []; | |
for (var i = 0, iMax = 10; i != iMax; i++) | |
{ | |
var waitTime = (i == iMax - 1 ? 500 : 2000); | |
var outPath = "TEMP" + i + ".DAT"; | |
try { fs.unlinkSync(outPath); } catch (e) {} | |
(i == iMax - 1 ? RunSync : RunAsync)(myCmd, myArgs.concat([ waitTime, outPath ]), outPath, procs, maxProcs); | |
} | |
console.log("Waiting for remaining processes ..."); | |
WaitProcs(procs); | |
console.log("Done!"); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment