Last active
July 10, 2024 09:34
-
-
Save unicornist/bd1c85a655e548291ca78477ebdeba77 to your computer and use it in GitHub Desktop.
Limited parallel async operations (aka Queue/Workers) in JavaScript ES6 (most compact code - essential parts only)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const queueSize = 10 | |
const parallelWorkerCount = 3 | |
const sampleQueue = ','.repeat(queueSize).split(',').map(()=>Math.floor(Math.random()*1000)) | |
console.log('sampleQueue:', sampleQueue) | |
const queueGenerator = (function *(){ | |
for (const item of sampleQueue) yield item; | |
})() | |
// Demonstrate how picking a work from the queue works using the queueGenerator | |
console.log('pick a work:', queueGenerator.next().value) | |
console.log('pick a work:', queueGenerator.next().value) | |
console.log('pick a work:', queueGenerator.next().value) | |
const sampleAsyncOp = (job, ms) => new Promise( (res, rej) => setTimeout(()=>res(job), ms) ) | |
const workers = ','.repeat(parallelWorkerCount).split(',').map((v,i)=>i+1) | |
const allAsyncOps = [] | |
let queueEndReached = false; | |
workers.forEach( async worker => { | |
console.log("Worker", worker, "started!") | |
for (const job of queueGenerator) { | |
const jobTime = Math.random()*6000+500; | |
console.log("Worker", worker, "PICKED Job", job, `(takes ${Math.floor(jobTime/100)/10}s)`) | |
const asyncOp = sampleAsyncOp(job, jobTime) | |
allAsyncOps.push(asyncOp) | |
const jobResult = await asyncOp | |
console.log("Worker", worker, "FINISHED Job", job, `(confirmed ${jobResult==job})`) | |
} | |
console.log("Worker", worker, "has no job!") | |
if (!queueEndReached) { | |
console.log("First jobless worker. End of Queue Reached! Waiting for all to finish.") | |
Promise.all(allAsyncOps).then(results => console.log("ALL FINISHED!!!", results)) | |
queueEndReached = true; | |
} | |
}) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const parallelWorkers = [....] | |
const jobs = [....] | |
const queueGenerator = (function* () { | |
for (const job of jobs) yield job | |
})() | |
parallelWorkers.forEach(worker => { | |
for (const nextUnpickedJob of queueGenerator) { | |
await assignTask(worker, nextUnpickedJob) | |
} | |
}) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment