Last active
March 24, 2022 13:25
-
-
Save Kirill89/9f2484ebde36792d33d2d2eb421e48f6 to your computer and use it in GitHub Desktop.
Run JS promises in parallel with limited concurrency
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
// Usage example: | |
async function scrapeUrl() { | |
// ... | |
} | |
const urls = ['http://foo.com', 'http://bar.com', 'more URLs...']; | |
// Scrape all URLs with maximun 10 in parralel. | |
const scrapeUrlLimit = limit(10, scrapeUrl); | |
await Promise.all(urls.map(scrapeUrlLimit)); | |
*/ | |
import {setTimeout} from 'node:timers/promises'; | |
export function limit(max, fn, delay = 100) { | |
let cnt = 0; | |
return async function (...args) { | |
while (cnt >= max) { | |
await setTimeout(delay); | |
} | |
cnt++; | |
try { | |
return await fn(...args); | |
} finally { | |
cnt--; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment