Skip to content

Instantly share code, notes, and snippets.

@marsgpl
Last active March 22, 2024 15:30
Show Gist options
  • Save marsgpl/1d2a0364e24abfdf840e39eaf84d5213 to your computer and use it in GitHub Desktop.
Save marsgpl/1d2a0364e24abfdf840e39eaf84d5213 to your computer and use it in GitHub Desktop.
const fetch = require("node-fetch")
const fetchAll = async function(urls, max=3) {
const results = Array(urls.length)
const workers = Array(max)
const index = {value:0}
for (let i=0; i<max; ++i) {
workers.push(fetchAll.worker(urls, results, index))
}
await Promise.all(workers)
return results
}
fetchAll.worker = async function(urls, results, index) {
while (index.value < urls.length) {
const url = urls[index.value]
results[index.value++] = await fetch(url)
.then(r => r.json())
.catch(_ => null) // no need to keep error in this example
}
}
const needUrls = 60
const urls = Array.from(Array(needUrls), (_,i) =>
`https://jsonplaceholder.typicode.com/todos/${i+1}`)
fetchAll(urls).then(console.log)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment