Last active
September 13, 2021 18:08
-
-
Save DiegoVictor/ca1b6173e4ae050657f40d4bc40f0573 to your computer and use it in GitHub Desktop.
Implementing a pool of requests
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require("fs"); | |
const path = require("path"); | |
const axios = require("axios"); | |
const { pipeline } = require("stream/promises"); | |
const api = axios.create({ | |
baseURL: "https://pokeapi.co/api/v2/pokemon", | |
}); | |
const requestsInParallel = 3; | |
const limit = 10; | |
let processed = 0; | |
let id = 1; | |
const pool = []; | |
const getPokemonBy = async (id) => | |
api.get(`/${id}`).then(({ data }) => { | |
const { | |
name, | |
sprites: { | |
other: { | |
"official-artwork": { front_default: artwork }, | |
}, | |
}, | |
} = data; | |
const ext = path.extname(artwork); | |
return { | |
filename: `${name}${ext}`, | |
name, | |
artwork, | |
}; | |
}); | |
const downloadImage = async (url, filename) => | |
api | |
.get(url, { responseType: "stream" }) | |
.then(({ data: stream }) => | |
pipeline(stream, fs.createWriteStream(filename)) | |
); | |
const push = () => { | |
while (pool.length - processed < requestsInParallel && id <= limit) { | |
pool.push( | |
getPokemonBy(id) | |
.then(({ filename, name, artwork }) => | |
downloadImage(artwork, filename).then(() => name) | |
) | |
.then((name) => { | |
// delay the requests | |
return new Promise((resolve) => { | |
setTimeout(() => { | |
processed++; | |
console.log(`(${processed}/${limit}) ${name} processed`); | |
resolve(); | |
}, 500); | |
}); | |
}) | |
); | |
id++; | |
} | |
return Promise.all(pool); | |
}; | |
const run = async () => { | |
if (processed < pool.length || processed < limit) { | |
return push().then(run); | |
} | |
return pool; | |
}; | |
console.time("Pipeline took"); | |
run().then(() => { | |
console.timeEnd("Pipeline took"); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment