Skip to content

Instantly share code, notes, and snippets.

@DiegoVictor
Last active September 13, 2021 18:08
Show Gist options
  • Save DiegoVictor/ca1b6173e4ae050657f40d4bc40f0573 to your computer and use it in GitHub Desktop.
Save DiegoVictor/ca1b6173e4ae050657f40d4bc40f0573 to your computer and use it in GitHub Desktop.
Implementing a pool of requests
const fs = require("fs");
const path = require("path");
const axios = require("axios");
const { pipeline } = require("stream/promises");
const api = axios.create({
baseURL: "https://pokeapi.co/api/v2/pokemon",
});
const requestsInParallel = 3;
const limit = 10;
let processed = 0;
let id = 1;
const pool = [];
const getPokemonBy = async (id) =>
api.get(`/${id}`).then(({ data }) => {
const {
name,
sprites: {
other: {
"official-artwork": { front_default: artwork },
},
},
} = data;
const ext = path.extname(artwork);
return {
filename: `${name}${ext}`,
name,
artwork,
};
});
const downloadImage = async (url, filename) =>
api
.get(url, { responseType: "stream" })
.then(({ data: stream }) =>
pipeline(stream, fs.createWriteStream(filename))
);
const push = () => {
while (pool.length - processed < requestsInParallel && id <= limit) {
pool.push(
getPokemonBy(id)
.then(({ filename, name, artwork }) =>
downloadImage(artwork, filename).then(() => name)
)
.then((name) => {
// delay the requests
return new Promise((resolve) => {
setTimeout(() => {
processed++;
console.log(`(${processed}/${limit}) ${name} processed`);
resolve();
}, 500);
});
})
);
id++;
}
return Promise.all(pool);
};
const run = async () => {
if (processed < pool.length || processed < limit) {
return push().then(run);
}
return pool;
};
console.time("Pipeline took");
run().then(() => {
console.timeEnd("Pipeline took");
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment