Skip to content

Instantly share code, notes, and snippets.

@tarekeldeeb
Last active March 22, 2020 12:17
Show Gist options
  • Save tarekeldeeb/e100654a5cd027b7fad185fc71a4e523 to your computer and use it in GitHub Desktop.
Save tarekeldeeb/e100654a5cd027b7fad185fc71a4e523 to your computer and use it in GitHub Desktop.
Javascript Apply thread throttle to fetch() of URL Array without any library
/*
* Adapted from: https://stackoverflow.com/questions/38385419/throttle-amount-of-promises-open-at-a-given-time
*/
/**
* Performs a list of callable fetch actions (promise factories) so that only a limited
* number of promises are pending at any given time.
*
* @param listOfCallableActions An array of callable functions, which should return promises.
* @param limit The maximum number of promises to have pending at once.
* @returns A Promise that resolves to the full list of values when everything is done.
*/
function throttleActions(listOfCallableActions, limit) {
// We'll need to store which is the next promise in the list.
let i = 0;
let resultArray = new Array(listOfCallableActions.length);
// Now define what happens when any of the actions completes. Javascript is
// (mostly) single-threaded, so only one completion handler will call at a
// given time. Because we return doNextAction, the Promise chain continues as
// long as there's an action left in the list.
function doNextAction() {
if (i < listOfCallableActions.length) {
// Save the current value of i, so we can put the result in the right place
let actionIndex = i++;
let nextAction = listOfCallableActions[actionIndex];
return Promise.resolve(nextAction())
.then(result => result.json())
.then(result => { // Save results to the correct array index.
resultArray[actionIndex] = result;
return;
}).then(doNextAction);
}
}
// Now start up the original <limit> number of promises.
// i advances in calls to doNextAction.
let listOfPromises = [];
while (i < limit && i < listOfCallableActions.length) {
listOfPromises.push(doNextAction());
}
return Promise.all(listOfPromises).then(() => resultArray);
}
// Test with an array of filenames to be replaced in URL
var ids = ['file850382', 'file831811', 'file855848', 'file838175', 'file853825', 'file328840', 'file807150', 'file676344', 'file683950', 'file429605', 'file787922', 'file843038', 'file818275', 'file616670', 'file723279', 'file616665', 'file616399'];
var funz = []; // An array of callable Functions that return a Promise
ids.forEach(function(id){funz.push(() => fetch('https://my-url.com/v1/objects/'+id+'/download?offset=0'))}); //Headers may be added!
var d = new Date(); throttleActions(funz, 3).then(result => {console.log("Completed in [ms] "+(new Date() - d));console.log(result)});
// Changing the throttle of 3 threads will affect the time
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment