Last active
March 22, 2020 12:17
-
-
Save tarekeldeeb/e100654a5cd027b7fad185fc71a4e523 to your computer and use it in GitHub Desktop.
Javascript Apply thread throttle to fetch() of URL Array without any library
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* Adapted from: https://stackoverflow.com/questions/38385419/throttle-amount-of-promises-open-at-a-given-time | |
*/ | |
/** | |
* Performs a list of callable fetch actions (promise factories) so that only a limited | |
* number of promises are pending at any given time. | |
* | |
* @param listOfCallableActions An array of callable functions, which should return promises. | |
* @param limit The maximum number of promises to have pending at once. | |
* @returns A Promise that resolves to the full list of values when everything is done. | |
*/ | |
function throttleActions(listOfCallableActions, limit) { | |
// We'll need to store which is the next promise in the list. | |
let i = 0; | |
let resultArray = new Array(listOfCallableActions.length); | |
// Now define what happens when any of the actions completes. Javascript is | |
// (mostly) single-threaded, so only one completion handler will call at a | |
// given time. Because we return doNextAction, the Promise chain continues as | |
// long as there's an action left in the list. | |
function doNextAction() { | |
if (i < listOfCallableActions.length) { | |
// Save the current value of i, so we can put the result in the right place | |
let actionIndex = i++; | |
let nextAction = listOfCallableActions[actionIndex]; | |
return Promise.resolve(nextAction()) | |
.then(result => result.json()) | |
.then(result => { // Save results to the correct array index. | |
resultArray[actionIndex] = result; | |
return; | |
}).then(doNextAction); | |
} | |
} | |
// Now start up the original <limit> number of promises. | |
// i advances in calls to doNextAction. | |
let listOfPromises = []; | |
while (i < limit && i < listOfCallableActions.length) { | |
listOfPromises.push(doNextAction()); | |
} | |
return Promise.all(listOfPromises).then(() => resultArray); | |
} | |
// Test with an array of filenames to be replaced in URL | |
var ids = ['file850382', 'file831811', 'file855848', 'file838175', 'file853825', 'file328840', 'file807150', 'file676344', 'file683950', 'file429605', 'file787922', 'file843038', 'file818275', 'file616670', 'file723279', 'file616665', 'file616399']; | |
var funz = []; // An array of callable Functions that return a Promise | |
ids.forEach(function(id){funz.push(() => fetch('https://my-url.com/v1/objects/'+id+'/download?offset=0'))}); //Headers may be added! | |
var d = new Date(); throttleActions(funz, 3).then(result => {console.log("Completed in [ms] "+(new Date() - d));console.log(result)}); | |
// Changing the throttle of 3 threads will affect the time |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment