retrieve urls in batches

url-batch-javascript

// This is a mock request function
const sendRequest = (url) => {
  return new Promise((resolve) => {
    setTimeout(() => {
      resolve(`request resolved ${url}`)
    }, 2000)
  })
}

// Helper: split array in batches
const splitInBatch = (arr, batchSize) => {
  return arr.reduce((accumulator, element, index) => {
    const batchIndex = Math.floor(index / batchSize);
    if (Array.isArray(accumulator[batchIndex])) {
      accumulator[batchIndex].push(element);
    } else {
      accumulator.push([element]);
    }
    return accumulator;
  }, []);
}

// App
const batchSize = 3;
const urls = [1, 2, 3, 4, 5, 6, 7];
const batches = splitInBatch(urls, batchSize);

const asyncForEach = async (array, callback) =>  {
  for (let index = 0; index < array.length; index++) {
    await callback(array[index], index, array);
  }
}

;(async function() {
  asyncForEach(batches, async (batch) => {
      console.log('--- sending batch ---')
      await Promise.all(batch.map(async(url) => {
        const response = await sendRequest(url);
        console.log(`✔ response`)
        return response;
      }))  

  });
})();

https://www.npmjs.com/package/wait-on