Run a series of async batch jobs

promise-batch.js

/**
 * Run a series of async functions in "batches",
 * so that `batchSize` number of fns are running concurrently
 *
 * @param {function():Promise<T>[]} fns
 * @param {int} batchSize
 * @return {Promise<T[]>}
 */
function batch(fns, batchSize) {
  const batchFns = _.chunk(fns, batchSize)
    .map(fnsInBatch =>
      () => Promise.all(fnsInBatch.map(fn => fn()))
    );

  // https://gist.github.com/eschwartz/565e92418d903b7e98dd7bb9679293c1
  return sequence(batchFns)
    // flatten
    .then(batchResults => batchResults.reduce((resList, res) => resList.concat(res), []))
}

promise-batch.ts

import sequence from './sequence';
import {IAsyncFn} from './sequence';

/**
 * Run a series of async functions in "batches",
 * so that `batchSize` number of fns are running concurrently
 */
function batch<TRes>(fns:IAsyncFn<TRes>[], batchSize:number):Promise<TRes[]> {
  const batchFns = chunk<IAsyncFn<TRes>>(fns, batchSize)
    .map(fnsInBatch =>
      () => Promise.all(fnsInBatch.map(fn => fn()))
    );

  return sequence<TRes[]>(batchFns)
    // flatten
    .then(batchResults => batchResults.reduce((resList, res) => resList.concat(res), []))
}

function chunk<TVal>(list:TVal[], chunkSize:number):TVal[][] {
  return list.reduce((chunks, item, i) => {
    const chunkNumber = Math.floor(i / chunkSize);
    chunks[chunkNumber] || (chunks[chunkNumber] = []);

    chunks[chunkNumber].push(item);

    return chunks;
  }, [])
}

export default batch;

eg

const delayed = (res, ms) => new Promise(done => {
  console.log(`starting ${res}`)
  setTimeout(() => { 
    console.log(`Finished ${res}`); done(res); 
  }, ms);
});

const batchA = yield batch([
  () => delayed('A', 500),
  () => delayed('B', 100),
  () => delayed('C', 500),
  () => delayed('D', 100),
], 2)
  .then(batch => {})  // ['A', 'B', 'C', 'D']

/* 
Log output:
starting A
starting B
Finished B
Finished A
starting C
starting D
Finished D
Finished C
*/

Note that it ran A and B in parallel, and waiting for them to complete for running C and D