Chunkable Promises

chunkablePromises.js

const _ = require('lodash')

/**
 * Async Waiting utility
 * @param  {Number} wait Number of MS to wait
 * @return {Promise}     A promise to await
 */
const waiter = async function(wait = 0) {
    return new Promise((resolve) => {
        if (wait <= 0) {
            return resolve()
        }
        setTimeout(() => {
            resolve()
        }, wait)
    })
}

/**
 * Map a collection that requries await/async
 * @param  {Array}   array    Array of items
 * @param  {Function} callback [description]
 * @return {Array}            Mapped results
 */
const asyncMap = async function(array, callback) {
    const map = []
    for (var index = 0; index < array.length; index++) {
        const value = await callback(array[index], index)
        map.push(value)
    }
    return map
}

/**
 * Chunk a Group of Requests and execute according to API Limitations
 * @param  {array<Function>} array an array of functions that return a thenable when called
 * @param  {Number} size     The Number of items per chunk
 * @param  {Number} throttle The Number of ms to wait until a chunk has completed
 *
 * Note: the idea is that if the api can handle 6 calls/second you can chunck a
 * batches of 6 api requests, that will wait up to 1000ms before the next chunk is called.
 *
 * @return {array}          Fulfilled data from the request.
 */
const chunkRequests = async function(array, size = 3, throttle = 1000) {
    const chunks = _.chunk(array, size)
    const total = chunks.length
    var count = 0

    return await asyncMap(chunks, async (chunk) => {
        console.log(
            `Processing Request Chunks ${++count} of ${total} (size ${size})`
        )

        const start = new Date().getTime()
        const results = await Promise.all(chunk.map((c) => c()))
        const end = new Date().getTime()
        const wait = start + throttle - end
        if (wait > 0) {
            await waiter(wait)
        }
        return results
    }).then(_.flatMap)
}

const Preq = {
    load: async function(item) {
        return new Promise((r) => {
            setTimeout(() => {
                r({ b: item.a })
            }, 2000)
        })
    },
}

const test = async function() {
    var items = new Array(17).fill().map((v, a) => {
        return { a }
    })

    const chunkables = items.map((request) => {
        return () => {
            return Preq.load(request).then(async (response) => {
                return { ...request, ...response }
            })
        }
    })
    const results = await chunkRequests(chunkables)
    console.log(results)
}

test()