Run many operations in parallel on a single array. Example: downloading 10 files by 10 files from a 1000 urls input

runInParallel.js

const runInParallel = async (dataList, numberOfParallelRequest, runFunction) => {
    return new Promise((resolve) => {
        const isRunCompleted = (runStatus) => {
            return Object.values(runStatus).every(status => status === false)
        }

        const dataListSplit = splitArray(dataList, numberOfParallelRequest)
        console.log('dataListSplit', dataListSplit.length)

        const runStatus = {}
        dataListSplit.forEach(async (data, index) => {
            runStatus[index] = true
            const length = data.length
            let i = 1
            for (const dataEl of data) {
                await runFunction(dataEl)
                console.log(`Run: ${index}: ${i}/${length}`)
                i++
            }
            runStatus[index] = false

            console.log("- " + index + " ended")
            if (isRunCompleted(runStatus)) {
                resolve()
            }
        })
    })
}

function splitArray(dataList, numberOfParallelRequest) {
    const result = []
    const chunkSize = Math.ceil(dataList.length / numberOfParallelRequest)

    for (let i = 0; i < dataList.length; i += chunkSize) {
        result.push(dataList.slice(i, i + chunkSize))
    }

    return result
}

possible improvement :

type RunFunction<T, R> = (data: T | undefined) => Promise<R>;

export const runInParallel = async <T, R>(
  dataList: T[] = [],
  numberOfParallelRequest: number,
  runFunction: RunFunction<T, R>
): Promise<R[]> => {
  return new Promise((resolve, reject) => {
    const queue = [...dataList];
    const results = new Array(dataList.length);
    let completedCount = 0;

    const processNext = async (index: number) => {
      while (queue.length > 0) {
        const dataIndex = dataList.length - queue.length;
        const dataEl = queue.shift();
        try {
          results[dataIndex] = await runFunction(dataEl);
        } catch (error) {
          console.error(`Error in Run ${index}:`, error);
          results[dataIndex] = undefined;
        }

        completedCount++;
        if (completedCount === dataList.length) {
          resolve(results);
          return;
        }
      }
    };

    const parallelRequests = Math.min(numberOfParallelRequest, dataList.length);

    if(parallelRequests === 0) {
      resolve([]);
      console.log('No parallel requests to run');
      return;
    }
    console.log(`Running ${parallelRequests} parallel requests`);

    for (let i = 0; i < parallelRequests; i++) {
      processNext(i).catch(reject);
    }
  });
};