Fetch all urls as fast as possible, limited by concurrency

fetchAll.ts

/**
 * Fetch all urls as fast as possible.
 * Concurrency limits the amount of simultaneous window.fetch calls.
 * Return fetched results in preserved order.
 */
export async function fetchAll<T>(urls: string[], concurrency: number = 3): Promise<T[]> {
  const results: T[] = [];
  const pool: Set<Promise<T>> = new Set();

  let next: () => void = () => {};
  let _finish: () => void = () => {};

  const done = new Promise<void>((resolve) => {
    _finish = resolve;
  });

  const waitForSlot = () =>
    new Promise<void>((resolve) => {
      next = resolve;
    });

  async function* queue() {
    for (let index = 0; index < urls.length; ) {
      if (pool.size < concurrency) {
        yield [urls[index], index] as const;
        index++;
      } else {
        console.log("Waiting for open slot");
        await waitForSlot();
      }
    }
    console.log("All requests queued");
    next = () => {
      if (pool.size === 0) {
        console.log("All requests finished!");
        _finish();
      }
    };
  }

  for await (const [url, index] of queue()) {
    console.log(`loading ${url}`);
    const request = fetch(url).then<T>((response) => response.json());
    pool.add(request);
    console.time(`loaded ${url}`);
    request.then((result) => {
      console.timeEnd(`loaded ${url}`);
      results[index] = result;
      pool.delete(request);
      next();
    });
  }

  await done;
  return results;
}