ky + pLimit + pThrottle

ky-limit-throttle.ts

import ky, { KyInstance } from 'ky';
import pLimit from 'p-limit';
import pThrottle from 'p-throttle';

/**
 * Creates a ky instance with concurrency limiting.
 *
 * @param concurrency Maximum number of simultaneous requests
 * @param options Optional ky configuration options
 * @returns A ky instance with concurrency limiting applied
 */
function limit(concurrency: number, options?: ky.Options): KyInstance {
  const limiter = pLimit(concurrency);

  return ky.create({
    ...options,
    hooks: {
      beforeRequest: [
        async () => {
          // Reserve a slot in the concurrency pool
          await limiter(async () => { });
        }
      ]
    }
  });
}

/**
 * Creates a ky instance with request throttling.
 *
 * @param limit Maximum number of requests allowed per interval
 * @param interval Time window in milliseconds
 * @param options Optional ky configuration options
 * @returns A ky instance with throttling applied
 */
function throttle(limit: number, interval: number, options?: ky.Options): KyInstance {
  const throttler = pThrottle({
    limit,
    interval
  });

  return ky.create({
    ...options,
    hooks: {
      beforeRequest: [
        async () => {
          // Wait for throttling to allow the request
          await throttler(async () => { })();
        }
      ]
    }
  });
}

export async function main() {
  const concurrency = 24;
  
  const scraper = limit(concurrency, {
    headers: { "X-API-KEY": 123 },
    timeout: 10000,
    retry: 2
  });
  
  const urls = [ ... ];
  
  const results = await Promise.allSettled(
    urls.map(async (url) => {
      // ... scrape here
    });
  );
}