Native Node.js HTTP Client with retries, proxy support, timeouts

httpClient.ts

import { ProxyAgent } from 'undici';

const SAFE_HTTP_METHODS = ['GET', 'HEAD', 'OPTIONS'];
const IDEMPOTENT_HTTP_METHODS = SAFE_HTTP_METHODS.concat(['PUT', 'DELETE']);
const HTTP_STATUS_TO_RETRY = [408, 429, 500, 501, 502, 503, 504];

const DEFAULT_TIMEOUT = 20_000;

const retryDenyList = new Set([
  'ENOTFOUND',
  'ENETUNREACH',

  // SSL errors from https://github.com/nodejs/node/blob/fc8e3e2cdc521978351de257030db0076d79e0ab/src/crypto/crypto_common.cc#L301-L328
  'UNABLE_TO_GET_ISSUER_CERT',
  'UNABLE_TO_GET_CRL',
  'UNABLE_TO_DECRYPT_CERT_SIGNATURE',
  'UNABLE_TO_DECRYPT_CRL_SIGNATURE',
  'UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY',
  'CERT_SIGNATURE_FAILURE',
  'CRL_SIGNATURE_FAILURE',
  'CERT_NOT_YET_VALID',
  'CERT_HAS_EXPIRED',
  'CRL_NOT_YET_VALID',
  'CRL_HAS_EXPIRED',
  'ERROR_IN_CERT_NOT_BEFORE_FIELD',
  'ERROR_IN_CERT_NOT_AFTER_FIELD',
  'ERROR_IN_CRL_LAST_UPDATE_FIELD',
  'ERROR_IN_CRL_NEXT_UPDATE_FIELD',
  'OUT_OF_MEM',
  'DEPTH_ZERO_SELF_SIGNED_CERT',
  'SELF_SIGNED_CERT_IN_CHAIN',
  'UNABLE_TO_GET_ISSUER_CERT_LOCALLY',
  'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
  'CERT_CHAIN_TOO_LONG',
  'CERT_REVOKED',
  'INVALID_CA',
  'PATH_LENGTH_EXCEEDED',
  'INVALID_PURPOSE',
  'CERT_UNTRUSTED',
  'CERT_REJECTED',
  'HOSTNAME_MISMATCH',
]);

export type HttpProxy =
  | string
  | {
      host: string;
      port: number;
      auth: {
        username: string;
        password: string;
      };
    };

type HttpClientConfig = {
  origin: string;
  timeout?: number;
  auth?: {
    username?: string;
    password?: string;
  };
  headers?: Record<string, any>;
  params?: Record<string, string | number | boolean> | URLSearchParams;
  proxy?: HttpProxy | false;
};

type UserRequestOptions = {
  headers?: Record<string, string>;
  params?: Record<string, string | number | boolean> | URLSearchParams;
  timeout?: number;
};

export class HttpClientException extends Error {
  response: globalThis.Response | null;
  request: globalThis.RequestInit | null;
  code: string | null;

  constructor(response: globalThis.Response | null, request: globalThis.RequestInit | null, cause?: any) {
    const status = response?.status;
    super(cause?.message || `HTTP Exception: Status ${status}`);
    this.cause = cause;

    this.response = response;
    this.request = request;
    this.code = cause?.code || null;
  }
}

export class HttpClient {
  constructor(private config?: HttpClientConfig) {}

  async post(url: string, body?: any, options?: UserRequestOptions) {
    return this.request('POST', url, body, options);
  }

  async put(url: string, body: any, options?: UserRequestOptions) {
    return this.request('PUT', url, body, options);
  }

  async get(url: string, options?: UserRequestOptions) {
    return this.request('GET', url, undefined, options);
  }

  async delete(url: string, options?: UserRequestOptions) {
    return this.request('DELETE', url, undefined, options);
  }

  async request(
    method: string,
    url: string,
    body: any,
    // eslint-disable-next-line @typescript-eslint/no-unused-vars
    options?: UserRequestOptions,
  ): Promise<globalThis.Response> {
    let attempts = 0;

    const headers = this.mergeHeaders(options, body);

    let dispatcher;

    if (this.config?.proxy) {
      const { uri, username, password } = this.getProxy();

      dispatcher = new ProxyAgent({
        uri,
        token: 'Basic ' + Buffer.from(username + ':' + password).toString('base64'),
      });
    }

    const request: RequestInit = {
      method,
      body: body ? JSON.stringify(body) : body,
      headers,
      // @ts-ignore this actually exists
      dispatcher,
    };

    const makeRequest = (): Promise<globalThis.Response> => {
      const urlParams = new URLSearchParams();

      const configParams = this.config?.params;
      if (configParams) {
        if (configParams instanceof URLSearchParams) {
          Array.from(configParams.entries()).forEach(([key, value]) => urlParams.append(key, value));
        } else {
          Object.keys(configParams).forEach((key) => {
            urlParams.set(key, configParams[key].toString());
          });
        }
      }

      const requestParams = options?.params;
      if (requestParams) {
        if (requestParams instanceof URLSearchParams) {
          Array.from(requestParams.entries()).forEach(([key, value]) => urlParams.append(key, value));
        } else {
          Object.keys(requestParams).forEach((key) => {
            urlParams.set(key, requestParams[key].toString());
          });
        }
      }

      const urlParamsToString = Array.from(urlParams.values()).length ? '?' + urlParams.toString() : '';

      const fullUrl = new URL(url + urlParamsToString, this.config?.origin);

      return fetch(fullUrl.toString(), request);
    };

    let response: globalThis.Response | null = null;
    let error: HttpClientException | null = null;

    while (attempts < this.maxRetries() + 1) {
      const sentAt = new Date();
      try {
        response = await withTimeout(options?.timeout || this.config?.timeout || DEFAULT_TIMEOUT, makeRequest());

        if (!response || !response.ok) {
          throw new HttpClientException(response, request);
        }

        break;
      } catch (err: any) {
        error = err instanceof HttpClientException ? err : new HttpClientException(response, request, err);

        this.logError(err, url, request, attempts, sentAt);

        if (this.isRetryableError(error)) {
          await this.sleep(this.retryDelay(error));
          attempts++;
          response = null;
        } else {
          throw error;
        }
      }
    }

    if (response) {
      return response;
    } else {
      throw error;
    }
  }

  private getProxy(): { uri: string; username: string; password: string } {
    let uri, username, password;
    const proxy = this.config!.proxy as HttpProxy;

    if (typeof proxy === 'string') {
      const split = proxy.split('@');

      uri = `http://${split[1]}`;
      const auth = split[0].replace('http://', '').split(':');
      username = auth[0];
      password = auth[1];
    } else {
      uri = `http://${proxy.host}:${proxy.port}`;
      username = proxy.auth.username;
      password = proxy.auth.password;
    }

    return { uri, username, password };
  }

  mergeHeaders(options?: UserRequestOptions, body?: any): Record<string, string> {
    const headers = this.config?.headers || {};

    if (this.config?.auth) {
      headers['Authorization'] =
        'Basic ' + Buffer.from(this.config.auth.username + ':' + this.config.auth.password).toString('base64');
    }

    if (body) {
      headers['Content-Type'] = 'application/json';
    }

    if (options?.headers) {
      for (const [key, val] of Object.entries(options.headers)) {
        headers[key] = val;
      }
    }

    return headers;
  }

  private isRetryableError(err: HttpClientException) {
    if (err.response?.status === 500) return false;

    if (
      !err.response &&
      Boolean(err.code) && // Prevents retrying cancelled requests
      err.code !== 'ECONNABORTED'
    ) {
      return true;
    }

    return this.isIdempotentRequestError(err) || (err?.code && !retryDenyList.has(err.code));
  }

  private isIdempotentRequestError(err: HttpClientException): boolean {
    const response = err.response;
    const request = err.request;
    if (!response || !request) return false;

    return (
      err.code !== 'ECONNABORTED' &&
      HTTP_STATUS_TO_RETRY.includes(response.status) &&
      IDEMPOTENT_HTTP_METHODS.includes(request.method!)
    );
  }

  private retryDelay(err: HttpClientException) {
    if (err.response?.status === 429) {
      return 1000;
    } else {
      return 100;
    }
  }

  private async sleep(ms: number) {
    await new Promise((r) => setTimeout(r, ms));
  }

  private logError(error: HttpClientException, url: string, request: RequestInit, retries: number, sentAt: Date) {
    let configParameters = {};
    const { origin } = this.config || {};
    const { method } = request;
    const responseTime = +Date.now() - +sentAt;

    configParameters = { baseURL: origin, url, method, retries, responseTime };

    const errorToLog = {
      message: `Got HTTP Error - ${error.message}`,
      status: error.response?.status || -1,
      ...configParameters,
    };

    console.info(errorToLog, errorToLog.message);
  }

  private maxRetries() {
    return 1;
  }
}

/**
 * Creates a new http client instance. Use this if you want to pass a default config that will
 * be used on all requests made by the created instance. A good example would be a
 * common base Url or Authorization headers that are needed for every request.
 */
export function createHttpClient(config?: HttpClientConfig): HttpClient {
  return new HttpClient(config);
}

const withTimeout = (millis: number, promise: Promise<any>): Promise<any> => {
  return new Promise((resolve, reject) => {
    const timer = setTimeout(() => {
      reject(new Error(`Timed out after ${millis} ms.`));
    }, millis);

    promise
      .then((value) => {
        clearTimeout(timer);
        resolve(value);
      })
      .catch((reason) => {
        clearTimeout(timer);
        reject(reason);
      });
  });
};