Last active
January 24, 2024 02:11
-
-
Save kevcodez/b0a105ba4d5cf9e2be32e5e98e51b419 to your computer and use it in GitHub Desktop.
Native Node.js HTTP Client with retries, proxy support, timeouts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { ProxyAgent } from 'undici'; | |
const SAFE_HTTP_METHODS = ['GET', 'HEAD', 'OPTIONS']; | |
const IDEMPOTENT_HTTP_METHODS = SAFE_HTTP_METHODS.concat(['PUT', 'DELETE']); | |
const HTTP_STATUS_TO_RETRY = [408, 429, 500, 501, 502, 503, 504]; | |
const DEFAULT_TIMEOUT = 20_000; | |
const retryDenyList = new Set([ | |
'ENOTFOUND', | |
'ENETUNREACH', | |
// SSL errors from https://github.com/nodejs/node/blob/fc8e3e2cdc521978351de257030db0076d79e0ab/src/crypto/crypto_common.cc#L301-L328 | |
'UNABLE_TO_GET_ISSUER_CERT', | |
'UNABLE_TO_GET_CRL', | |
'UNABLE_TO_DECRYPT_CERT_SIGNATURE', | |
'UNABLE_TO_DECRYPT_CRL_SIGNATURE', | |
'UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY', | |
'CERT_SIGNATURE_FAILURE', | |
'CRL_SIGNATURE_FAILURE', | |
'CERT_NOT_YET_VALID', | |
'CERT_HAS_EXPIRED', | |
'CRL_NOT_YET_VALID', | |
'CRL_HAS_EXPIRED', | |
'ERROR_IN_CERT_NOT_BEFORE_FIELD', | |
'ERROR_IN_CERT_NOT_AFTER_FIELD', | |
'ERROR_IN_CRL_LAST_UPDATE_FIELD', | |
'ERROR_IN_CRL_NEXT_UPDATE_FIELD', | |
'OUT_OF_MEM', | |
'DEPTH_ZERO_SELF_SIGNED_CERT', | |
'SELF_SIGNED_CERT_IN_CHAIN', | |
'UNABLE_TO_GET_ISSUER_CERT_LOCALLY', | |
'UNABLE_TO_VERIFY_LEAF_SIGNATURE', | |
'CERT_CHAIN_TOO_LONG', | |
'CERT_REVOKED', | |
'INVALID_CA', | |
'PATH_LENGTH_EXCEEDED', | |
'INVALID_PURPOSE', | |
'CERT_UNTRUSTED', | |
'CERT_REJECTED', | |
'HOSTNAME_MISMATCH', | |
]); | |
export type HttpProxy = | |
| string | |
| { | |
host: string; | |
port: number; | |
auth: { | |
username: string; | |
password: string; | |
}; | |
}; | |
type HttpClientConfig = { | |
origin: string; | |
timeout?: number; | |
auth?: { | |
username?: string; | |
password?: string; | |
}; | |
headers?: Record<string, any>; | |
params?: Record<string, string | number | boolean> | URLSearchParams; | |
proxy?: HttpProxy | false; | |
}; | |
type UserRequestOptions = { | |
headers?: Record<string, string>; | |
params?: Record<string, string | number | boolean> | URLSearchParams; | |
timeout?: number; | |
}; | |
export class HttpClientException extends Error { | |
response: globalThis.Response | null; | |
request: globalThis.RequestInit | null; | |
code: string | null; | |
constructor(response: globalThis.Response | null, request: globalThis.RequestInit | null, cause?: any) { | |
const status = response?.status; | |
super(cause?.message || `HTTP Exception: Status ${status}`); | |
this.cause = cause; | |
this.response = response; | |
this.request = request; | |
this.code = cause?.code || null; | |
} | |
} | |
export class HttpClient { | |
constructor(private config?: HttpClientConfig) {} | |
async post(url: string, body?: any, options?: UserRequestOptions) { | |
return this.request('POST', url, body, options); | |
} | |
async put(url: string, body: any, options?: UserRequestOptions) { | |
return this.request('PUT', url, body, options); | |
} | |
async get(url: string, options?: UserRequestOptions) { | |
return this.request('GET', url, undefined, options); | |
} | |
async delete(url: string, options?: UserRequestOptions) { | |
return this.request('DELETE', url, undefined, options); | |
} | |
async request( | |
method: string, | |
url: string, | |
body: any, | |
// eslint-disable-next-line @typescript-eslint/no-unused-vars | |
options?: UserRequestOptions, | |
): Promise<globalThis.Response> { | |
let attempts = 0; | |
const headers = this.mergeHeaders(options, body); | |
let dispatcher; | |
if (this.config?.proxy) { | |
const { uri, username, password } = this.getProxy(); | |
dispatcher = new ProxyAgent({ | |
uri, | |
token: 'Basic ' + Buffer.from(username + ':' + password).toString('base64'), | |
}); | |
} | |
const request: RequestInit = { | |
method, | |
body: body ? JSON.stringify(body) : body, | |
headers, | |
// @ts-ignore this actually exists | |
dispatcher, | |
}; | |
const makeRequest = (): Promise<globalThis.Response> => { | |
const urlParams = new URLSearchParams(); | |
const configParams = this.config?.params; | |
if (configParams) { | |
if (configParams instanceof URLSearchParams) { | |
Array.from(configParams.entries()).forEach(([key, value]) => urlParams.append(key, value)); | |
} else { | |
Object.keys(configParams).forEach((key) => { | |
urlParams.set(key, configParams[key].toString()); | |
}); | |
} | |
} | |
const requestParams = options?.params; | |
if (requestParams) { | |
if (requestParams instanceof URLSearchParams) { | |
Array.from(requestParams.entries()).forEach(([key, value]) => urlParams.append(key, value)); | |
} else { | |
Object.keys(requestParams).forEach((key) => { | |
urlParams.set(key, requestParams[key].toString()); | |
}); | |
} | |
} | |
const urlParamsToString = Array.from(urlParams.values()).length ? '?' + urlParams.toString() : ''; | |
const fullUrl = new URL(url + urlParamsToString, this.config?.origin); | |
return fetch(fullUrl.toString(), request); | |
}; | |
let response: globalThis.Response | null = null; | |
let error: HttpClientException | null = null; | |
while (attempts < this.maxRetries() + 1) { | |
const sentAt = new Date(); | |
try { | |
response = await withTimeout(options?.timeout || this.config?.timeout || DEFAULT_TIMEOUT, makeRequest()); | |
if (!response || !response.ok) { | |
throw new HttpClientException(response, request); | |
} | |
break; | |
} catch (err: any) { | |
error = err instanceof HttpClientException ? err : new HttpClientException(response, request, err); | |
this.logError(err, url, request, attempts, sentAt); | |
if (this.isRetryableError(error)) { | |
await this.sleep(this.retryDelay(error)); | |
attempts++; | |
response = null; | |
} else { | |
throw error; | |
} | |
} | |
} | |
if (response) { | |
return response; | |
} else { | |
throw error; | |
} | |
} | |
private getProxy(): { uri: string; username: string; password: string } { | |
let uri, username, password; | |
const proxy = this.config!.proxy as HttpProxy; | |
if (typeof proxy === 'string') { | |
const split = proxy.split('@'); | |
uri = `http://${split[1]}`; | |
const auth = split[0].replace('http://', '').split(':'); | |
username = auth[0]; | |
password = auth[1]; | |
} else { | |
uri = `http://${proxy.host}:${proxy.port}`; | |
username = proxy.auth.username; | |
password = proxy.auth.password; | |
} | |
return { uri, username, password }; | |
} | |
mergeHeaders(options?: UserRequestOptions, body?: any): Record<string, string> { | |
const headers = this.config?.headers || {}; | |
if (this.config?.auth) { | |
headers['Authorization'] = | |
'Basic ' + Buffer.from(this.config.auth.username + ':' + this.config.auth.password).toString('base64'); | |
} | |
if (body) { | |
headers['Content-Type'] = 'application/json'; | |
} | |
if (options?.headers) { | |
for (const [key, val] of Object.entries(options.headers)) { | |
headers[key] = val; | |
} | |
} | |
return headers; | |
} | |
private isRetryableError(err: HttpClientException) { | |
if (err.response?.status === 500) return false; | |
if ( | |
!err.response && | |
Boolean(err.code) && // Prevents retrying cancelled requests | |
err.code !== 'ECONNABORTED' | |
) { | |
return true; | |
} | |
return this.isIdempotentRequestError(err) || (err?.code && !retryDenyList.has(err.code)); | |
} | |
private isIdempotentRequestError(err: HttpClientException): boolean { | |
const response = err.response; | |
const request = err.request; | |
if (!response || !request) return false; | |
return ( | |
err.code !== 'ECONNABORTED' && | |
HTTP_STATUS_TO_RETRY.includes(response.status) && | |
IDEMPOTENT_HTTP_METHODS.includes(request.method!) | |
); | |
} | |
private retryDelay(err: HttpClientException) { | |
if (err.response?.status === 429) { | |
return 1000; | |
} else { | |
return 100; | |
} | |
} | |
private async sleep(ms: number) { | |
await new Promise((r) => setTimeout(r, ms)); | |
} | |
private logError(error: HttpClientException, url: string, request: RequestInit, retries: number, sentAt: Date) { | |
let configParameters = {}; | |
const { origin } = this.config || {}; | |
const { method } = request; | |
const responseTime = +Date.now() - +sentAt; | |
configParameters = { baseURL: origin, url, method, retries, responseTime }; | |
const errorToLog = { | |
message: `Got HTTP Error - ${error.message}`, | |
status: error.response?.status || -1, | |
...configParameters, | |
}; | |
console.info(errorToLog, errorToLog.message); | |
} | |
private maxRetries() { | |
return 1; | |
} | |
} | |
/** | |
* Creates a new http client instance. Use this if you want to pass a default config that will | |
* be used on all requests made by the created instance. A good example would be a | |
* common base Url or Authorization headers that are needed for every request. | |
*/ | |
export function createHttpClient(config?: HttpClientConfig): HttpClient { | |
return new HttpClient(config); | |
} | |
const withTimeout = (millis: number, promise: Promise<any>): Promise<any> => { | |
return new Promise((resolve, reject) => { | |
const timer = setTimeout(() => { | |
reject(new Error(`Timed out after ${millis} ms.`)); | |
}, millis); | |
promise | |
.then((value) => { | |
clearTimeout(timer); | |
resolve(value); | |
}) | |
.catch((reason) => { | |
clearTimeout(timer); | |
reject(reason); | |
}); | |
}); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment