For getting working proxies and send requests by circulating proxies

fetchHandler.js

import fetch from "node-fetch";
import { HttpsProxyAgent } from "https-proxy-agent";
import { ProxieProvider } from "./proxies.js";
import logger from "./logger.js";


export async function fetchHandler(target, options = {}, proxy = {}, retry = 0) {
  // fetch handeler  -
  //  retry with different proxy and with no proxy
  //  log the request and response
  //  make a fetch request with proxy if provided

  if (retry < 1) {
    logger.log('Requesting:', target);
    // get new proxy and assign to options
  } else if (retry === 1) {
    logger.log('Retry :', retry, target);
    // retry with new proxy or first proxy
    var proxy = await (new ProxieProvider()).getOneWorkingProxy();
    logger.log('using new proxy:', proxy);
  } else if (retry === 2) {
    logger.log('Retry :', retry, target, 'without proxy');
    // retry with without proxy 
  } else {
    logger.error('Retry at limti :', retry, target);
    return
  }

  // make a fetch request with proxy if provided
  try {
    var res = await fetchWithProxy(target, options, proxy);
    if (res.status === 200) {
      console.log(await res.text());

      return res;
    } else {
      // retry the request with different proxy
      if (retry === 2) {
        throw new Error('Max retries exceeded', res.status, res.statusText);
      }
      return fetchHandler(target, options, proxy, ++retry);
    }
  } catch (error) {
    logger.log('[logger] request fails :', error);
    // retry the request without proxys
    if (retry === 2) {
      throw new Error(error);
    }
    return fetchHandler(target, options, proxy, ++retry);
  }
}

export async function fetchWithProxy(
  target,
  options = {},
  proxy = {},
) {
  var response;

  try {
    // logger.log('working with',proxy);
    if (proxy.ip && proxy.port) {
      // Create a new Proxy Agent
      const proxyUrl = `http://${proxy.ip}:${proxy.port}`;
      const proxyAgent = new HttpsProxyAgent(proxyUrl);
      response = fetchWithTimeout(target, { agent: proxyAgent, ...options });
    } else {
      response = fetchWithTimeout(target, options);
    }
    return await response
  } catch (error) {
    return error;
  }
}

export const fetchWithTimeout = async (url, options = {}, ms = 5000) => {
  const controller = new AbortController();
  const signal = controller.signal;

  setTimeout(() => {
    controller.abort();
  }, ms);

  try {
    return await fetch(url, { signal, ...options });
  } catch (error) {
    // if (error.name === 'AbortError') {
    //   logger.log('request was aborted');
    // } else {
    //   logger.log(error);
    // }
    return error
  }
}

index.js

import { fetchHandler } from "./fetchHandler.js";

(async () => {
  const targetUrl = "https://ttpbin.org/ip";
  try {
    const response = await fetchHandler(targetUrl, {
      method: 'GET',
      headers: {
        'Content-Type': 'application/json'
      }
    })
    console.log(await response.json());
  } catch (error) {
    console.log(error);
  }
})()

proxies.js

import cheerio from "cheerio";
import fetch from "node-fetch";
import logger from "./logger.js";
import { fetchWithProxy } from "./fetchHandler.js";


export class ProxieProvider {
	constructor() {
		this.proxies = [];
		this.newProxiesFetchFailedLimit = 5;
		this.newProxiesFetchFailedRetry = 0;

		// singleton pattern instance implemented 
		if (ProxieProvider._instance) {
			logger.log("ProxieProvider instance already exists, returning the same instance.");
			return ProxieProvider._instance
		}
		ProxieProvider._instance = this;
	}

	init = async () => {
		if (this.newProxiesFetchFailedRetry === this.newProxiesFetchFailedLimit) {
			this.proxies = []
			return
		}
		// check if the proxy is working
		try {
			const data = await this.filterAndSortProxies();

			if (!data.length) {
				this.newProxiesFetchFailedRetry++;
			}
			// logger.log(data);
			this.proxies = data;
		} catch (error) {
			throw new Error("Error filtering and sorting proxies:", error);
		}
	}

	getAllProxies = () => {
		return this.proxies;
	}

	// function to circulate the proxy and test it
	getOneWorkingProxy = async () => {
		// get the first proxy
		const proxy = this.proxies.shift();
		// if no proxy then init the proxies again
		if (!proxy) {
			await this.init();
			if (this.proxies.length === 0 && this.newProxiesFetchFailedRetry === this.newProxiesFetchFailedLimit) {
				throw new Error("No proxies found");
			}
			return this.getOneWorkingProxy();
		}


		const isWorking = await this.checkProxy(proxy);
		// if working add it to the end of the list
		if (isWorking) {
			this.proxies.push(proxy);
			return proxy;
		}
		// if not working try the next one
		return this.getOneWorkingProxy();
	}

	fetchProxies = async () => {
		var ProxyData = [];
		try {
			const allProxies = await Promise.all([
				(async () => {

					var ProxyData = []
					const response = await fetch("https://sslproxies.org/");
					if (!response.ok) {
						throw new Error("Error loading proxy, please try again");
					}

					const html = await response.text();
					const $ = cheerio.load(html);

					$("tbody:first td:nth-child(1)").each(function (index, value) {
						ProxyData[index] = { ip: $(this).text() };
					});

					$("tbody:first td:nth-child(2)").each(function (index, value) {
						ProxyData[index]["port"] = $(this).text();
					});

					$("tbody:first td:nth-child(3)").each(function (index, value) {
						ProxyData[index]["country"] = $(this).text();
					});
					return ProxyData;

				})(),
				(async () => {
					// fetch from another source
					var ProxyData = []

					const response = await fetch('https://proxylist.geonode.com/api/proxy-list?limit=100&page=1&sort_by=lastChecked&sort_type=desc')
					if (!response.ok) {
						return
						throw new Error("Error loading proxy, please try again");
					}
					const data = await response.json();

					for (const element of data.data) {
						if (element.protocols.includes("http")) {
							ProxyData.push({ ip: element.ip, port: element.port, country: element.country });
						}
					}
					// logger.log('total proxies:', ProxyData.length);

					return ProxyData;
				})(),
				(async () => {
					var ProxyData = []

					// fetch from another source
					const response = await fetch('https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=http&proxy_format=ipport&format=json&timeout=200')
					if (!response.ok) {
						return
						throw new Error("Error loading proxy, please try again");
					}
					const data = await response.json();

					for (const element of data.proxies) {
						ProxyData.push({ ip: element.ip, port: element.port, country: element.country });
					}
					await data.proxies.forEach((element) => {
						ProxyData.push({ ip: element.ip, port: element.port, country: element.ip_data.countryCode });
					});
					// logger.log('total proxies:', ProxyData.length);

					return ProxyData;
				})()
			]);
			allProxies.map((data) => {
				ProxyData.push(...data);
			});
		} catch (error) {
			logger.log(error);
			return [];
		}
		finally {
			return ProxyData;
		}
	};

	checkProxy = async (proxy) => {
		try {
			// Target website URL
			const targetUrl = "https://httpbin.org/ip";
			// Create a new Proxy Agent
			const response = await fetchWithProxy(targetUrl, {}, proxy, 1000);
			// console.log(await response);

			return response.status === 200;
		} catch (error) {
			logger.error(error);
			return false;
		}
	};

	// Function to filter out non-working proxies and sort them by response time
	filterAndSortProxies = async () => {
		var proxies = await this.fetchProxies()
		logger.log("Total proxies found:", proxies.length);

		// Check if the proxy is working
		const workingProxies = await Promise.all(
			proxies.map(async (proxy) => {
				const isWorking = await this.checkProxy(proxy);
				return { proxy, isWorking };
			})
		);

		// filter out non-working proxies
		const sortedProxies = workingProxies
			.filter(({ isWorking }) => isWorking)

		// get only need properties
		const returnData = sortedProxies.map(({ proxy }) => proxy);
		logger.log("Current working proxies :", returnData.length);
		return returnData;
	};
}

// (async () => {
// 	const fetched = new ProxieProvider();

// 	await fetched.init();
// 	logger.log(await fetched.getOneWorkingProxy());
// 	// logger.log(await fetched.getOneWorkingProxy());
// 	// logger.log(await fetched.getOneWorkingProxy());
// 	// logger.log(await fetched.getOneWorkingProxy());
// 	// logger.log(await fetched.getOneWorkingProxy());
// 	// logger.log(await fetched.getOneWorkingProxy());
// })();