custom_proxy_rule_for_requests.py

from urllib.parse import urlparse

from requests.adapters import HTTPAdapter


class CustomHttpAdapter(HTTPAdapter):
    def __init__(self, domain_proxies, *args, **kwargs):
        self.domain_proxies = domain_proxies
        self._pm_cache = {}
        super(CustomHttpAdapter, self).__init__(*args, **kwargs)

    def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
        temp_proxies = self.chose_proxies(request.url)
        if temp_proxies:
            proxies = temp_proxies
        return super(CustomHttpAdapter, self).send(request, stream, timeout, verify, cert, proxies)

    def chose_proxies(self, url):
        parse_url = urlparse(url)
        if parse_url.hostname in self.domain_proxies:
            return {"all": self.domain_proxies[parse_url.hostname]}
        return None