A requests Session that rotates (free) proxies for GET-requests.

rotatingproxysession.py

from itertools import cycle

from lxml import html
from requests import Response, Session


class RotatingProxySession(Session):

    def __init__(self) -> None:
        super().__init__()
        r = super().get('https://free-proxy-list.net/')
        tree = html.fromstring(r.content)
        ip_addresses = tree.xpath('//tbody/tr/td[7][contains(text(),"yes")]/../td[1]/text()')
        ports = tree.xpath('//tbody/tr/td[7][contains(text(),"yes")]/../td[2]/text()')
        assert len(ip_addresses) == len(ports)
        self.proxies_cycle = cycle([f'{ip_address}:{port}' for ip_address, port in zip(ip_addresses, ports)])

    def get(self, url, **kwargs) -> Response:
        kwargs.pop('proxies', None)
        proxy = next(self.proxies_cycle)
        return super().get(url, proxies={'http': proxy, 'https': proxy}, **kwargs)