Rotate proxies properly in Python — retry on the errors that mean "new IP", not on the ones that don't https://quanticdata.io/blog/how-to-rotate-proxies-python/

qd_rotate_retry.py

"""Rotation with a retry policy that knows which failures a new IP can fix.

The mistake that wastes most of a proxy budget: retrying everything. A 404 is a
404 from every exit in the world. Rotating on it costs bandwidth and gets you the
same answer.

  worth a new exit   403, 407-after-success, 429, 503, connection reset, timeouts
  never worth it     400, 401, 404, 410  — the target answered, and it meant it

    pip install requests
    export QD_PROXY_USER=... QD_PROXY_PASS=...
    python3 qd_rotate_retry.py https://example.com

https://quanticdata.io/rotating-proxies/
Guide: https://quanticdata.io/blog/how-to-rotate-proxies-python/
"""
import os
import random
import sys
import time

import requests

GATEWAY = "pr.quanticdata.io:7777"
USER = os.environ["QD_PROXY_USER"]
PASS = os.environ["QD_PROXY_PASS"]

ROTATE_ON = {403, 429, 503, 502, 520, 521, 522, 1020}
GIVE_UP_ON = {400, 401, 404, 410, 451}


def proxies(country="us"):
    url = f"http://{USER}-country-{country}:{PASS}@{GATEWAY}"
    return {"http": url, "https": url}


def fetch(url, country="us", attempts=5):
    """Each attempt gets a fresh exit (no session token = new IP)."""
    delay = 1.0
    for attempt in range(1, attempts + 1):
        try:
            r = requests.get(url, proxies=proxies(country), timeout=40,
                             headers={"User-Agent": "Mozilla/5.0"})
        except requests.RequestException as exc:
            print(f"  {attempt}: {type(exc).__name__} — rotating", file=sys.stderr)
        else:
            if r.status_code in GIVE_UP_ON:
                print(f"  {attempt}: HTTP {r.status_code} — the target meant it, not retrying",
                      file=sys.stderr)
                return r
            if r.status_code in ROTATE_ON:
                print(f"  {attempt}: HTTP {r.status_code} — rotating", file=sys.stderr)
            else:
                return r

        if attempt < attempts:
            # Jittered backoff: a fixed retry interval is itself a signature.
            time.sleep(delay * random.uniform(0.6, 1.5))
            delay = min(delay * 2, 20)
    return None


url = sys.argv[1] if len(sys.argv) > 1 else "https://ipinfo.io/json"
response = fetch(url)
if response is None:
    sys.exit("gave up after rotating through every attempt")
print(f"HTTP {response.status_code}  {len(response.content):,} bytes")
print(response.text[:400])