Check robots.txt before you scrape — a 40-line gate for any crawl https://quanticdata.io/tools/robots-txt-tester/

qd_robots_gate.py

"""A robots.txt gate you can put in front of any crawl.

    python3 qd_robots_gate.py https://example.com/page https://example.com/private
    cat urls.txt | python3 qd_robots_gate.py --stdin > allowed.txt

Standard library only, and it avoids the trap that makes most robots checks wrong:

  RobotFileParser.read() fetches with urllib's default user-agent. Behind Cloudflare
  that is frequently a 403 — and the parser treats 401/403 as "disallow everything".
  So a site whose robots.txt says `Allow: /` comes back as fully disallowed, and your
  crawler silently does nothing. Fetching the file with a real UA and handing the text
  to .parse() fixes it.

Interactive tester: https://quanticdata.io/tools/robots-txt-tester/
Generator:          https://quanticdata.io/tools/robots-txt-generator/
Background:         https://quanticdata.io/blog/is-web-crawling-legal/
"""
import sys
import urllib.error
import urllib.request
import urllib.robotparser
from functools import lru_cache
from urllib.parse import urlparse

UA = "MyCrawler/1.0 (+https://example.com/bot)"


@lru_cache(maxsize=256)
def parser_for(origin: str):
    """Fetch robots.txt with a real UA, parse it, cache per origin.

    Returns None when the file could not be read at all — which is not consent.
    """
    request = urllib.request.Request(f"{origin}/robots.txt", headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(request, timeout=15) as response:
            body = response.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as exc:
        if exc.code in (401, 403):
            return None                    # explicitly withheld
        if exc.code >= 400:
            body = ""                      # 404: no robots.txt means no restrictions
        else:
            return None
    except Exception:
        return None

    rp = urllib.robotparser.RobotFileParser()
    rp.parse(body.splitlines())
    return rp


def allowed(url: str) -> tuple[bool, str]:
    parts = urlparse(url)
    if parts.scheme not in ("http", "https") or not parts.netloc:
        return False, "not an http(s) URL"

    rp = parser_for(f"{parts.scheme}://{parts.netloc}")
    if rp is None:
        return False, "robots.txt unreachable or withheld — not treating that as permission"
    if not rp.can_fetch(UA, url):
        return False, "disallowed by robots.txt"

    delay = rp.crawl_delay(UA)
    return True, f"allowed (crawl-delay {delay}s)" if delay else "allowed"


stdin_mode = "--stdin" in sys.argv
urls = (line.strip() for line in sys.stdin) if stdin_mode else iter(
    a for a in sys.argv[1:] if not a.startswith("--"))

for url in urls:
    if not url:
        continue
    ok, why = allowed(url)
    if stdin_mode:
        print(url) if ok else print(f"skip {url}: {why}", file=sys.stderr)
    else:
        print(f"{'ALLOW' if ok else 'DENY '}  {url}   {why}")