robots.py

from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser

# Look at this cool function to respect robots.txt


USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
)


def _respect_robots(url: str, timeout: int = 10) -> bool:
    parsed = urlparse(url)
    
    # the scheme is something like "https"
    # the netloc is the domain, e.g. www.dim.fr
    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"

    rp = RobotFileParser()
    
    try:
        rp.set_url(robots_url)
        rp.read()
    
    except Exception:
        # If robots.txt is not reachable, default to allow (common practice)
        return True

    return rp.can_fetch(USER_AGENT, url)