Every URL of any website in one call, flat $0.0005 — QuanticData Map API https://quanticdata.io/crawl-map/

qd_map.py

"""Every URL a site's sitemaps and homepage know about — one call, one flat price.

    pip install requests
    export QD_API_KEY=qd_live_...
    python3 qd_map.py https://quanticdata.io          # section histogram + urls.txt
    python3 qd_map.py https://quanticdata.io /blog/   # only URLs containing /blog/

$0.0005 whatever the site's size. `total` and `summary` describe the WHOLE site even
when `links` is capped, so you can size a crawl before you pay for one.
https://quanticdata.io/crawl-map/
"""
import os
import pathlib
import sys

import requests

seed = sys.argv[1] if len(sys.argv) > 1 else "https://quanticdata.io"
needle = sys.argv[2] if len(sys.argv) > 2 else None

body = {"url": seed, "limit": 5000}
if needle:
    body["search"] = needle

r = requests.post("https://api.quanticdata.io/v1/map", json=body,
                  headers={"Authorization": f"Bearer {os.environ['QD_API_KEY']}"}, timeout=120)
data = r.json()
if data.get("type") == "error":
    sys.exit(data["message"])
p = data["payload"]

links = p.get("links") or []
print(f"{p['total']} URLs site-wide, {len(links)} returned")
print(f"sources: {p.get('sources')}  in {p.get('durationMs')} ms\n")

summary = p.get("summary") or {}
top = max(summary.values()) if summary else 1
for section, count in list(summary.items())[:20]:
    print(f"{section:<28}{count:>6}  {'#' * min(50, count * 50 // top)}")

pathlib.Path("urls.txt").write_text("\n".join(links) + "\n", encoding="utf-8")
print(f"\nwrote urls.txt ({len(links)} URLs) — feed it to POST /v1/batch at $0.0002 each")