BenHamm · June 30, 2026 17:48
diff --git a/README.md b/README.md
diff --git a/scrape_artificial_analysis.py b/scrape_artificial_analysis.py
 #!/usr/bin/env python3
 """
 Scrape Artificial Analysis model intelligence — programmatically.
 =================================================================

 Artificial Analysis (https://artificialanalysis.ai) benchmarks LLM *capability*
 (coding, math, reasoning, agentic, etc.) — the dimension InferenceX does NOT
 cover. Where InferenceX tells you which models run *efficiently* on a given GPU,
 AA tells you which models are *good at a given use case*. Join the two and you
 can answer: "Given my B200s, for code generation, which models are both fast
 AND capable?"

 One JSON endpoint, one API key:

    GET https://artificialanalysis.ai/api/v2/data/llms/models
        header: x-api-key: <YOUR_KEY>

  * Get a free key at https://artificialanalysis.ai/ (account -> API). The free
    tier is rate-limited (a modest number of requests/day) — this endpoint
    returns ALL models in one shot, so one call is all you need; cache it.
  * Attribution is REQUIRED by AA's terms — credit "Artificial Analysis" and
    link https://artificialanalysis.ai/ wherever you display the data.
  * Returns {"data": [ {model}, ... ]} — ~540 models as of 2026-06.

 Stdlib only. Python 3.9+. Set AA_API_KEY in the environment.
 """

 import json
 import os
 import ssl
 import sys
 import urllib.request
 from urllib.error import HTTPError

 AA_API_URL = "https://artificialanalysis.ai/api/v2/data/llms/models"

 # AA per-model `evaluations` benchmarks, grouped into use-case buckets.
 # Each value is 0..100 (the *_index fields) or 0..1 (raw benchmarks).
 USE_CASES = {
    "overall":      ["artificial_analysis_intelligence_index"],
    "coding":       ["artificial_analysis_coding_index", "livecodebench",
                     "scicode", "terminalbench_hard", "terminalbench_v2_1"],
    "math":         ["artificial_analysis_math_index", "aime", "aime_25", "math_500"],
    "reasoning":    ["gpqa", "mmlu_pro", "hle"],
    "agentic":      ["tau2", "tau_banking", "terminalbench_hard"],
    "instruction":  ["ifbench"],
    "long_context": ["lcr"],
 }


 def fetch_models(api_key=None):
    """Fetch the full AA model list. Returns a list of model dicts."""
    api_key = api_key or os.environ.get("AA_API_KEY", "")
    if not api_key:
        raise RuntimeError("Set AA_API_KEY (get a free key at https://artificialanalysis.ai/)")
    req = urllib.request.Request(AA_API_URL, headers={
        "x-api-key": api_key,
        "Accept": "application/json",
        "User-Agent": "aa-scraper/1.0",
    })
    # Some corporate networks have cert issues; fall back to an unverified context.
    try:
        ctx = ssl.create_default_context()
    except Exception:
        ctx = ssl._create_unverified_context()
    try:
        with urllib.request.urlopen(req, timeout=60, context=ctx) as resp:
            data = json.loads(resp.read().decode("utf-8"))
    except HTTPError as e:
        raise RuntimeError(f"AA API HTTP {e.code} {e.reason} "
                           f"(401/403 = bad/missing key; 429 = rate-limited)")
    return data.get("data", data if isinstance(data, list) else [])


 def use_case_profile(model):
    """Return {use_case: score} for one model, averaging its benchmarks.

    *_index fields are 0..100; raw benchmarks are 0..1 — we scale the latter to
    0..100 so a use case mixing both stays on one scale. Missing benchmarks are
    skipped; a use case with no data is omitted.
    """
    evals = model.get("evaluations") or {}
    profile = {}
    for use_case, keys in USE_CASES.items():
        vals = []
        for k in keys:
            v = evals.get(k)
            if v is None:
                continue
            vals.append(v if k.endswith("_index") else v * 100.0)
        if vals:
            profile[use_case] = round(sum(vals) / len(vals), 1)
    return profile


 def rank_for_use_case(models, use_case, top=10, open_only=False):
    """Rank models by a single use case. Returns [(name, score, model), ...]."""
    keys = USE_CASES[use_case]
    scored = []
    for m in models:
        evals = m.get("evaluations") or {}
        vals = [(evals[k] if k.endswith("_index") else evals[k] * 100.0)
                for k in keys if evals.get(k) is not None]
        if not vals:
            continue
        if open_only and not _looks_open(m):
            continue
        scored.append((m.get("name", "?"), round(sum(vals) / len(vals), 1), m))
    scored.sort(key=lambda t: -t[1])
    return scored[:top]


 def _looks_open(model):
    """Heuristic open-weights filter — AA doesn't flag it directly. Tune as needed."""
    creator = (model.get("model_creator") or {}).get("name", "").lower()
    open_creators = {"deepseek", "meta", "mistral", "alibaba", "qwen", "moonshot",
                     "minimax", "z.ai", "zhipu", "nvidia", "openai"}  # gpt-oss is open
    name = model.get("name", "").lower()
    if "gpt-oss" in name:
        return True
    if any(c in creator for c in open_creators) and "gpt-5" not in name and "gpt-4" not in name:
        return True
    return False


 def demo():
    models = fetch_models()
    print(f"Fetched {len(models)} models from Artificial Analysis\n")

    for uc in ("coding", "math", "agentic"):
        print(f"Top 5 — {uc}:")
        for name, score, _ in rank_for_use_case(models, uc, top=5):
            print(f"  {score:5.1f}  {name}")
        print()

    # Full use-case profile + price/speed for a few specific models.
    print("Per-model use-case profiles (intelligence is the headline index):")
    for m in models:
        if m.get("name", "").startswith(("gpt-oss-120b (high)", "MiniMax-M3", "Qwen3.5 397B")):
            prof = use_case_profile(m)
            price = (m.get("pricing") or {}).get("price_1m_blended_3_to_1")
            spd = m.get("median_output_tokens_per_second")
            print(f"\n  {m['name']}  (${price}/M blended, {spd} tok/s on AA's host)")
            for uc, s in prof.items():
                print(f"      {uc:13s} {s}")


 if __name__ == "__main__":
    if "--json" in sys.argv:
        json.dump(fetch_models(), sys.stdout, indent=2)
    else:
        demo()
diff --git a/scrape_inferencex.py b/scrape_inferencex.py
 #!/usr/bin/env python3
 """
 Scrape SemiAnalysis InferenceX benchmark results — programmatically.
 ====================================================================

 InferenceX (https://inferencex.semianalysis.com) publishes NVIDIA-vs-AMD GPU
 inference benchmarks. The dashboard is a JS single-page app, but it is backed
 by three plain public JSON endpoints. You do NOT need a browser, Playwright,
 or an API key — just HTTP GET.

    GET /api/v1/benchmarks?model=<DISPLAY_NAME>   # speed/throughput, per model
    GET /api/v1/evaluations                        # accuracy evals (all models)
    GET /api/v1/reliability                        # GPU reliability (all models)

 Notes that bite you:
  * Responses are gzip-encoded. Send `Accept-Encoding: gzip` and decompress,
    or use a client that auto-decompresses (requests does; raw urllib does not).
  * /benchmarks REQUIRES the human DISPLAY name (e.g. "DeepSeek-R1-0528"),
    not the internal db name ("dsr1"). Wrong/empty name -> {"error":"Unknown model"}.
  * /evaluations and /reliability take no params and return everything; the
    `model` field there is the internal db name.
  * There is no public endpoint that lists models — discover db names from
    /evaluations, then map to display names (see DISPLAY_NAMES below).

 Stdlib only. Python 3.9+.
 """

 import gzip
 import json
 import sys
 import urllib.error
 import urllib.parse
 import urllib.request

 API_BASE = "https://inferencex.semianalysis.com"

 # db-name -> display-name. The /benchmarks endpoint wants the display name.
 # db names are discoverable from /evaluations; display names live only in the
 # site's JS bundle. This snapshot was lifted from that bundle (2026-06-30) —
 # or call discover_display_names() below to pull a fresh copy. Note several
 # db names can share one display name (e.g. glm5 and glm5.1 -> "GLM-5"), so
 # querying that display name returns rows for all of them.
 DISPLAY_NAMES = {
    "dsr1":          "DeepSeek-R1-0528",
    "dsv4":          "DeepSeek-V4-Pro",
    "gptoss120b":    "gpt-oss-120b",
    "llama70b":      "Llama-3.3-70B-Instruct-FP8",
    "qwen3.5":       "Qwen-3.5-397B-A17B",
    "kimik2.5":      "Kimi-K2.5",
    "kimik2.6":      "Kimi-K2.5",
    "kimik2.7-code": "Kimi-K2.5",
    "minimaxm2.5":   "MiniMax-M2.5",
    "minimaxm2.7":   "MiniMax-M2.5",
    "minimaxm3":     "MiniMax-M3",
    "glm5":          "GLM-5",
    "glm5.1":        "GLM-5",
 }

 NVIDIA_HW = {"b200", "b300", "gb200", "gb300", "h100", "h200"}
 AMD_HW = {"mi300x", "mi325x", "mi355x"}


 def fetch_json(url):
    """GET a URL and parse JSON, handling gzip and HTTP errors."""
    req = urllib.request.Request(url, headers={
        "Accept": "application/json",
        "Accept-Encoding": "gzip",
        "User-Agent": "inferencex-scraper/1.0",
    })
    with urllib.request.urlopen(req, timeout=20) as resp:
        data = resp.read()
        if resp.headers.get("Content-Encoding") == "gzip":
            data = gzip.decompress(data)
    return json.loads(data)


 def get_benchmarks(display_name):
    """Speed/throughput rows for one model. `display_name` must be the
    human name, e.g. 'DeepSeek-R1-0528'."""
    q = urllib.parse.quote(display_name)
    return fetch_json(f"{API_BASE}/api/v1/benchmarks?model={q}")


 def get_evaluations():
    """All accuracy-eval rows (every model/task/GPU)."""
    return fetch_json(f"{API_BASE}/api/v1/evaluations")


 def get_reliability():
    """All GPU reliability rows (per-day n_success / total)."""
    return fetch_json(f"{API_BASE}/api/v1/reliability")


 def discover_models():
    """Return the set of internal db model names currently in the dataset,
    derived from the evaluations endpoint (no models endpoint exists)."""
    rows = get_evaluations()
    return sorted({r["model"] for r in rows if r.get("model")})


 def discover_display_names():
    """Best-effort: extract the live db-name -> display-name map from the
    site's JS bundle so you don't have to hand-maintain DISPLAY_NAMES.

    The map is emitted in a chunk as a JS object literal like
    {dsr1:"DeepSeek-R1-0528", ... ,dsv4:"DeepSeek-V4-Pro"}. We find the chunk
    that mentions a known model and pull the surrounding object. Falls back to
    the static DISPLAY_NAMES on any failure.
    """
    import re

    try:
        page = fetch_text(f"{API_BASE}/inference")
        chunks = sorted(set(re.findall(r"/_next/static/chunks/[\w/.-]+\.js", page)))
        anchor = "dsr1"  # a stable db name present in the mapping object
        for path in chunks:
            js = fetch_text(f"{API_BASE}{path}")
            if f'{anchor}:"' not in js:
                continue
            # Grab the {...} object literal that contains the anchor pair.
            m = re.search(r'\{[^{}]*' + anchor + r':"[^{}]*\}', js)
            if not m:
                continue
            blob = m.group(0)
            # Parse "key":"val" and bareword key:"val" pairs.
            pairs = re.findall(r'"?([\w.\-]+)"?:"([^"]+)"', blob)
            mapping = {k: v for k, v in pairs}
            if mapping:
                return mapping
    except Exception:
        pass
    return dict(DISPLAY_NAMES)


 def fetch_text(url):
    """GET a URL and return decoded text (handles gzip)."""
    req = urllib.request.Request(url, headers={
        "Accept-Encoding": "gzip",
        "User-Agent": "inferencex-scraper/1.0",
    })
    with urllib.request.urlopen(req, timeout=20) as resp:
        data = resp.read()
        if resp.headers.get("Content-Encoding") == "gzip":
            data = gzip.decompress(data)
    return data.decode("utf-8", "replace")


 def vendor(hw):
    hw = (hw or "").lower()
    if hw in NVIDIA_HW:
        return "nvidia"
    if hw in AMD_HW:
        return "amd"
    return "other"


 def demo():
    print("Discovering models from /evaluations ...")
    db_names = discover_models()
    print(f"  {len(db_names)} models: {', '.join(db_names)}\n")

    # Pull speed benchmarks for whatever we have a display name for.
    # Multiple db names can share a display name (glm5/glm5.1 -> "GLM-5"), and
    # one /benchmarks call returns rows for all of them — so dedupe by display.
    grand_total = 0
    seen = set()
    for db in db_names:
        display = DISPLAY_NAMES.get(db)
        if not display:
            print(f"  ! no display-name mapping for '{db}' — add it to DISPLAY_NAMES, skipping")
            continue
        if display in seen:
            continue
        seen.add(display)
        try:
            rows = get_benchmarks(display)
        except urllib.error.HTTPError as e:
            print(f"  ! {display}: HTTP {e.code}")
            continue
        grand_total += len(rows)
        nv = sum(1 for r in rows if vendor(r.get("hardware")) == "nvidia")
        amd = sum(1 for r in rows if vendor(r.get("hardware")) == "amd")
        print(f"  {display:22s} {len(rows):5d} rows  (NVIDIA:{nv}  AMD:{amd})")

    evals = get_evaluations()
    rel = get_reliability()
    print(f"\nTotals: {grand_total} benchmark rows, "
          f"{len(evals)} eval rows, {len(rel)} reliability rows")

    # Example: best decode throughput per GPU for one model/config.
    print("\nExample — DeepSeek-R1-0528, best tput_per_gpu at ISL=1024/OSL=1024:")
    best = {}
    for r in get_benchmarks("DeepSeek-R1-0528"):
        if r.get("isl") != 1024 or r.get("osl") != 1024:
            continue
        hw = r.get("hardware", "")
        t = (r.get("metrics") or {}).get("tput_per_gpu")
        if t and (hw not in best or t > best[hw]):
            best[hw] = t
    for hw, t in sorted(best.items(), key=lambda kv: -kv[1]):
        print(f"  [{vendor(hw):6s}] {hw:8s} {t:8.1f} tok/s/gpu")


 if __name__ == "__main__":
    demo()
ISL / OSL	Shape	Typical use cases
1024 / 1024	balanced	Chat, assistants
8192 / 1024	long in, short out	RAG, summarization, document Q&A
1024 / 8192	short in, long out	Reasoning, agents, long-form / code generation
Endpoint	Params	Returns
`GET /api/v1/benchmarks`	`?model=<DISPLAY_NAME>` (required)	Speed/throughput rows for one model, across every GPU + framework + ISL/OSL/concurrency config
`GET /api/v1/evaluations`	none	Every accuracy-eval row (model × task × GPU)
`GET /api/v1/reliability`	none	Per-day GPU reliability rows (`n_success` / `total`)
Use case	AA benchmarks
Overall	`artificial_analysis_intelligence_index`
Coding	`artificial_analysis_coding_index`, `livecodebench`, `scicode`, `terminalbench_hard`, `terminalbench_v2_1`
Math	`artificial_analysis_math_index`, `aime`, `aime_25`, `math_500`
Reasoning / knowledge	`gpqa`, `mmlu_pro`, `hle`
Agentic / tool-use	`tau2`, `tau_banking`, `terminalbench_hard`
Instruction-following	`ifbench`
Long-context reasoning	`lcr`
	#!/usr/bin/env python3
	"""
	Scrape Artificial Analysis model intelligence — programmatically.
	=================================================================

	Artificial Analysis (https://artificialanalysis.ai) benchmarks LLM capability
	(coding, math, reasoning, agentic, etc.) — the dimension InferenceX does NOT
	cover. Where InferenceX tells you which models run efficiently on a given GPU,
	AA tells you which models are good at a given use case. Join the two and you
	can answer: "Given my B200s, for code generation, which models are both fast
	AND capable?"

	One JSON endpoint, one API key:

	GET https://artificialanalysis.ai/api/v2/data/llms/models
	header: x-api-key: <YOUR_KEY>

	* Get a free key at https://artificialanalysis.ai/ (account -> API). The free
	tier is rate-limited (a modest number of requests/day) — this endpoint
	returns ALL models in one shot, so one call is all you need; cache it.
	* Attribution is REQUIRED by AA's terms — credit "Artificial Analysis" and
	link https://artificialanalysis.ai/ wherever you display the data.
	* Returns {"data": [ {model}, ... ]} — ~540 models as of 2026-06.

	Stdlib only. Python 3.9+. Set AA_API_KEY in the environment.
	"""

	import json
	import os
	import ssl
	import sys
	import urllib.request
	from urllib.error import HTTPError

	AA_API_URL = "https://artificialanalysis.ai/api/v2/data/llms/models"

	# AA per-model `evaluations` benchmarks, grouped into use-case buckets.
	# Each value is 0..100 (the *_index fields) or 0..1 (raw benchmarks).
	USE_CASES = {
	"overall": ["artificial_analysis_intelligence_index"],
	"coding": ["artificial_analysis_coding_index", "livecodebench",
	"scicode", "terminalbench_hard", "terminalbench_v2_1"],
	"math": ["artificial_analysis_math_index", "aime", "aime_25", "math_500"],
	"reasoning": ["gpqa", "mmlu_pro", "hle"],
	"agentic": ["tau2", "tau_banking", "terminalbench_hard"],
	"instruction": ["ifbench"],
	"long_context": ["lcr"],
	}


	def fetch_models(api_key=None):
	"""Fetch the full AA model list. Returns a list of model dicts."""
	api_key = api_key or os.environ.get("AA_API_KEY", "")
	if not api_key:
	raise RuntimeError("Set AA_API_KEY (get a free key at https://artificialanalysis.ai/)")
	req = urllib.request.Request(AA_API_URL, headers={
	"x-api-key": api_key,
	"Accept": "application/json",
	"User-Agent": "aa-scraper/1.0",
	})
	# Some corporate networks have cert issues; fall back to an unverified context.
	try:
	ctx = ssl.create_default_context()
	except Exception:
	ctx = ssl._create_unverified_context()
	try:
	with urllib.request.urlopen(req, timeout=60, context=ctx) as resp:
	data = json.loads(resp.read().decode("utf-8"))
	except HTTPError as e:
	raise RuntimeError(f"AA API HTTP {e.code} {e.reason} "
	f"(401/403 = bad/missing key; 429 = rate-limited)")
	return data.get("data", data if isinstance(data, list) else [])


	def use_case_profile(model):
	"""Return {use_case: score} for one model, averaging its benchmarks.

	*_index fields are 0..100; raw benchmarks are 0..1 — we scale the latter to
	0..100 so a use case mixing both stays on one scale. Missing benchmarks are
	skipped; a use case with no data is omitted.
	"""
	evals = model.get("evaluations") or {}
	profile = {}
	for use_case, keys in USE_CASES.items():
	vals = []
	for k in keys:
	v = evals.get(k)
	if v is None:
	continue
	vals.append(v if k.endswith("_index") else v * 100.0)
	if vals:
	profile[use_case] = round(sum(vals) / len(vals), 1)
	return profile


	def rank_for_use_case(models, use_case, top=10, open_only=False):
	"""Rank models by a single use case. Returns [(name, score, model), ...]."""
	keys = USE_CASES[use_case]
	scored = []
	for m in models:
	evals = m.get("evaluations") or {}
	vals = [(evals[k] if k.endswith("_index") else evals[k] * 100.0)
	for k in keys if evals.get(k) is not None]
	if not vals:
	continue
	if open_only and not _looks_open(m):
	continue
	scored.append((m.get("name", "?"), round(sum(vals) / len(vals), 1), m))
	scored.sort(key=lambda t: -t[1])
	return scored[:top]


	def _looks_open(model):
	"""Heuristic open-weights filter — AA doesn't flag it directly. Tune as needed."""
	creator = (model.get("model_creator") or {}).get("name", "").lower()
	open_creators = {"deepseek", "meta", "mistral", "alibaba", "qwen", "moonshot",
	"minimax", "z.ai", "zhipu", "nvidia", "openai"} # gpt-oss is open
	name = model.get("name", "").lower()
	if "gpt-oss" in name:
	return True
	if any(c in creator for c in open_creators) and "gpt-5" not in name and "gpt-4" not in name:
	return True
	return False


	def demo():
	models = fetch_models()
	print(f"Fetched {len(models)} models from Artificial Analysis\n")

	for uc in ("coding", "math", "agentic"):
	print(f"Top 5 — {uc}:")
	for name, score, _ in rank_for_use_case(models, uc, top=5):
	print(f" {score:5.1f} {name}")
	print()

	# Full use-case profile + price/speed for a few specific models.
	print("Per-model use-case profiles (intelligence is the headline index):")
	for m in models:
	if m.get("name", "").startswith(("gpt-oss-120b (high)", "MiniMax-M3", "Qwen3.5 397B")):
	prof = use_case_profile(m)
	price = (m.get("pricing") or {}).get("price_1m_blended_3_to_1")
	spd = m.get("median_output_tokens_per_second")
	print(f"\n {m['name']} (${price}/M blended, {spd} tok/s on AA's host)")
	for uc, s in prof.items():
	print(f" {uc:13s} {s}")


	if __name__ == "__main__":
	if "--json" in sys.argv:
	json.dump(fetch_models(), sys.stdout, indent=2)
	else:
	demo()
	#!/usr/bin/env python3
	"""
	Scrape SemiAnalysis InferenceX benchmark results — programmatically.
	====================================================================

	InferenceX (https://inferencex.semianalysis.com) publishes NVIDIA-vs-AMD GPU
	inference benchmarks. The dashboard is a JS single-page app, but it is backed
	by three plain public JSON endpoints. You do NOT need a browser, Playwright,
	or an API key — just HTTP GET.

	GET /api/v1/benchmarks?model=<DISPLAY_NAME> # speed/throughput, per model
	GET /api/v1/evaluations # accuracy evals (all models)
	GET /api/v1/reliability # GPU reliability (all models)

	Notes that bite you:
	* Responses are gzip-encoded. Send `Accept-Encoding: gzip` and decompress,
	or use a client that auto-decompresses (requests does; raw urllib does not).
	* /benchmarks REQUIRES the human DISPLAY name (e.g. "DeepSeek-R1-0528"),
	not the internal db name ("dsr1"). Wrong/empty name -> {"error":"Unknown model"}.
	* /evaluations and /reliability take no params and return everything; the
	`model` field there is the internal db name.
	* There is no public endpoint that lists models — discover db names from
	/evaluations, then map to display names (see DISPLAY_NAMES below).

	Stdlib only. Python 3.9+.
	"""

	import gzip
	import json
	import sys
	import urllib.error
	import urllib.parse
	import urllib.request

	API_BASE = "https://inferencex.semianalysis.com"

	# db-name -> display-name. The /benchmarks endpoint wants the display name.
	# db names are discoverable from /evaluations; display names live only in the
	# site's JS bundle. This snapshot was lifted from that bundle (2026-06-30) —
	# or call discover_display_names() below to pull a fresh copy. Note several
	# db names can share one display name (e.g. glm5 and glm5.1 -> "GLM-5"), so
	# querying that display name returns rows for all of them.
	DISPLAY_NAMES = {
	"dsr1": "DeepSeek-R1-0528",
	"dsv4": "DeepSeek-V4-Pro",
	"gptoss120b": "gpt-oss-120b",
	"llama70b": "Llama-3.3-70B-Instruct-FP8",
	"qwen3.5": "Qwen-3.5-397B-A17B",
	"kimik2.5": "Kimi-K2.5",
	"kimik2.6": "Kimi-K2.5",
	"kimik2.7-code": "Kimi-K2.5",
	"minimaxm2.5": "MiniMax-M2.5",
	"minimaxm2.7": "MiniMax-M2.5",
	"minimaxm3": "MiniMax-M3",
	"glm5": "GLM-5",
	"glm5.1": "GLM-5",
	}

	NVIDIA_HW = {"b200", "b300", "gb200", "gb300", "h100", "h200"}
	AMD_HW = {"mi300x", "mi325x", "mi355x"}


	def fetch_json(url):
	"""GET a URL and parse JSON, handling gzip and HTTP errors."""
	req = urllib.request.Request(url, headers={
	"Accept": "application/json",
	"Accept-Encoding": "gzip",
	"User-Agent": "inferencex-scraper/1.0",
	})
	with urllib.request.urlopen(req, timeout=20) as resp:
	data = resp.read()
	if resp.headers.get("Content-Encoding") == "gzip":
	data = gzip.decompress(data)
	return json.loads(data)


	def get_benchmarks(display_name):
	"""Speed/throughput rows for one model. `display_name` must be the
	human name, e.g. 'DeepSeek-R1-0528'."""
	q = urllib.parse.quote(display_name)
	return fetch_json(f"{API_BASE}/api/v1/benchmarks?model={q}")


	def get_evaluations():
	"""All accuracy-eval rows (every model/task/GPU)."""
	return fetch_json(f"{API_BASE}/api/v1/evaluations")


	def get_reliability():
	"""All GPU reliability rows (per-day n_success / total)."""
	return fetch_json(f"{API_BASE}/api/v1/reliability")


	def discover_models():
	"""Return the set of internal db model names currently in the dataset,
	derived from the evaluations endpoint (no models endpoint exists)."""
	rows = get_evaluations()
	return sorted({r["model"] for r in rows if r.get("model")})


	def discover_display_names():
	"""Best-effort: extract the live db-name -> display-name map from the
	site's JS bundle so you don't have to hand-maintain DISPLAY_NAMES.

	The map is emitted in a chunk as a JS object literal like
	{dsr1:"DeepSeek-R1-0528", ... ,dsv4:"DeepSeek-V4-Pro"}. We find the chunk
	that mentions a known model and pull the surrounding object. Falls back to
	the static DISPLAY_NAMES on any failure.
	"""
	import re

	try:
	page = fetch_text(f"{API_BASE}/inference")
	chunks = sorted(set(re.findall(r"/_next/static/chunks/[\w/.-]+\.js", page)))
	anchor = "dsr1" # a stable db name present in the mapping object
	for path in chunks:
	js = fetch_text(f"{API_BASE}{path}")
	if f'{anchor}:"' not in js:
	continue
	# Grab the {...} object literal that contains the anchor pair.
	m = re.search(r'\{[^{}]' + anchor + r':"[^{}]\}', js)
	if not m:
	continue
	blob = m.group(0)
	# Parse "key":"val" and bareword key:"val" pairs.
	pairs = re.findall(r'"?([\w.\-]+)"?:"([^"]+)"', blob)
	mapping = {k: v for k, v in pairs}
	if mapping:
	return mapping
	except Exception:
	pass
	return dict(DISPLAY_NAMES)


	def fetch_text(url):
	"""GET a URL and return decoded text (handles gzip)."""
	req = urllib.request.Request(url, headers={
	"Accept-Encoding": "gzip",
	"User-Agent": "inferencex-scraper/1.0",
	})
	with urllib.request.urlopen(req, timeout=20) as resp:
	data = resp.read()
	if resp.headers.get("Content-Encoding") == "gzip":
	data = gzip.decompress(data)
	return data.decode("utf-8", "replace")


	def vendor(hw):
	hw = (hw or "").lower()
	if hw in NVIDIA_HW:
	return "nvidia"
	if hw in AMD_HW:
	return "amd"
	return "other"


	def demo():
	print("Discovering models from /evaluations ...")
	db_names = discover_models()
	print(f" {len(db_names)} models: {', '.join(db_names)}\n")

	# Pull speed benchmarks for whatever we have a display name for.
	# Multiple db names can share a display name (glm5/glm5.1 -> "GLM-5"), and
	# one /benchmarks call returns rows for all of them — so dedupe by display.
	grand_total = 0
	seen = set()
	for db in db_names:
	display = DISPLAY_NAMES.get(db)
	if not display:
	print(f" ! no display-name mapping for '{db}' — add it to DISPLAY_NAMES, skipping")
	continue
	if display in seen:
	continue
	seen.add(display)
	try:
	rows = get_benchmarks(display)
	except urllib.error.HTTPError as e:
	print(f" ! {display}: HTTP {e.code}")
	continue
	grand_total += len(rows)
	nv = sum(1 for r in rows if vendor(r.get("hardware")) == "nvidia")
	amd = sum(1 for r in rows if vendor(r.get("hardware")) == "amd")
	print(f" {display:22s} {len(rows):5d} rows (NVIDIA:{nv} AMD:{amd})")

	evals = get_evaluations()
	rel = get_reliability()
	print(f"\nTotals: {grand_total} benchmark rows, "
	f"{len(evals)} eval rows, {len(rel)} reliability rows")

	# Example: best decode throughput per GPU for one model/config.
	print("\nExample — DeepSeek-R1-0528, best tput_per_gpu at ISL=1024/OSL=1024:")
	best = {}
	for r in get_benchmarks("DeepSeek-R1-0528"):
	if r.get("isl") != 1024 or r.get("osl") != 1024:
	continue
	hw = r.get("hardware", "")
	t = (r.get("metrics") or {}).get("tput_per_gpu")
	if t and (hw not in best or t > best[hw]):
	best[hw] = t
	for hw, t in sorted(best.items(), key=lambda kv: -kv[1]):
	print(f" [{vendor(hw):6s}] {hw:8s} {t:8.1f} tok/s/gpu")


	if __name__ == "__main__":
	demo()