Benchmarks behind 'The Optimization That Wasn't': LM Studio /v1/responses previous_response_id token-balloon repro, plus Ollama-vs-LM-Studio prompt caching & throughput. Python 3 stdlib only.

cachecmp.py

#!/usr/bin/env python3
"""Caching + fair-path speed: Ollama vs LM Studio, same model (qwen3-4b).

Uses each engine's NATIVE stats endpoint so prefill (TTFT) and generation
tok/s are read straight from the server, not inferred from wall-clock.

  Ollama   : POST /api/chat            -> prompt_eval_count/duration, eval_count/duration
  LM Studio: POST /api/v0/chat/completions -> stats.time_to_first_token, stats.tokens_per_second, usage.prompt_tokens

Two tests per engine:
  (1) COLD vs WARM: send an identical ~1500-token prompt twice. If the prefix
      KV cache works, the 2nd prefill (TTFT) is far cheaper than the 1st.
  (2) GROWING CONVERSATION (the path you actually use for multi-turn on BOTH
      engines): resend full history each turn. If the prefix is cached, TTFT
      stays low even as prompt_tokens climbs.
"""
import json, urllib.request, os

def post(base, path, body, t=600):
    req = urllib.request.Request(base+path, data=json.dumps(body).encode(),
                                 headers={"Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=t))

def call(base, model, kind, messages, npred):
    """Returns (ttft_ms, gen_tok_s, prompt_tokens, gen_tokens)."""
    if kind == "ollama":
        r = post(base, "/api/chat", {"model": model, "messages": messages,
                 "stream": False, "options": {"num_predict": npred, "temperature": 0}})
        ped = r.get("prompt_eval_duration", 0)/1e6   # ns -> ms
        ed = r.get("eval_duration", 0)/1e9           # ns -> s
        return (ped, (r.get("eval_count", 0)/ed if ed else 0),
                r.get("prompt_eval_count"), r.get("eval_count"))
    else:
        r = post(base, "/api/v0/chat/completions", {"model": model, "messages": messages,
                 "max_tokens": npred, "temperature": 0, "stream": False})
        st = r.get("stats", {}); u = r.get("usage", {})
        return ((st.get("time_to_first_token") or 0)*1000, st.get("tokens_per_second") or 0,
                u.get("prompt_tokens"), u.get("completion_tokens"))

def bigprompt():
    return ("Reference text follows; reply with only the word OK. " +
            "The history of computing spans many decades and involves countless "
            "innovations across hardware and software. " * 110)

TURNS = [
    "Remember: my dog is named Rex and he is brown. Reply with just: OK.",
    "What is my dog's name? One word.",
    "What color is Rex? One word.",
    "I also have a cat named Mia. Reply with just: OK.",
    "How many pets do I have now? Reply with one number.",
    "List my pets' names, comma-separated.",
    "Rex is 3 years old. Reply with just: OK.",
    "How old is Rex? Reply with one number.",
    "Which pet is brown? One word.",
    "Summarize what you know about my pets in one short sentence.",
]

def run_engine(name, base, model, kind):
    print(f"\n===== {name}  ({model}) =====")
    # warm/load
    try:
        call(base, model, kind, [{"role": "user", "content": "hi"}], 1)
    except Exception as e:
        print("  LOAD ERROR:", str(e)[:80]); return

    # (1) cold vs warm, identical big prompt
    bp = [{"role": "user", "content": bigprompt()}]
    t_cold, _, pt, _ = call(base, model, kind, bp, 8)
    t_warm, _, _, _ = call(base, model, kind, bp, 8)
    sp = (t_cold/t_warm) if t_warm else 0
    print(f"  [1] cold-vs-warm  prompt_tokens={pt}:  cold TTFT={t_cold:7.0f} ms   warm TTFT={t_warm:7.0f} ms   -> {sp:5.1f}x faster")

    # (2) growing conversation, full resend each turn
    print(f"  [2] growing conversation (full resend each turn):")
    print(f"      {'turn':>4} {'prompt_tok':>10} {'TTFT ms':>9} {'gen tok/s':>9}")
    msgs = []
    for i, t in enumerate(TURNS, 1):
        msgs.append({"role": "user", "content": t})
        ttft, gts, pt, ct = call(base, model, kind, msgs, 32)
        msgs.append({"role": "assistant", "content": "OK"})  # terse, fixed history growth
        print(f"      {i:>4} {str(pt):>10} {ttft:>9.0f} {gts:>9.1f}")

import sys
SPECS = sys.argv[1:] or [
    "Ollama   GGUF Q4_K_M|http://localhost:11434|qwen3:4b|ollama",
    "LM Studio  4bit MLX |http://localhost:1234|qwen3-4b-mlx|lms",
]
for spec in SPECS:
    name, base, model, kind = spec.split("|")
    run_engine(name, base, model, kind)

probe.py

#!/usr/bin/env python3
"""Minimal repro for the LM Studio /v1/responses token-balloon bug.

Runs the SAME fixed conversation two ways and prints per-turn token counts
and latency so you can see them diverge:

  [A] /v1/responses with previous_response_id chaining — send ONLY the new
      message each turn and let the server reconstruct the history.
  [B] /v1/chat/completions — resend the full history every turn.

On LM Studio, [A]'s input_tokens roughly DOUBLES every turn (geometric
balloon) while [B] stays linear/flat. On OpenAI and Ollama, [A] does not
balloon. See https://github.com/lmstudio-ai/lmstudio-bug-tracker/issues/2074

Usage:   python3 probe.py "<model-id>" [n_turns]
Env:     PROBE_BASE  (default http://localhost:1234)
         PROBE_KEY   (optional bearer token)
Deps:    Python 3 standard library only.
"""
import json, time, urllib.request, sys, os

BASE = os.environ.get("PROBE_BASE", "http://localhost:1234")
KEY = os.environ.get("PROBE_KEY", "")
MODEL = sys.argv[1] if len(sys.argv) > 1 else "mlx-community/llama-3.2-3b-instruct"
N = int(sys.argv[2]) if len(sys.argv) > 2 else 12

TURNS = [
    "Remember: my dog is named Rex and he is brown. Reply with just: OK.",
    "What is my dog's name? One word.",
    "What color is Rex? One word.",
    "I also have a cat named Mia. Reply with just: OK.",
    "How many pets do I have now? Reply with one number.",
    "List my pets' names, comma-separated.",
    "Rex is 3 years old. Reply with just: OK.",
    "How old is Rex? Reply with one number.",
    "Which pet is brown? One word.",
    "Summarize what you know about my pets in one short sentence.",
    "What did I tell you first? One short sentence.",
    "How many facts have I given you so far? One number.",
][:N]

def post(path, body):
    data = json.dumps(body).encode()
    headers = {"Content-Type": "application/json"}
    if KEY:
        headers["Authorization"] = "Bearer " + KEY
    req = urllib.request.Request(BASE + path, data=data, headers=headers)
    t0 = time.time()
    resp = json.load(urllib.request.urlopen(req, timeout=600))
    return (time.time() - t0) * 1000.0, resp

def cached_of(usage, key):
    det = usage.get(key) or {}
    return det.get("cached_tokens")

print(f"\n########## MODEL: {MODEL} ##########")

# --- Path A: Responses API with previous_response_id chaining (only the new turn is sent) ---
print("\n[A] /v1/responses  (previous_response_id chaining; client sends ONLY the new message)")
print(f"{'turn':>4} {'ms':>8} {'input_tokens':>13} {'cached':>8} {'out':>5}")
prev = None
for i, t in enumerate(TURNS, 1):
    body = {"model": MODEL, "input": t, "store": True, "max_output_tokens": 64}
    if prev:
        body["previous_response_id"] = prev
    try:
        ms, r = post("/v1/responses", body)
    except Exception as e:
        print(f"{i:>4}  ERROR: {e}")
        break
    u = r.get("usage", {}) or {}
    print(f"{i:>4} {ms:>8.0f} {str(u.get('input_tokens')):>13} {str(cached_of(u,'input_tokens_details')):>8} {str(u.get('output_tokens')):>5}")
    prev = r.get("id")

# --- Path B: Chat Completions resending full history each turn ---
print("\n[B] /v1/chat/completions  (full history resent each turn)")
print(f"{'turn':>4} {'ms':>8} {'prompt_tokens':>13} {'cached':>8} {'out':>5}")
msgs = []
for i, t in enumerate(TURNS, 1):
    msgs.append({"role": "user", "content": t})
    body = {"model": MODEL, "messages": msgs, "max_tokens": 64, "temperature": 0}
    try:
        ms, r = post("/v1/chat/completions", body)
    except Exception as e:
        print(f"{i:>4}  ERROR: {e}")
        break
    u = r.get("usage", {}) or {}
    asst = (r.get("choices") or [{}])[0].get("message", {}).get("content", "")
    msgs.append({"role": "assistant", "content": asst})
    print(f"{i:>4} {ms:>8.0f} {str(u.get('prompt_tokens')):>13} {str(cached_of(u,'prompt_tokens_details')):>8} {str(u.get('completion_tokens')):>5}")

thrubench.py

#!/usr/bin/env python3
"""Reliable sustained generation-throughput for a matched model pair.

Generates a long, fixed number of tokens (NPRED) so tok/s reflects
sustained decode speed, not a short-burst artifact. Unique prompt per
trial => cold prefill (also yields an honest prefill tok/s). Best-of-3.
"""
import json, urllib.request, os, sys
NPRED = 200
def post(base, path, body, t=600):
    req = urllib.request.Request(base+path, data=json.dumps(body).encode(),
                                 headers={"Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=t))
def prompt():
    return (f"[{os.urandom(6).hex()}] " +
            "The history of computing spans many decades and involves countless "
            "innovations across hardware and software. " * 60 +
            "\n\nWrite a detailed 180-word essay expanding on the text above.")
def trial(base, model, kind):
    p = [{"role": "user", "content": prompt()}]
    if kind == "ollama":
        r = post(base, "/api/chat", {"model": model, "messages": p, "stream": False,
                 "options": {"num_predict": NPRED, "temperature": 0}})
        ped = r.get("prompt_eval_duration", 0)/1e9; ed = r.get("eval_duration", 0)/1e9
        return ((r.get("prompt_eval_count", 0)/ped if ped else 0),
                (r.get("eval_count", 0)/ed if ed else 0), r.get("eval_count"))
    else:
        r = post(base, "/api/v0/chat/completions", {"model": model, "messages": p,
                 "max_tokens": NPRED, "temperature": 0, "stream": False})
        st = r.get("stats", {}); u = r.get("usage", {})
        ttft = st.get("time_to_first_token") or 0; pt = u.get("prompt_tokens") or 0
        return ((pt/ttft if ttft else 0), st.get("tokens_per_second") or 0, u.get("completion_tokens"))
def run(name, base, model, kind):
    try: post(base, ("/api/chat" if kind=="ollama" else "/api/v0/chat/completions"),
              ({"model":model,"messages":[{"role":"user","content":"hi"}],"stream":False,"options":{"num_predict":1}} if kind=="ollama"
               else {"model":model,"messages":[{"role":"user","content":"hi"}],"max_tokens":1,"stream":False}))
    except Exception as e: print(f"  {name:30} LOAD ERR {str(e)[:50]}"); return
    rs = [trial(base, model, kind) for _ in range(3)]
    print(f"  {name:30} prefill={max(r[0] for r in rs):>6.0f} tok/s   gen={max(r[1] for r in rs):>6.1f} tok/s   (gen_tokens={rs[-1][2]})")

print(f"=== Sustained throughput, matched nvfp4 gemma-4-e4b (generate {NPRED}, best-of-3) ===")
for spec in (sys.argv[1:] or [
    "Ollama    nvfp4|http://localhost:11434|gemma4:e4b-mlx|ollama",
    "LM Studio nvfp4|http://localhost:1234|gemma-4-e4b-it-nvfp4|lms",
]):
    name, base, model, kind = spec.split("|"); run(name, base, model, kind)