teknium1 · June 22, 2026 01:13
diff --git a/README.md b/README.md
diff --git a/analyze.py b/analyze.py
 """Aggregate results.json + quality.json into the PR-body metrics tables."""
 import json
 import statistics as st
 from pathlib import Path

 R = json.loads(Path("/tmp/bgreview_bench/results.json").read_text())
 Q = json.loads(Path("/tmp/bgreview_bench/quality.json").read_text())


 def prf(graded):
    """Precision/recall/F1 of the save decision against the fixture oracle."""
    tp = fp = fn = tn = 0
    for g in graded:
        want = g["expect_save"]
        did = bool(g["did_save"])
        if want and did:
            tp += 1
        elif want and not did:
            fn += 1
        elif (not want) and did:
            fp += 1
        else:
            tn += 1
    prec = tp / (tp + fp) if (tp + fp) else 1.0
    rec = tp / (tp + fn) if (tp + fn) else 1.0
    f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
    return dict(tp=tp, fp=fp, fn=fn, tn=tn, precision=prec, recall=rec, f1=f1)


 def cost_stats(runs):
    costs = [r["cost"] for r in runs if not r.get("error")]
    toks = [sum(r["usage"].values()) for r in runs if not r.get("error")]
    return dict(
        n=len(costs),
        cost_mean=st.mean(costs) if costs else 0,
        cost_total=sum(costs),
        tok_mean=st.mean(toks) if toks else 0,
        errors=sum(1 for r in runs if r.get("error")),
    )


 def quality_stats(graded):
    qs = [g["quality"] for g in graded if isinstance(g.get("quality"), (int, float))]
    cap = [g for g in graded if g.get("captures_signal") is True]
    fps = [g for g in graded if g.get("false_positive") is True]
    return dict(
        n=len(graded),
        quality_mean=st.mean(qs) if qs else 0,
        captured=len(cap),
        false_positives=len(fps),
    )


 bc, tc = cost_stats(R["baseline"]), cost_stats(R["treatment"])
 bp, tp_ = prf(Q["baseline"]), prf(Q["treatment"])
 bq, tq = quality_stats(Q["baseline"]), quality_stats(Q["treatment"])

 print("=" * 64)
 print("COST (per review fork, mean across all scenarios x reps)")
 print(f"  baseline (opus, full):   ${bc['cost_mean']:.5f}  total ${bc['cost_total']:.4f}  n={bc['n']} err={bc['errors']}")
 print(f"  treatment (haiku,digest):${tc['cost_mean']:.5f}  total ${tc['cost_total']:.4f}  n={tc['n']} err={tc['errors']}")
 if tc['cost_mean']:
    print(f"  → {bc['cost_mean']/tc['cost_mean']:.1f}x cheaper per review")
 print(f"  tokens/fork: baseline {bc['tok_mean']:.0f}  treatment {tc['tok_mean']:.0f}")
 print()
 print("QUALITY — save-decision accuracy vs fixture oracle")
 print(f"  baseline : P={bp['precision']:.2f} R={bp['recall']:.2f} F1={bp['f1']:.2f}  (tp{bp['tp']} fp{bp['fp']} fn{bp['fn']} tn{bp['tn']})")
 print(f"  treatment: P={tp_['precision']:.2f} R={tp_['recall']:.2f} F1={tp_['f1']:.2f}  (tp{tp_['tp']} fp{tp_['fp']} fn{tp_['fn']} tn{tp_['tn']})")
 print()
 print("QUALITY — LLM-judge edit grading")
 print(f"  baseline : mean quality {bq['quality_mean']:.2f}/5  captured {bq['captured']}/{bq['n']}  false-pos {bq['false_positives']}")
 print(f"  treatment: mean quality {tq['quality_mean']:.2f}/5  captured {tq['captured']}/{tq['n']}  false-pos {tq['false_positives']}")
 print("=" * 64)

 Path("/tmp/bgreview_bench/summary.json").write_text(json.dumps({
    "cost": {"baseline": bc, "treatment": tc},
    "prf": {"baseline": bp, "treatment": tp_},
    "quality": {"baseline": bq, "treatment": tq},
 }, indent=2))
diff --git a/cache_correction.py b/cache_correction.py
 """Production-cache cost correction (analytical, no new API calls).

 The live benchmarks ran each fork against a COLD cache (fresh parent per run),
 so BOTH arms paid cache-WRITE for the prompt. That hid the dominant production
 effect: in a real long-lived chat the parent history is ALREADY WARM, so:

  - OLD (full replay): the fork re-sends the identical history -> cache READ
    (cheap, ~0.1x of write). The cache was already paid for by prior turns.
  - DIGEST (same model): the fork sends digest+tail, a NOVEL prefix the cache
    has never seen -> cache WRITE for that prefix. Cutting history DESTROYS the
    warm-cache hit and re-writes fewer-but-uncached tokens at ~12.5x the read price.
  - ROUTING (Haiku): a different model can't use the opus parent cache at all,
    so the haiku fork is cold-prefix REGARDLESS of digest. Here cutting history
    is free (cache already lost by switching models), and the win is the ~15x
    cheaper per-token model.

 We reconstruct production cost from the measured token COUNTS (which are valid;
 only the cache *categorization* was wrong for the cold benchmark).
 """
 import json
 import statistics as st

 P = {
    "opus":  {"in": 15.0, "out": 75.0, "cache_read": 1.50, "cache_write": 18.75},
    "haiku": {"in": 1.0,  "out": 5.0,  "cache_read": 0.10, "cache_write": 1.25},
 }

 d = json.load(open("/tmp/bgreview_bench/hard_results.json"))


 def mean_tokens(arm):
    """Total input tokens the model SEES (fresh+read+write are disjoint prompt
    slices) and output tokens, averaged per run."""
    ins, outs = [], []
    for row in d["runs"]:
        r = row[arm]
        if r.get("error"):
            continue
        u = r["usage"]
        ins.append(u["input_tokens"] + u["cache_read_tokens"] + u["cache_write_tokens"])
        outs.append(u["output_tokens"])
    return st.mean(ins), st.mean(outs)


 full_in, full_out = mean_tokens("full")
 dig_in, dig_out = mean_tokens("digest")

 # How much of the full history does the digest keep? (sets how much is novel)
 keep_frac = dig_in / full_in
 novel_frac = 1.0  # the digest text is a freshly-composed prefix -> all novel

 print(f"measured tokens/run: full sees {full_in:.0f} in, digest sees {dig_in:.0f} in "
      f"({keep_frac*100:.0f}% of full)\n")

 # ---- SAME MODEL (opus), production cache ----
 p = P["opus"]
 # OLD: entire history warm -> all input is cache_read; tiny incremental write for the new turn ignored.
 old = full_in/1e6*p["cache_read"] + full_out/1e6*p["out"]
 # DIGEST: novel prefix. The fork runs ~multiple iterations, so the digest prefix is
 # written once then read on subsequent iterations within the fork. Model two bounds:
 #   worst: all digest input billed as write (single-iteration / no internal reuse)
 #   typical: write once, then ~60% re-read across the fork's own iterations
 dig_worst = dig_in/1e6*p["cache_write"] + dig_out/1e6*p["out"]
 dig_typ = (dig_in*0.4/1e6*p["cache_write"] + dig_in*0.6/1e6*p["cache_read"]) + dig_out/1e6*p["out"]

 print("=== SAME MODEL (Opus), PRODUCTION warm-parent cache ===")
 print(f"  OLD full replay (warm reads):   ${old:.4f}")
 print(f"  DIGEST (novel prefix, typical): ${dig_typ:.4f}   -> {dig_typ/old:.2f}x vs old  ({'CHEAPER' if dig_typ<old else 'MORE EXPENSIVE'})")
 print(f"  DIGEST (novel prefix, worst):   ${dig_worst:.4f}   -> {dig_worst/old:.2f}x vs old")
 print()

 # ---- ROUTING (Haiku review), production cache ----
 po, ph = P["opus"], P["haiku"]
 # OLD routing baseline is still the opus fork on the WARM parent cache:
 old_route = full_in/1e6*po["cache_read"] + full_out/1e6*po["out"]
 # Haiku fork: cannot reuse opus cache (different model) -> cold prefix on haiku.
 #   It sends the digest (fewer tokens) and writes-then-reads on haiku's own cache.
 hk_typ = (dig_in*0.4/1e6*ph["cache_write"] + dig_in*0.6/1e6*ph["cache_read"]) + dig_out/1e6*ph["out"]
 hk_worst = dig_in/1e6*ph["cache_write"] + dig_out/1e6*ph["out"]
 # For completeness: what if haiku replayed FULL history (no digest)? still cold on haiku.
 hk_full_typ = (full_in*0.4/1e6*ph["cache_write"] + full_in*0.6/1e6*ph["cache_read"]) + full_out/1e6*ph["out"]

 print("=== ROUTING (Opus parent, Haiku review), PRODUCTION cache ===")
 print(f"  OLD baseline = opus fork on warm parent cache:  ${old_route:.4f}")
 print(f"  HAIKU + digest (typical):   ${hk_typ:.4f}   -> {old_route/hk_typ:.1f}x cheaper than old")
 print(f"  HAIKU + digest (worst):     ${hk_worst:.4f}   -> {old_route/hk_worst:.1f}x cheaper than old")
 print(f"  HAIKU + full replay (no digest, for ref): ${hk_full_typ:.4f}  -> {old_route/hk_full_typ:.1f}x cheaper")
 print()
 print("Note: digest helps ROUTING (fewer cold tokens to write on haiku) but HURTS")
 print("same-model (destroys the warm opus cache). The two cases are opposite.")
diff --git a/digest.out b/digest.out
 Large scenario: 182 msgs, ~159501 tokens of content

  💾 Self-improvement review: Skill 'response-brevity' created.
 [full  ] rep0 cost=$0.15236 save=True in+cache=305662
  💾 Self-improvement review: Skill 'response-brevity' created.
 [digest] rep0 cost=$0.06417 save=True in+cache=147254
  💾 Self-improvement review: Skill 'response-conciseness' created.
 [full  ] rep1 cost=$0.16579 save=True in+cache=408628
  💾 Self-improvement review: Skill 'response-brevity' created.
 [digest] rep1 cost=$0.05204 save=True in+cache=72759
  💾 Self-improvement review: Skill 'terse-communication' created.
 [full  ] rep2 cost=$0.16185 save=True in+cache=406854
  💾 Self-improvement review: Skill 'response-brevity' created.
 [digest] rep2 cost=$0.05197 save=True in+cache=72751

 FULL mean $0.16000  DIGEST mean $0.05606  → 2.9x cheaper
 DIGEST captured signal (saved): 3/3
diff --git a/digest_bench.py b/digest_bench.py
 """Supplementary benchmark: isolate the DIGEST path (ideas ②③) on a LARGE session.

 The main harness scenarios are small, so the digest never engages — they prove
 idea ① (routing) only. This isolates ②③: take a save-worthy signal, bury it in
 a large filler transcript that exceeds the token budget, and compare on the
 SAME model:

  FULL   — replay the entire snapshot (current behaviour; James's pathological case)
  DIGEST — replay recent tail verbatim + a summary of older turns (this PR)

 Same model both arms → any cost delta is purely the digest, and we check the
 DIGEST arm still captures the signal (it lives in the recent tail).

 Run after the main harness. Uses claude-haiku (cheap) for both arms since the
 point here is the token/cost delta from trimming, not model price.
 """
 import os
 import sys
 import json
 import shutil
 import tempfile
 from pathlib import Path

 WORKTREE = "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8"
 sys.path.insert(0, WORKTREE)
 sys.path.insert(0, "/tmp/bgreview_bench")

 MODEL = "claude-haiku-4-5-20251001"
 PRICES = {"in": 1.00, "out": 5.00, "cache_read": 0.10, "cache_write": 1.25}
 REPS = int(os.environ.get("DIGEST_REPS", "3"))


 def _cost(u):
    return (u["input_tokens"]/1e6*PRICES["in"] + u["output_tokens"]/1e6*PRICES["out"]
            + u["cache_read_tokens"]/1e6*PRICES["cache_read"]
            + u["cache_write_tokens"]/1e6*PRICES["cache_write"])


 def _big_scenario():
    """A genuine skill signal in the TAIL, buried under ~60 filler turns."""
    msgs = []
    topics = ["refactoring the cache layer", "the CI matrix", "a flaky import",
              "the config loader", "docs phrasing", "a rename", "test fixtures"]
    for i in range(90):
        t = topics[i % len(topics)]
        msgs.append({"role": "user", "content":
            f"Quick question about {t}, iteration {i}. " + ("Some detailed context here. " * 90)})
        msgs.append({"role": "assistant", "content":
            f"Here's the answer for {t} #{i}. " + ("Detailed explanation follows. " * 90)})
    # The save-worthy signal, in the recent tail:
    msgs += [
        {"role": "user", "content":
            "ok different thing — you keep writing these giant verbose answers. "
            "stop. from now on give me one or two sentences max, always. i mean it."},
        {"role": "assistant", "content": "Understood — terse from here on."},
    ]
    return {
        "id": "large_session_style_correction",
        "expect_save": True,
        "signal": "User told the agent (in the recent tail, after a long session) "
                  "to always be terse — one or two sentences. A durable style preference.",
        "messages": msgs,
    }


 def run_arm(scenario, digest_on, api_key):
    hermes_home = tempfile.mkdtemp(prefix="dig_")
    hh = os.path.join(hermes_home, ".hermes")
    os.makedirs(hh, exist_ok=True)
    os.environ["HERMES_HOME"] = hh
    for mod in [m for m in sys.modules if m.startswith(("hermes", "agent", "run_agent", "tools", "model_tools"))]:
        del sys.modules[mod]
    sys.path.insert(0, WORKTREE)

    # Same model both arms. Route to the aux model (forces the digest branch);
    # toggle the digest by setting max_context_tokens very high (FULL) vs the
    # default 48000 (DIGEST). Routing alone keeps model price identical across
    # arms, isolating the trimming effect.
    import yaml
    cfg = {
        "model": {"provider": "anthropic", "model": MODEL, "default": MODEL},
        "auxiliary": {"background_review": {
            "provider": "anthropic", "model": MODEL,
            "max_context_tokens": (10_000_000 if not digest_on else 48000),
            "digest_tail_messages": 24,
        }},
        "memory": {"memory_enabled": True, "user_profile_enabled": True},
        "skills": {"creation_nudge_interval": 10},
    }
    Path(hh, "config.yaml").write_text(yaml.safe_dump(cfg))
    os.environ["ANTHROPIC_API_KEY"] = api_key

    from run_agent import AIAgent
    from agent import background_review as br
    import anthropic

    tally = {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0}

    def _add(u):
        if u is None:
            return
        tally["input_tokens"] += int(getattr(u, "input_tokens", 0) or 0)
        tally["output_tokens"] += int(getattr(u, "output_tokens", 0) or 0)
        tally["cache_read_tokens"] += int(getattr(u, "cache_read_input_tokens", 0) or 0)
        tally["cache_write_tokens"] += int(getattr(u, "cache_creation_input_tokens", 0) or 0)

    Messages = anthropic.resources.messages.Messages
    o_stream, o_create = Messages.stream, Messages.create

    class _P:
        def __init__(self, m): self._m, self._c = m, False
        def __enter__(self):
            s = self._m.__enter__(); og = s.get_final_message
            def g(*a, **k):
                msg = og(*a, **k)
                if not self._c:
                    self._c = True; _add(getattr(msg, "usage", None))
                return msg
            s.get_final_message = g; return s
        def __exit__(self, *a): return self._m.__exit__(*a)

    Messages.stream = lambda self, *a, **k: _P(o_stream(self, *a, **k))
    def _wc(self, *a, **k):
        r = o_create(self, *a, **k); _add(getattr(r, "usage", None)); return r
    Messages.create = _wc

    agent = AIAgent(model=MODEL, provider="anthropic", api_key=api_key,
                    api_mode="anthropic_messages", quiet_mode=True,
                    skip_context_files=True, skip_memory=True, platform="cli",
                    session_id="dig-parent")
    try:
        target, _ = br.spawn_background_review_thread(
            agent, messages_snapshot=list(scenario["messages"]),
            review_memory=True, review_skills=True)
        target()
    finally:
        Messages.stream, Messages.create = o_stream, o_create
        try:
            agent.close()
        except Exception:
            pass

    skills_dir = Path(hh) / "skills"
    saved = {
        "memory": (Path(hh, "MEMORY.md").read_text() if Path(hh, "MEMORY.md").exists() else ""),
        "user": (Path(hh, "USER.md").read_text() if Path(hh, "USER.md").exists() else ""),
        "skill_files": [str(p.relative_to(skills_dir)) for p in skills_dir.rglob("*")
                        if p.is_file() and ".archive" not in str(p)
                        and not str(p).endswith((".usage.json", ".lock"))] if skills_dir.exists() else [],
    }
    did_save = bool(saved["memory"].strip() or saved["user"].strip() or saved["skill_files"])
    shutil.rmtree(hermes_home, ignore_errors=True)
    return {"usage": dict(tally), "cost": _cost(tally), "did_save": did_save, "saved": saved}


 def main():
    api_key = os.environ["ANTHROPIC_API_KEY"]
    sc = _big_scenario()
    full_tok = sum(len(str(m.get("content", ""))) for m in sc["messages"]) // 3
    print(f"Large scenario: {len(sc['messages'])} msgs, ~{full_tok} tokens of content\n")
    out = {"full": [], "digest": [], "scenario": sc["id"], "signal": sc["signal"]}
    for rep in range(REPS):
        for arm, dig in (("full", False), ("digest", True)):
            r = run_arm(sc, dig, api_key)
            r["rep"] = rep
            out[arm].append(r)
            print(f"[{arm:6}] rep{rep} cost=${r['cost']:.5f} save={r['did_save']} "
                  f"in+cache={r['usage']['input_tokens']+r['usage']['cache_read_tokens']+r['usage']['cache_write_tokens']}")
    Path("/tmp/bgreview_bench/digest_results.json").write_text(json.dumps(out, indent=2))
    import statistics as stt
    fc = stt.mean(x["cost"] for x in out["full"])
    dc = stt.mean(x["cost"] for x in out["digest"])
    print(f"\nFULL mean ${fc:.5f}  DIGEST mean ${dc:.5f}  → {fc/dc:.1f}x cheaper")
    print(f"DIGEST captured signal (saved): {sum(x['did_save'] for x in out['digest'])}/{REPS}")


 if __name__ == "__main__":
    main()
diff --git a/digest_results.json b/digest_results.json
 {
  "full": [
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 808,
        "cache_read_tokens": 203262,
        "cache_write_tokens": 102384
      },
      "cost": 0.1523622,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "response-brevity/SKILL.md"
        ]
      },
      "rep": 0
    },
    {
      "usage": {
        "input_tokens": 20,
        "output_tokens": 1309,
        "cache_read_tokens": 305683,
        "cache_write_tokens": 102925
      },
      "cost": 0.16578955,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "communication/response-conciseness/SKILL.md"
        ]
      },
      "rep": 1
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 711,
        "cache_read_tokens": 304584,
        "cache_write_tokens": 102254
      },
      "cost": 0.16184690000000002,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "terse-communication/SKILL.md"
        ]
      },
      "rep": 2
    }
  ],
  "digest": [
    {
      "usage": {
        "input_tokens": 20,
        "output_tokens": 1248,
        "cache_read_tokens": 109683,
        "cache_write_tokens": 37551
      },
      "cost": 0.06416705,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "communication/response-brevity/SKILL.md"
        ]
      },
      "rep": 0
    },
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 507,
        "cache_read_tokens": 36032,
        "cache_write_tokens": 36717
      },
      "cost": 0.05204445,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "response-brevity/SKILL.md"
        ]
      },
      "rep": 1
    },
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 495,
        "cache_read_tokens": 36035,
        "cache_write_tokens": 36706
      },
      "cost": 0.051971,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          "communication/response-brevity/SKILL.md"
        ]
      },
      "rep": 2
    }
  ],
  "scenario": "large_session_style_correction",
  "signal": "User told the agent (in the recent tail, after a long session) to always be terse \u2014 one or two sentences. A durable style preference."
 }
diff --git a/hard.out b/hard.out
  💾 Self-improvement review: User profile updated · Skill 'response-style' created.
 [buried_early_style_correction      r0 full  ] cost=$4.3323 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'response-style-discipline' created.
 [buried_early_style_correction      r0 digest] cost=$2.0846 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'response-style-control' created.
 [buried_early_style_correction      r1 full  ] cost=$4.5304 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'honoring-standing-user-constraints' created.
 [buried_early_style_correction      r1 digest] cost=$1.9965 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'response-style' created.
 [buried_early_style_correction      r2 full  ] cost=$4.3227 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'response-style-adherence' created.
 [buried_early_style_correction      r2 digest] cost=$1.9905 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
 [buried_mid_workflow_rule           r0 full  ] cost=$4.1601 cap=1/1 distractor=None err=None
  💾 Self-improvement review: Skill 'git-commit-workflow' created. · User profile updated
 [buried_mid_workflow_rule           r0 digest] cost=$2.0312 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
 [buried_mid_workflow_rule           r1 full  ] cost=$4.1733 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'commit-workflow' created.
 [buried_mid_workflow_rule           r1 digest] cost=$2.0943 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
 [buried_mid_workflow_rule           r2 full  ] cost=$4.5172 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
 [buried_mid_workflow_rule           r2 digest] cost=$1.9198 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated
 [buried_early_persona_fact          r0 full  ] cost=$4.3584 cap=1/1 distractor=None err=None
  💾 Self-improvement review: Memory updated
 [buried_early_persona_fact          r0 digest] cost=$1.7843 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated
 [buried_early_persona_fact          r1 full  ] cost=$4.5543 cap=1/1 distractor=None err=None
  💾 Self-improvement review: Memory updated
 [buried_early_persona_fact          r1 digest] cost=$1.7803 cap=1/1 distractor=None err=None
  💾 Self-improvement review: User profile updated
 [buried_early_persona_fact          r2 full  ] cost=$4.3539 cap=1/1 distractor=None err=None
  💾 Self-improvement review: Memory updated
 [buried_early_persona_fact          r2 digest] cost=$1.8815 cap=1/1 distractor=None err=None
  💾 Self-improvement review: Skill 'debugging-subprocess-startup' created.
 [mid_technique_with_retracted_distractor r0 full  ] cost=$3.3596 cap=1/1 distractor=True err=None
  💾 Self-improvement review: Skill 'subprocess-startup-debugging' created.
 [mid_technique_with_retracted_distractor r0 digest] cost=$1.9333 cap=1/1 distractor=True err=None
  💾 Self-improvement review: Skill 'debugging-subprocess-startup' created.
 [mid_technique_with_retracted_distractor r1 full  ] cost=$3.3837 cap=1/1 distractor=True err=None
  💾 Self-improvement review: Skill 'debugging-subprocess-lifecycle' created.
 [mid_technique_with_retracted_distractor r1 digest] cost=$1.9117 cap=1/1 distractor=True err=None
  💾 Self-improvement review: Skill 'subprocess-lifecycle-debugging' created.
 [mid_technique_with_retracted_distractor r2 full  ] cost=$3.6094 cap=1/1 distractor=True err=None
  💾 Self-improvement review: Skill 'debugging-subprocess-spawn' created.
 [mid_technique_with_retracted_distractor r2 digest] cost=$1.9391 cap=1/1 distractor=True err=None
  💾 Self-improvement review: User profile updated
 [multi_signal_session               r0 full  ] cost=$3.4050 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated · Memory updated · Skill 'python-linting' created.
 [multi_signal_session               r0 digest] cost=$2.0442 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'python-linting-formatting' created.
 [multi_signal_session               r1 full  ] cost=$3.6372 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated · Memory updated
 [multi_signal_session               r1 digest] cost=$1.8918 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'python-lint-format' created.
 [multi_signal_session               r2 full  ] cost=$3.6079 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated · Skill 'python-linting' created.
 [multi_signal_session               r2 digest] cost=$1.9342 cap=2/2 distractor=None err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r0 full  ] cost=$2.6726 cap=1/1 distractor=False err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r0 digest] cost=$1.6533 cap=1/1 distractor=False err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r1 full  ] cost=$2.6620 cap=1/1 distractor=False err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r1 digest] cost=$1.7590 cap=1/1 distractor=False err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r2 full  ] cost=$2.6657 cap=1/1 distractor=False err=None
  💾 Self-improvement review: User profile updated
 [durable_vs_trivia_memory           r2 digest] cost=$1.7559 cap=1/1 distractor=False err=None
 [long_noop_smooth                   r0 full  ] cost=$1.9358 cap=0/0 distractor=None err=None
 [long_noop_smooth                   r0 digest] cost=$1.0411 cap=0/0 distractor=None err=None
 [long_noop_smooth                   r1 full  ] cost=$1.9339 cap=0/0 distractor=None err=None
 [long_noop_smooth                   r1 digest] cost=$1.0446 cap=0/0 distractor=None err=None
 [long_noop_smooth                   r2 full  ] cost=$1.9443 cap=0/0 distractor=None err=None
 [long_noop_smooth                   r2 digest] cost=$1.0443 cap=0/0 distractor=None err=None

 Wrote hard_results.json
diff --git a/hard_harness.py b/hard_harness.py
 """Intense same-model A/B: Opus-4.8 (full replay) vs Opus-4.8 (signal-aware digest).

 Model price is OUT of the equation — both arms run claude-opus-4-8. So:
  • any COST delta is purely the digest replaying fewer tokens, and
  • any QUALITY delta is purely the digest's signal-foregrounding changing what
    the SAME model captures.

 Per scenario × rep × arm we capture real wire usage + the saved artifacts, then
 score quality three ways:
  1. keyword capture: did the saved memory/skill contain the ground-truth signal
     keywords? (per-signal recall; distractor must_not_contain = false-save check)
  2. blind pairwise judge: opus judge sees BOTH arms' saves unlabeled (order
     randomized) and picks which better captures the ground-truth signal, or tie.
  3. memory-durability rubric: judge rates whether the saved memory is the
     durable/important fact vs trivia.

 Output: /tmp/bgreview_bench/hard_results.json
 """
 import os
 import sys
 import json
 import time
 import random
 import shutil
 import tempfile
 from pathlib import Path

 WORKTREE = "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8"
 sys.path.insert(0, WORKTREE)
 sys.path.insert(0, "/tmp/bgreview_bench")

 MODEL = "claude-opus-4-8"
 REPS = int(os.environ.get("HARD_REPS", "4"))
 PRICES_BY_MODEL = {
    "claude-opus-4-8":           {"in": 15.00, "out": 75.00, "cache_read": 1.50, "cache_write": 18.75},
    "claude-haiku-4-5-20251001": {"in": 1.00,  "out": 5.00,  "cache_read": 0.10, "cache_write": 1.25},
 }
 PRICES = PRICES_BY_MODEL["claude-opus-4-8"]  # back-compat default


 def _cost(u, model="claude-opus-4-8"):
    p = PRICES_BY_MODEL.get(model, PRICES)
    return (u["input_tokens"]/1e6*p["in"] + u["output_tokens"]/1e6*p["out"]
            + u["cache_read_tokens"]/1e6*p["cache_read"]
            + u["cache_write_tokens"]/1e6*p["cache_write"])


 def run_arm(scenario, digest_on, api_key, prepass=False, review_model=None):
    hermes_home = tempfile.mkdtemp(prefix="hard_")
    hh = os.path.join(hermes_home, ".hermes")
    os.makedirs(hh, exist_ok=True)
    os.environ["HERMES_HOME"] = hh
    for mod in [m for m in sys.modules if m.startswith(("hermes", "agent", "run_agent", "tools", "model_tools"))]:
        del sys.modules[mod]
    sys.path.insert(0, WORKTREE)

    import yaml
    # The parent (main) model is always MODEL (opus). review_model lets the
    # REVIEW FORK route to a different model (e.g. haiku) — the routing arm —
    # while keeping the parent on opus. None = review runs on the main model.
    rev_model = review_model or MODEL
    br_cfg = {
        "provider": "anthropic", "model": rev_model,
        "max_context_tokens": (10_000_000 if not digest_on else 48000),
        "digest_tail_messages": 24,
        "prepass": prepass,
    }
    cfg = {
        "model": {"provider": "anthropic", "model": MODEL, "default": MODEL},
        "auxiliary": {"background_review": br_cfg},
        "memory": {"memory_enabled": True, "user_profile_enabled": True},
        "skills": {"creation_nudge_interval": 10},
    }
    Path(hh, "config.yaml").write_text(yaml.safe_dump(cfg))
    os.environ["ANTHROPIC_API_KEY"] = api_key

    from run_agent import AIAgent
    from agent import background_review as br
    import anthropic

    tally = {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0}

    def _add(u):
        if u is None:
            return
        tally["input_tokens"] += int(getattr(u, "input_tokens", 0) or 0)
        tally["output_tokens"] += int(getattr(u, "output_tokens", 0) or 0)
        tally["cache_read_tokens"] += int(getattr(u, "cache_read_input_tokens", 0) or 0)
        tally["cache_write_tokens"] += int(getattr(u, "cache_creation_input_tokens", 0) or 0)

    Messages = anthropic.resources.messages.Messages
    o_stream, o_create = Messages.stream, Messages.create

    class _P:
        def __init__(self, m): self._m, self._c = m, False
        def __enter__(self):
            s = self._m.__enter__(); og = s.get_final_message
            def g(*a, **k):
                msg = og(*a, **k)
                if not self._c:
                    self._c = True; _add(getattr(msg, "usage", None))
                return msg
            s.get_final_message = g; return s
        def __exit__(self, *a): return self._m.__exit__(*a)

    Messages.stream = lambda self, *a, **k: _P(o_stream(self, *a, **k))
    def _wc(self, *a, **k):
        r = o_create(self, *a, **k); _add(getattr(r, "usage", None)); return r
    Messages.create = _wc

    agent = AIAgent(model=MODEL, provider="anthropic", api_key=api_key,
                    api_mode="anthropic_messages", quiet_mode=True,
                    skip_context_files=True, skip_memory=False, platform="cli",
                    session_id="hard-parent")
    try:
        agent._cached_system_prompt = agent._build_system_prompt()
    except Exception:
        agent._cached_system_prompt = "HARD-PARENT-SYS"
    try:
        target, _ = br.spawn_background_review_thread(
            agent, messages_snapshot=list(scenario["messages"]),
            review_memory=True, review_skills=True)
        target()
    finally:
        Messages.stream, Messages.create = o_stream, o_create
        try:
            agent.close()
        except Exception:
            pass

    skills_dir = Path(hh) / "skills"
    mem_dir = Path(hh) / "memories"
    skill_text = ""
    skill_files = []
    if skills_dir.exists():
        for p in skills_dir.rglob("*"):
            if p.is_file() and ".archive" not in str(p) and not str(p).endswith((".usage.json", ".lock")):
                skill_files.append(str(p.relative_to(skills_dir)))
                try:
                    skill_text += "\n" + p.read_text()
                except Exception:
                    pass
    saved = {
        "memory": ((mem_dir / "MEMORY.md").read_text() if (mem_dir / "MEMORY.md").exists() else ""),
        "user": ((mem_dir / "USER.md").read_text() if (mem_dir / "USER.md").exists() else ""),
        "skill_files": skill_files,
        "skill_text": skill_text[:4000],
    }
    blob = (saved["memory"] + "\n" + saved["user"] + "\n" + saved["skill_text"]).lower()

    # Keyword capture scoring (objective, per signal).
    sig_results = []
    for s in scenario.get("signals", []):
        hit = any(k.lower() in blob for k in s["must_contain"])
        sig_results.append({"kind": s["kind"], "desc": s["desc"], "captured": hit})
    # Distractor false-save check.
    distractor_saved = None
    if scenario.get("distractor"):
        distractor_saved = any(k.lower() in blob for k in scenario["distractor"]["must_not_contain"])

    shutil.rmtree(hermes_home, ignore_errors=True)
    return {
        "usage": dict(tally), "cost": _cost(tally, rev_model),
        "review_model": rev_model,
        "did_save": bool(blob.strip()), "saved": saved,
        "signal_capture": sig_results, "distractor_saved": distractor_saved,
    }


 def main():
    import hard_scenarios as HS
    api_key = os.environ["ANTHROPIC_API_KEY"]
    out = {"runs": [], "meta": {"reps": REPS, "model": MODEL, "prices": PRICES}}
    for sc in HS.SCENARIOS:
        for rep in range(REPS):
            row = {"scenario": sc["id"], "rep": rep, "position": sc.get("position"),
                   "n_signals": len(sc.get("signals", [])),
                   "expect_noop": len(sc.get("signals", [])) == 0}
            for arm, dig in (("full", False), ("digest", True)):
                t0 = time.time()
                try:
                    r = run_arm(sc, dig, api_key)
                    r["error"] = None
                except Exception as e:
                    import traceback
                    r = {"error": str(e), "trace": traceback.format_exc()[-600:],
                         "usage": {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0},
                         "cost": 0, "did_save": None, "saved": {}, "signal_capture": [], "distractor_saved": None}
                r["latency_s"] = round(time.time() - t0, 1)
                row[arm] = r
                cap = sum(1 for s in r["signal_capture"] if s["captured"])
                tot = len(r["signal_capture"])
                print(f"[{sc['id']:34} r{rep} {arm:6}] cost=${r['cost']:.4f} "
                      f"cap={cap}/{tot} distractor={r['distractor_saved']} err={r['error']}", flush=True)
            out["runs"].append(row)
    Path("/tmp/bgreview_bench/hard_results.json").write_text(json.dumps(out, indent=2))
    print("\nWrote hard_results.json")


 if __name__ == "__main__":
    main()
diff --git a/hard_judge.out b/hard_judge.out
  pairwise buried_early_style_correction      r0 → full
  pairwise buried_early_style_correction      r1 → digest
  pairwise buried_early_style_correction      r2 → full
  pairwise buried_mid_workflow_rule           r0 → full
  pairwise buried_mid_workflow_rule           r1 → digest
  pairwise buried_mid_workflow_rule           r2 → full
  pairwise buried_early_persona_fact          r0 → tie
  pairwise buried_early_persona_fact          r1 → tie
  pairwise buried_early_persona_fact          r2 → tie
  pairwise mid_technique_with_retracted_distractor r0 → full
  pairwise mid_technique_with_retracted_distractor r1 → full
  pairwise mid_technique_with_retracted_distractor r2 → full
  pairwise multi_signal_session               r0 → digest
  pairwise multi_signal_session               r1 → full
  pairwise multi_signal_session               r2 → full
  pairwise durable_vs_trivia_memory           r0 → digest
  pairwise durable_vs_trivia_memory           r1 → tie
  pairwise durable_vs_trivia_memory           r2 → tie

 ==================================================================
 SAME-MODEL A/B (Opus-4.8 both arms) — cost is purely the digest
  cost/review:  full $3.5295   digest $1.7864   → 2.0x cheaper
  tokens/review: full 305468   digest 172434
  total spend:  full $74.12   digest $37.52

 QUALITY — objective keyword capture (recall of ground-truth signals)
  full:   21/21 = 1.00
  digest: 21/21 = 1.00
  distractor false-saves: full 3  digest 3 (of 6 distractor runs each)
  no-op scenarios correct: full 3/3  digest 3/3

 QUALITY — blind pairwise judge (which arm captured the signal better)
  digest wins: 4   full wins: 9   tie: 5   parse_fail: 0
 ==================================================================
diff --git a/hard_judge.py b/hard_judge.py
 """Blind pairwise judge + aggregation for the intense same-model A/B.

 Reads hard_results.json. For each scenario×rep with ≥1 ground-truth signal,
 shows the opus judge BOTH arms' saved artifacts UNLABELED in randomized order
 and asks which better captures the ground-truth signal (or tie). Blind + order-
 randomized to remove position/label bias. Also computes the objective keyword-
 capture recall and distractor false-save rate per arm.

 Outputs hard_summary.json + prints the PR-ready tables.
 """
 import os
 import sys
 import json
 import random
 import statistics as st
 from pathlib import Path

 R = json.loads(Path("/tmp/bgreview_bench/hard_results.json").read_text())

 PAIR_PROMPT = """Two background-review passes each looked at the same conversation \
 and saved memory/skill notes. Below are their saved artifacts, labeled A and B \
 (order is randomized; you don't know which system produced which).

 GROUND-TRUTH the review should have captured:
 {signals}

 --- ARTIFACT A ---
 {a}

 --- ARTIFACT B ---
 {b}

 Judge ONLY on fidelity to the ground-truth: does the artifact capture the actual \
 rule/fact correctly, completely, and durably (will it still be useful next \
 session)? IGNORE length, prose polish, formatting, and verbosity — a shorter \
 artifact that captures the signal just as faithfully is EQUAL, not worse. Only \
 prefer one over the other if it captures the ground-truth signal MORE accurately \
 or would mislead LESS. If both capture it faithfully, answer "tie".

 Reply with ONLY compact JSON:
 {{"winner": "A"|"B"|"tie", "reason": "<one sentence on signal fidelity, not style>"}}"""


 def _artifact(saved):
    parts = []
    if saved.get("user", "").strip():
        parts.append("USER PROFILE:\n" + saved["user"][:1200])
    if saved.get("memory", "").strip():
        parts.append("MEMORY:\n" + saved["memory"][:1200])
    if saved.get("skill_text", "").strip():
        parts.append("SKILL(S):\n" + saved["skill_text"][:1800])
    return "\n\n".join(parts) or "(nothing saved)"


 def pairwise(client, signals, full_saved, digest_saved):
    sig_txt = "\n".join(f"  • [{s['kind']}] {s['desc']}" for s in signals)
    # Randomize which arm is A vs B to kill position bias.
    flip = random.random() < 0.5
    a_saved, b_saved = (digest_saved, full_saved) if flip else (full_saved, digest_saved)
    # A is full, B is digest UNLESS flipped.
    a_is_full = not flip
    msg = client.messages.create(
        model="claude-opus-4-8", max_tokens=300,
        messages=[{"role": "user", "content": PAIR_PROMPT.format(
            signals=sig_txt, a=_artifact(a_saved), b=_artifact(b_saved))}],
    )
    text = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text").strip()
    if text.startswith("```"):
        text = text.split("```")[1].lstrip("json").strip()
    try:
        verdict = json.loads(text)
    except Exception:
        return {"winner_arm": "parse_fail", "raw": text[:120]}
    w = verdict.get("winner")
    if w == "tie":
        return {"winner_arm": "tie", "reason": verdict.get("reason", "")}
    won_full = (w == "A" and a_is_full) or (w == "B" and not a_is_full)
    return {"winner_arm": "full" if won_full else "digest", "reason": verdict.get("reason", "")}


 def main():
    import anthropic
    client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])

    # Objective capture + cost.
    cost = {"full": [], "digest": []}
    tok = {"full": [], "digest": []}
    sig_total = {"full": 0, "digest": 0}
    sig_hit = {"full": 0, "digest": 0}
    distractor_bad = {"full": 0, "digest": 0}
    distractor_n = 0
    noop_correct = {"full": 0, "digest": 0}
    noop_n = 0
    pair = {"full": 0, "digest": 0, "tie": 0, "parse_fail": 0}

    for row in R["runs"]:
        for arm in ("full", "digest"):
            r = row[arm]
            if r.get("error"):
                continue
            cost[arm].append(r["cost"])
            tok[arm].append(sum(r["usage"].values()))
            for s in r["signal_capture"]:
                sig_total[arm] += 1
                if s["captured"]:
                    sig_hit[arm] += 1
            if r.get("distractor_saved") is not None and arm == "full":
                distractor_n += 1
            if r.get("distractor_saved") is True:
                distractor_bad[arm] += 1
            if row["expect_noop"] and arm == "full":
                noop_n += 1
            if row["expect_noop"] and not r["did_save"]:
                noop_correct[arm] += 1
        # Blind pairwise on scenarios with signals.
        if row["n_signals"] > 0 and not row["full"].get("error") and not row["digest"].get("error"):
            import hard_scenarios as HS
            sc = next(s for s in HS.SCENARIOS if s["id"] == row["scenario"])
            v = pairwise(client, sc["signals"], row["full"]["saved"], row["digest"]["saved"])
            pair[v["winner_arm"]] = pair.get(v["winner_arm"], 0) + 1
            print(f"  pairwise {row['scenario']:34} r{row['rep']} → {v['winner_arm']}", flush=True)

    def m(x):
        return st.mean(x) if x else 0

    print("\n" + "=" * 66)
    print("SAME-MODEL A/B (Opus-4.8 both arms) — cost is purely the digest")
    print(f"  cost/review:  full ${m(cost['full']):.4f}   digest ${m(cost['digest']):.4f}"
          f"   → {m(cost['full'])/max(m(cost['digest']),1e-9):.1f}x cheaper")
    print(f"  tokens/review: full {m(tok['full']):.0f}   digest {m(tok['digest']):.0f}")
    print(f"  total spend:  full ${sum(cost['full']):.2f}   digest ${sum(cost['digest']):.2f}")
    print("\nQUALITY — objective keyword capture (recall of ground-truth signals)")
    print(f"  full:   {sig_hit['full']}/{sig_total['full']} = {sig_hit['full']/max(sig_total['full'],1):.2f}")
    print(f"  digest: {sig_hit['digest']}/{sig_total['digest']} = {sig_hit['digest']/max(sig_total['digest'],1):.2f}")
    print(f"  distractor false-saves: full {distractor_bad['full']}  digest {distractor_bad['digest']} (of {distractor_n} distractor runs each)")
    print(f"  no-op scenarios correct: full {noop_correct['full']}/{noop_n}  digest {noop_correct['digest']}/{noop_n}")
    print("\nQUALITY — blind pairwise judge (which arm captured the signal better)")
    print(f"  digest wins: {pair['digest']}   full wins: {pair['full']}   tie: {pair['tie']}   parse_fail: {pair.get('parse_fail',0)}")
    print("=" * 66)

    Path("/tmp/bgreview_bench/hard_summary.json").write_text(json.dumps({
        "cost": {k: {"mean": m(v), "total": sum(v), "n": len(v)} for k, v in cost.items()},
        "tokens": {k: m(v) for k, v in tok.items()},
        "capture": {k: {"hit": sig_hit[k], "total": sig_total[k]} for k in ("full", "digest")},
        "distractor_false_saves": distractor_bad, "distractor_n": distractor_n,
        "noop_correct": noop_correct, "noop_n": noop_n,
        "pairwise": pair,
    }, indent=2))


 if __name__ == "__main__":
    sys.path.insert(0, "/tmp/bgreview_bench")
    sys.path.insert(0, "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8")
    main()
diff --git a/hard_judge2.out b/hard_judge2.out
  pairwise buried_early_style_correction      r0 → tie
  pairwise buried_early_style_correction      r1 → tie
  pairwise buried_early_style_correction      r2 → tie
  pairwise buried_mid_workflow_rule           r0 → tie
  pairwise buried_mid_workflow_rule           r1 → tie
  pairwise buried_mid_workflow_rule           r2 → tie
  pairwise buried_early_persona_fact          r0 → tie
  pairwise buried_early_persona_fact          r1 → tie
  pairwise buried_early_persona_fact          r2 → tie
  pairwise mid_technique_with_retracted_distractor r0 → full
  pairwise mid_technique_with_retracted_distractor r1 → tie
  pairwise mid_technique_with_retracted_distractor r2 → full
  pairwise multi_signal_session               r0 → tie
  pairwise multi_signal_session               r1 → full
  pairwise multi_signal_session               r2 → tie
  pairwise durable_vs_trivia_memory           r0 → tie
  pairwise durable_vs_trivia_memory           r1 → tie
  pairwise durable_vs_trivia_memory           r2 → tie

 ==================================================================
 SAME-MODEL A/B (Opus-4.8 both arms) — cost is purely the digest
  cost/review:  full $3.5295   digest $1.7864   → 2.0x cheaper
  tokens/review: full 305468   digest 172434
  total spend:  full $74.12   digest $37.52

 QUALITY — objective keyword capture (recall of ground-truth signals)
  full:   21/21 = 1.00
  digest: 21/21 = 1.00
  distractor false-saves: full 3  digest 3 (of 6 distractor runs each)
  no-op scenarios correct: full 3/3  digest 3/3

 QUALITY — blind pairwise judge (which arm captured the signal better)
  digest wins: 0   full wins: 3   tie: 15   parse_fail: 0
 ==================================================================
diff --git a/hard_results.json b/hard_results.json
 {
  "runs": [
    {
      "scenario": "buried_early_style_correction",
      "rep": 0,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1057,
          "cache_read_tokens": 249790,
          "cache_write_tokens": 206837
        },
        "cost": 4.33227375,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers terse answers: 1-2 sentences max, no preamble, no recap. Stated this as a standing rule (\"I mean it, always\"). Verbosity is a recurring frustration.",
          "skill_files": [
            "communication/response-style/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style\ndescription: How to calibrate response length, format, and tone to user preferences. Load when a user states or implies a preference about verbosity, formatting, or communication style.\n---\n\n# Response Style\n\nGoverns how to match the user's stated communication preferences. The default failure mode is over-explaining; the fix is to honor brevity requests literally and persistently.\n\n## Core rules\n\n1. When a user requests brevity, treat it as a HARD, STANDING constraint \u2014 not a one-turn hint. Phrases like \"be terse\", \"one or two sentences max\", \"no preamble\", \"I mean it, always\" mean every subsequent answer must obey, for the rest of the session.\n2. Do not open with preamble (\"Here's the rundown on...\", \"Great question...\") or close with recap when brevity is requested. Answer the question directly, then stop.\n3. Match the answer's length to the request, NOT to the length of the user's message. A long, padded question does not license a long answer \u2014 many users pad context but still want a one-line reply.\n4. If a genuinely complex answer cannot fit the stated limit, give the short answer first, then offer to expand (\"Want the details?\") rather than ignoring the limit.\n5. CLI/terminal context: plain text, no markdown decoration, no MEDIA: tags.\n\n## Pitfalls\n\n- Reverting to verbosity after a few turns. The constraint does not decay; re-read it as active on every turn.\n- Mirroring the user's verbosity when they send long messages \u2014 this is the most common trap. Length of input != desired length of output.\n- Adding \"just to be thorough\" expansions the user did not ask for.\n\n## Verification\n\nBefore sending: if the user asked for N sentences, count them. If over, cut.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 30.6
      },
      "digest": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1250,
          "cache_read_tokens": 189862,
          "cache_write_tokens": 90983
        },
        "cost": 2.08462425,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers extreme terseness: every answer 1-2 sentences max, no preamble, no recap. Stated emphatically (\"I mean it, always\") as a standing rule for all responses.",
          "skill_files": [
            "communication/response-style-discipline/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style-discipline\ndescription: Honor a user's stated response-style constraints (verbosity, format, tone) on EVERY turn, including under long repetitive conversations where the original instruction has scrolled out of view.\n---\n\n# Response Style Discipline\n\nWhen a user states how they want responses shaped \u2014 length, verbosity, format, tone \u2014 that is a STANDING constraint that applies to every subsequent turn, not just the next one. The failure mode this skill guards against: acknowledging the preference once, then drifting back to old habits as the conversation continues.\n\n## Trigger\nAny user statement like:\n- \"keep answers short / one or two sentences max / no preamble / no recap\"\n- \"stop explaining so much\" / \"you're too verbose\"\n- \"always do X\" / \"from now on\" / \"I mean it, always\"\n- format demands: \"just give me the command\", \"no markdown\", \"bullet points only\"\n\n## Rules\n1. Treat the constraint as PERMANENT until the user revokes it. It does not expire because the topic changed or many turns passed.\n2. Save it to user memory immediately so it survives context summarization (long sessions get digested; the original instruction can vanish from view).\n3. Re-check your draft against the constraint on every turn. If the rule is \"1-2 sentences,\" count them. If \"no preamble,\" delete the opening framing sentence.\n4. When a digest/summary scrolls the original instruction away, the memory entry is your backstop \u2014 trust it.\n\n## Pitfalls\n- Verbosity creep: each individual answer feels \"only slightly long,\" but across a session you've abandoned the rule entirely. Hold the line per-turn.\n- Don't add hedges, recaps, or \"here's the rundown on X\" lead-ins when terseness was requested \u2014 those are exactly the preamble the user rejected.\n- Acknowledging the preference (\"understood, terse from here on\") and then not following it is worse than not acknowledging at all.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 29.3
      }
    },
    {
      "scenario": "buried_early_style_correction",
      "rep": 1,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1341,
          "cache_read_tokens": 364062,
          "cache_write_tokens": 207122
        },
        "cost": 4.5303555,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers terse answers: one or two sentences max, no preamble, no recap. Stated as a standing rule (\"always\"). Verbosity is a recurring irritation.",
          "skill_files": [
            "communication/response-style-control/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style-control\ndescription: How to honor a user's stated response-style preferences (verbosity, length, format, tone) consistently across an entire session. Load when a user sets a standing rule about how replies should look.\n---\n\n# Response Style Control\n\nWhen a user states a preference about HOW you reply \u2014 length, verbosity, tone, format \u2014 it is a STANDING constraint that applies to every subsequent turn, not just the next one. Long sessions are where this slips: the rule is given once at the top and then quietly violated for the rest of the conversation.\n\n## Core rule\nA style instruction given once persists for the whole session. \"Always\", \"from now on\", \"every answer\" = hard constraint. Re-read it as binding on turn 40 exactly as on turn 1.\n\n## When the user says \"be terse / one or two sentences / no preamble\"\n- Answer in 1-2 sentences. No \"Here's the rundown on...\", no recap of their question, no restating context back at them.\n- Do not pad. If the real answer is one sentence, send one sentence.\n- Length of the user's message does NOT license a long reply. A wall-of-text question still gets a terse answer when terseness was requested.\n- Get to the actual content immediately. The first words should be the answer, not a framing clause.\n\n## Pitfalls\n- Preamble creep: openers like \"Here's the rundown on X\" / \"Great question\" / \"So, regarding Y\" are exactly the preamble the user banned. Cut them.\n- Recap creep: echoing the user's question back before answering wastes the budget. Skip it.\n- Drift over a long session: the constraint does not weaken because many turns passed or the topic changed. If you catch yourself expanding, stop and compress.\n- Treating the preference as a one-turn request rather than a standing rule.\n\n## Verification\nBefore sending: count sentences against the stated cap. If over, cut. Check the opening \u2014 if it's a framing/recap clause rather than substance, delete it.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 37.4
      },
      "digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1217,
          "cache_read_tokens": 133619,
          "cache_write_tokens": 90914
        },
        "cost": 1.9964609999999998,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Demands extreme brevity: every answer must be one or two sentences max, no preamble, no recap. Stated this emphatically (\"I mean it, always\") as a standing rule.",
          "skill_files": [
            "interaction/honoring-standing-user-constraints/SKILL.md"
          ],
          "skill_text": "\n---\nname: honoring-standing-user-constraints\ndescription: How to obey a user's standing behavioral constraints (brevity, tone, format, language) across an entire session, especially long multi-turn ones where the original instruction scrolls out of recent context.\n---\n\n# Honoring Standing User Constraints\n\nWhen a user sets a standing rule about HOW you respond \u2014 verbosity, tone, format, language, \"always/never do X\" \u2014 it governs every subsequent turn until they retract it. The most common failure: obeying for one or two turns, then drifting back to default behavior once the instruction is no longer in recent context.\n\n## Trigger\nUser says something like: \"from now on...\", \"every answer...\", \"always/never...\", \"I mean it\", \"stop doing X\", \"keep it short\", \"one or two sentences max\". Treat emphatic phrasing (\"I mean it, always\") as a hard contract, not a suggestion.\n\n## Steps\n1. On receiving a standing-style constraint, save it to USER memory immediately so it survives context compaction and future sessions.\n2. Acknowledge in a single short line \u2014 do not pad the acknowledgement itself.\n3. On EVERY subsequent turn, re-check the response against the constraint BEFORE sending. Ask: \"Does this violate the standing rule?\" A long session digest may summarize the rule away from recent turns \u2014 the rule still binds.\n4. If a turn genuinely needs more length (e.g. a code block, a list the user asked for), keep prose minimal and let the requested artifact be the length, not your commentary.\n\n## Pitfalls\n- Verbosity drift: the #1 failure. Repetitive or boilerplate user turns (\"Quick one about X...\") do NOT license long answers. Keep answers tight no matter how the user phrases the question.\n- Don't re-explain or recap across turns \u2014 a brevity constraint forbids restating context the user already has.\n- Don't let a summarized/digested conversation lull you into ignoring an early instruction. Early instructions are still active instructions.\n- Acknowledging the rule once is not enough; compliance is per-turn and ongoing.\n\n## Verification\nBefore sending: count sentences / scan length against the stated limit. If over, cut to fit.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 30.0
      }
    },
    {
      "scenario": "buried_early_style_correction",
      "rep": 2,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 954,
          "cache_read_tokens": 249739,
          "cache_write_tokens": 206745
        },
        "cost": 4.322747250000001,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers extremely terse replies \u2014 one or two sentences max, no preamble, no recap. Stated this as a standing rule (\"always\"). Verbosity is a recurring frustration.",
          "skill_files": [
            "communication/response-style/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style\ndescription: How to calibrate response length, tone, and formatting to the user's stated preferences. Load when the user comments on verbosity, format, tone, or legibility.\n---\n\n# Response Style\n\nGovern how every reply is shaped to match the user's communication preferences.\n\n## Core rule: honor brevity directives literally and persistently\n\nWhen the user asks for short answers, they mean it for EVERY turn \u2014 not just the next one. A directive like \"one or two sentences max, no preamble, no recap, always\" is a standing constraint that does not expire because the conversation continues or the topics change.\n\n### Steps\n1. When the user states a length/format preference, treat it as binding for the entire session and beyond (also save to memory).\n2. Before sending, check the reply against the directive: count sentences if a sentence cap was given.\n3. Strip preamble (\"Here's the rundown on...\", \"Great question...\") and recap/summary tails. Answer directly.\n4. If the answer genuinely cannot fit the cap, give the shortest correct answer and offer to expand \u2014 do not silently overflow.\n\n## Pitfalls\n- Drifting back to verbose output after a few turns. The most common failure: complying once, then reverting. The rule persists; re-read it each turn.\n- Padding with filler sentences. If content repeats, cut it.\n- Treating long/complex user input as license for long output. Input length does not set output length; the user's stated preference does.\n- Markdown in CLI contexts: prefer plain text renderable in a terminal unless the user wants markdown.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 27.9
      },
      "digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1158,
          "cache_read_tokens": 133655,
          "cache_write_tokens": 90828
        },
        "cost": 1.9904775,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Demands maximum brevity: every answer one or two sentences max, no preamble, no recap. Stated emphatically (\"I mean it, always\") \u2014 treat as a hard standing rule across all topics.",
          "skill_files": [
            "communication/response-style-adherence/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style-adherence\ndescription: How to honor a user's standing output-style constraints (verbosity, tone, format) on EVERY turn, not just the turn they state them. Use whenever a user states a persistent preference about how you should respond.\n---\n\n# Response Style Adherence\n\nWhen a user states how they want responses shaped \u2014 length, tone, format, no-preamble \u2014 it is a STANDING rule, not a one-turn request. It persists for the entire session and across sessions (save to memory too).\n\n## Core rule\nA style constraint stated once applies to ALL subsequent turns regardless of topic. Do not let a long or technical question \"reset\" you back to verbose defaults. The constraint outranks your instinct to be thorough.\n\n## When the user demands brevity (most common)\n- \"one or two sentences max\", \"no preamble\", \"no recap\", \"stop being verbose\" \u2192 answer in 1-2 sentences. Lead with the answer. No \"Here's the rundown on\u2026\", no restating the question, no closing summary.\n- A long, detailed question does NOT license a long answer. Match the requested OUTPUT length, not the input length.\n- If the answer genuinely needs more room, give the 1-2 sentence answer first, then ask \"want me to expand?\" \u2014 don't unilaterally override the constraint.\n\n## Pitfalls\n- Drift: obeying for a few turns then sliding back to verbose templates. Re-check the constraint on every turn; emphatic phrasing (\"I mean it, always\") means zero tolerance for drift.\n- Treating the preference as topic-scoped. It is global \u2014 it applies to CI questions, docs phrasing, refactors, everything.\n- Padding boilerplate (\"Here's the rundown on X. Explanation continues\u2026\") is exactly what brevity-demanding users hate. Cut it entirely.\n\n## Verification\nBefore sending: count sentences against the stated limit. If over, cut to the answer and nothing else.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 31.2
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 0,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1036,
          "cache_read_tokens": 135882,
          "cache_write_tokens": 206852
        },
        "cost": 4.160088,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing workflow rule: ALWAYS run the full test suite BEFORE committing, never after. The required sequence is tests \u2192 then commit. User has corrected this more than once.",
          "skill_files": [
            "devops/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to commit code changes in this user's repos \u2014 the required ordering of test/lint/commit steps and verification before committing.\n---\n\n# Git commit workflow\n\nGoverns any task that ends in a `git commit`: feature work, bug fixes, refactors, renames, dependency bumps, docs/changelog edits.\n\n## Hard rule: tests BEFORE commit, never after\n\nALWAYS run the full test suite BEFORE you commit. Never commit first and run tests afterward. The user has corrected this repeatedly and wants it as the standard sequence.\n\nRequired order for every change that touches code:\n\n1. Make the change.\n2. Run the FULL test suite (not just the tests you think are affected).\n3. Only if tests pass, run lint/format checks if the repo uses them.\n4. THEN `git commit`.\n\nIf tests fail, fix the failure (or revert) and re-run \u2014 do NOT commit a red tree.\n\n## Why this matters\n\nA commit that precedes a passing test run can bake in a broken state and pollute history. Running tests first means the commit always represents a known-good state. This ordering is non-negotiable for this user.\n\n## Pitfalls\n\n- Don't shortcut to \"tests probably pass\" \u2014 actually run them before committing.\n- Don't run only a subset (\"the affected tests\"); run the full suite unless the user explicitly scopes it down.\n- Don't commit, then run tests, then \"fix forward\" with a second commit. Tests gate the commit, not the other way around.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 27.5
      },
      "digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1578,
          "cache_read_tokens": 134773,
          "cache_write_tokens": 91228
        },
        "cost": 2.0311545,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing workflow rule (stated firmly, after repeated reminders): ALWAYS run the full test suite BEFORE committing, never after. Tests-first, then commit \u2014 make this the default sequence without being asked.",
          "skill_files": [
            "devops/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to stage, verify, and commit changes for this user. Covers the mandatory test-before-commit sequence and pre-commit verification gates. Load before running git commit on any code change.\n---\n\n# Git Commit Workflow\n\nGoverns how to land code changes safely for this user.\n\n## The cardinal rule: TEST BEFORE COMMIT\n\nALWAYS run the full test suite BEFORE you commit \u2014 never after. The user has\ncorrected this more than once and considers it the standard, non-negotiable\nsequence. Committing first and testing afterward is a hard mistake here, even\nif the tests end up passing.\n\nCorrect order, every time:\n1. Make / stage the code changes.\n2. Run the FULL test suite (not just the tests you think are affected).\n3. Only if the suite is green: `git commit`.\n4. If the suite is red: fix first, re-run, do NOT commit until green.\n\nWrong order (do not do this):\n- `git commit` then run tests \"to confirm\" \u2014 this is the exact thing the user\n  has repeatedly told you to stop doing.\n\n## Why this matters\n- Keeps every commit in history green / bisectable.\n- The user reads \"commit then test\" as you not having internalized their\n  standing instruction. Treat it as a workflow invariant, not a suggestion.\n\n## Pitfalls\n- Don't shortcut to a partial/targeted test run to save time before a commit;\n  the rule is the FULL suite unless the user explicitly scopes it down.\n- Don't assume a trivial change (typo, docs, rename) is exempt \u2014 run the suite\n  before committing those too unless the user says otherwise.\n- If the suite is slow, that's a reason to raise it with the user, not a reason\n  to commit first and test later.\n\n## Verification\nBefore announcing a commit is done, confirm in this order: suite ran, suite\npassed, then the commit. Report them in that sequence so the user can see the\ngate was respected.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 34.6
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 1,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1186,
          "cache_read_tokens": 135912,
          "cache_write_tokens": 206955
        },
        "cost": 4.17331425,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Workflow rule (stated emphatically, repeat correction): ALWAYS run the full test suite BEFORE committing, never after. This is the user's standard required sequence \u2014 tests first, then commit.",
          "skill_files": [
            "version-control/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: Standard sequence for committing code changes \u2014 running checks, tests, and the commit itself in the correct order. Use whenever about to commit or push code.\n---\n\n# Git Commit Workflow\n\nHow to commit code changes for this user, in the required order.\n\n## Required sequence (do NOT reorder)\n\n1. Stage/review the changes (`git status`, `git diff`).\n2. **Run the FULL test suite and wait for it to pass.** This happens BEFORE the commit, never after.\n3. Only once tests are green, create the commit.\n4. Push (if requested).\n\n## Critical rule: tests BEFORE commit\n\nThe user has explicitly and repeatedly corrected this: ALWAYS run the full\ntest suite before committing, NEVER after. Committing first and testing\nafterward is wrong even if the tests would have passed. Treat \"tests then\ncommit\" as the invariant sequence for every commit.\n\n- Do not commit \"to save progress\" and run tests in a follow-up step.\n- If tests fail, fix them (or stash) and re-run before committing.\n- Run the *full* suite, not just the subset touching changed files, unless\n  the user explicitly narrows the scope.\n\n## Pitfalls\n\n- Committing before tests, then running tests \"to confirm\" \u2014 this is the\n  exact mistake the user flagged. Order matters, not just that both happen.\n- Skipping tests because a change \"looks trivial\" (typo, log line, rename).\n  The rule has no trivial-change exception unless the user grants one.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 28.3
      },
      "digest": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1383,
          "cache_read_tokens": 189885,
          "cache_write_tokens": 90964
        },
        "cost": 2.0942775,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing workflow rule (user has stated this more than once, with frustration): ALWAYS run the full test suite BEFORE committing, never after. Tests-first, then commit, is the required default sequence.",
          "skill_files": [
            "dev/commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: commit-workflow\ndescription: How to make code commits for this user \u2014 the required sequence of validation, testing, and committing changes. Use whenever you are about to commit code or finish a code-change task.\n---\n\n# Commit Workflow\n\nHow to land code changes for this user. Follow this sequence every time you commit.\n\n## Required sequence (NON-NEGOTIABLE)\n\nThe user has corrected this MORE THAN ONCE, with visible frustration (\"I've told you this\", \"AGAIN\"). Run the full test suite BEFORE committing \u2014 never after.\n\n1. Make the code change.\n2. **Run the FULL test suite and confirm it passes.** Do not shortcut to a subset unless the user explicitly scopes it.\n3. Only after tests are green: stage and commit.\n4. (If applicable) push / open PR.\n\nCommitting first and testing afterward is the specific mistake the user keeps catching. Tests are a GATE on the commit, not a follow-up to it.\n\n## Pitfalls\n\n- Do NOT commit \"to save progress\" and run tests afterward. That is exactly the inverted order the user rejects.\n- Do NOT assume a small/obvious change is exempt \u2014 the rule is unconditional.\n- If tests fail, fix and re-run until green before committing; never commit a red suite with intent to fix later.\n\n## Verification\n\nBefore reporting a commit done, confirm in your own trace that the test run happened and passed BEFORE the commit command, not after.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 37.4
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 2,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1199,
          "cache_read_tokens": 364044,
          "cache_write_tokens": 206990
        },
        "cost": 4.5172035,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing workflow rule: ALWAYS run the full test suite BEFORE committing, never after. User has corrected this more than once and asked it be the default sequence going forward.",
          "skill_files": [
            "version-control/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to make commits safely for this user \u2014 sequencing tests, lint, and the commit itself. Use whenever you are about to commit code or stage changes for a commit.\n---\n\n# Git Commit Workflow\n\nGoverns the order of operations around committing code. The user is strict about sequencing; getting this wrong is a repeated source of frustration.\n\n## Hard rule: tests BEFORE commit, never after\n\nALWAYS run the full test suite BEFORE creating a commit. Never commit first and run tests afterward. The user has corrected this multiple times and wants it as the default standard sequence.\n\nCorrect sequence:\n1. Make/stage the code changes.\n2. Run the FULL test suite (not just the touched files) and confirm it passes.\n3. (If the project uses one) run lint/format checks.\n4. Only then create the commit.\n\nIf tests fail at step 2, fix them (or stop and report) \u2014 do NOT commit a red tree and \"fix it in a follow-up.\"\n\n## Pitfalls\n- Do not commit and then run tests \"to verify\" \u2014 that inverts the required order.\n- \"Quick\" or trivial-looking changes are NOT exempt; run the suite anyway.\n- Running only the subset of tests near your change is not a substitute for the full suite when committing.\n\n## Verification\nBefore announcing a commit is done, confirm in this order: test suite green \u2192 (lint green) \u2192 commit created. State the commit only after the prior steps actually passed.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 30.9
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1299,
          "cache_read_tokens": 77932,
          "cache_write_tokens": 90953
        },
        "cost": 1.91978175,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing workflow rule (stated emphatically, after repeat correction): ALWAYS run the full test suite BEFORE committing, never after. Tests-first, then commit \u2014 this is the required sequence for all code work.",
          "skill_files": [
            "version-control/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: Conventions and required sequencing for committing code changes \u2014 when to test, how to stage, what order to run things. Load before any task that ends in a git commit.\n---\n\n# Git Commit Workflow\n\nClass-level guidance for making commits cleanly and in the order this user expects.\n\n## Required sequence: TEST BEFORE COMMIT (non-negotiable)\n\nALWAYS run the full test suite BEFORE creating a commit \u2014 never after. This is a\nhard rule from the user, stated emphatically after repeated violations\n(\"the rule is: ALWAYS run the full test suite BEFORE you commit, never after\").\n\nCorrect order for any code change:\n1. Make the change.\n2. Run the FULL test suite (not a subset, not just the touched file's tests).\n3. Only if tests pass, stage and commit.\n4. If tests fail, fix first \u2014 do not commit broken or unverified code.\n\nDo NOT commit and then run tests \"to confirm.\" Committing first defeats the\npurpose: a failing suite means the commit should never have happened. Treat the\ntest run as a gate that precedes the commit, not a follow-up to it.\n\n## Pitfalls\n- Running only the tests near your change instead of the full suite. The rule is\n  the FULL suite \u2014 a change can break something distant.\n- Committing first \"because it's quick\" and testing after. This is the exact\n  pattern the user has corrected multiple times. Never do it.\n- Forgetting the gate under time pressure or in a long back-and-forth session.\n  The sequence is standing and does not lapse between turns.\n\n## Verification\nBefore announcing a commit is done, confirm in this order: tests ran, tests\npassed, THEN commit created. If you can't show test output preceding the commit,\nyou did it out of order.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 27.7
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 0,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 556,
          "cache_read_tokens": 21782,
          "cache_write_tokens": 228477
        },
        "cost": 4.358376750000001,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Self-hosting runs on a 3-node Proxmox cluster with Ceph storage at home. For any deployment request, default to this target and tailor to Proxmox LXC/VMs.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 19.6
      },
      "digest": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 466,
          "cache_read_tokens": 21785,
          "cache_write_tokens": 91552
        },
        "cost": 1.7842874999999998,
        "did_save": true,
        "saved": {
          "memory": "User's self-hosting target is a 3-node Proxmox cluster with Ceph storage at home. When asked about deploying anything, assume this is the target and tailor to Proxmox LXC/VMs.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 16.1
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 1,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 624,
          "cache_read_tokens": 147045,
          "cache_write_tokens": 228633
        },
        "cost": 4.55432625,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Self-hosting target: a 3-node Proxmox cluster with Ceph storage at home. For any deployment question, assume this is the target and tailor to Proxmox LXC/VMs.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 19.8
      },
      "digest": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 415,
          "cache_read_tokens": 21788,
          "cache_write_tokens": 91543
        },
        "cost": 1.78029825,
        "did_save": true,
        "saved": {
          "memory": "User's self-hosting target is a 3-node Proxmox cluster with Ceph storage (at home). When asked about deploying anything, assume this target and tailor recommendations to Proxmox LXC/VMs.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 15.0
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 2,
      "position": "early",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 514,
          "cache_read_tokens": 21779,
          "cache_write_tokens": 228405
        },
        "cost": 4.35387225,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Self-hosts on a 3-node Proxmox cluster with Ceph storage at home. For any deployment question, assume this is the target and tailor to Proxmox LXC/VMs.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 16.5
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 598,
          "cache_read_tokens": 78509,
          "cache_write_tokens": 91670
        },
        "cost": 1.881516,
        "did_save": true,
        "saved": {
          "memory": "User's self-hosting target is a 3-node Proxmox cluster with Ceph storage at home. When asked about deploying anything, assume this is the target and tailor to Proxmox LXC/VMs.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 18.2
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 0,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1871,
          "cache_read_tokens": 113297,
          "cache_write_tokens": 162624
        },
        "cost": 3.3595604999999997,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/debugging-subprocess-startup/SKILL.md"
          ],
          "skill_text": "\n---\nname: debugging-subprocess-startup\ndescription: Diagnose and fix child/subprocess startup failures \u2014 SIGPIPE/SIGCHLD on launch, premature exits, inherited-fd problems, and deadlocks when spawning processes via subprocess.Popen, os.fork/exec, or similar.\n---\n\n# Debugging subprocess startup failures\n\nWhen a spawned child process (bridge, daemon, worker, helper binary) dies or\nmisbehaves immediately at startup, the root cause is frequently in how file\ndescriptors and signals are INHERITED from the parent \u2014 not in the child's own\nlogic, and not in filesystem permissions.\n\n## Triage order (cheapest / most common first)\n\n1. **Check inherited stdin/stdout/stderr before anything else.**\n   A child that inherits the parent's stdin and then writes to or has its stdin\n   closed can take SIGPIPE on startup. This looks like a mysterious instant death\n   with no useful log output.\n   - Fix: pass explicit fds on the Popen call instead of inheriting:\n     `subprocess.Popen(..., stdin=subprocess.DEVNULL)`\n     (also set stdout/stderr to PIPE or DEVNULL deliberately rather than letting\n     them inherit, unless you specifically want the parent's).\n   - This was the actual fix for a \"raft bridge dies with SIGPIPE on startup\"\n     case: the bridge inherited the parent's stdin and got SIGPIPE when it closed.\n     `stdin=subprocess.DEVNULL` resolved it.\n\n2. **Do NOT reflexively blame permissions.** chmod on the socket/binary is a\n   common first guess and is usually a dead end. Confirm perms quickly, then move\n   on \u2014 don't loop on it.\n\n3. **Capture the real signal/exit code.** Check `proc.returncode`: negative N\n   means killed by signal N (e.g. -13 = SIGPIPE, -11 = SIGSEGV, -6 = SIGABRT).\n   This tells you immediately whether it's a signal-on-startup problem vs. a clean\n   non-zero exit (config/arg error) vs. a crash.\n\n4. **Other inherited-fd pitfalls:**\n   - Leaked fds the child shouldn't have (use `close_fds=True`, the Popen default\n     on POSIX) or, conversely, fds the child NEEDS that got closed.\n   - SIGPIPE handling: Python ignores SIGPIPE by default (SIG_IGN) in the parent,\n     but a child execed from it may reset to SIG_DFL and die on a broken pipe.\n\n## Pitfalls\n- A child dying with no log output is the classic signature of a signal-on-startup\n  (SIGPIPE/SIGSEGV) rather than a logged error path \u2014 reach for `returncode`/signal\n  inspection, not log-grepping.\n- Reproduce by running the child's exact argv manually with its stdin closed\n  (`cmd < /dev/null` or `cmd <&-`) to confirm the inherited-stdin hypothesis fast.\n\n## Verification\nAfter applying `stdin=subprocess.DEVNULL` (or the relevant fd fix), restart and\nconfirm the child stays up past its startup phase and `returncode` is no longer a\nnegative signal value.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 35.0
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 2111,
          "cache_read_tokens": 76268,
          "cache_write_tokens": 88557
        },
        "cost": 1.93326075,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/subprocess-startup-debugging/SKILL.md"
          ],
          "skill_text": "\n---\nname: subprocess-startup-debugging\ndescription: Diagnose and fix child processes (bridges, daemons, workers spawned via Popen/fork/exec) that die immediately on startup \u2014 SIGPIPE, broken inherited fds, signal/exit-code triage.\n---\n\n# Debugging subprocess startup deaths\n\nWhen a spawned child process (a \"bridge\", daemon, worker, sidecar) dies the moment\nit starts, the cause is almost always something about the spawn environment it\nINHERITED from the parent, not the child's own logic. Diagnose the inherited state\nbefore touching the child's code.\n\n## First: identify HOW it died, don't guess WHY\n\nDo not jump to a guess (e.g. \"must be a file-permissions problem, try chmod\").\nGet the actual death signal/exit code first. Guessing wastes a round-trip and\nannoys the user. Steps:\n\n1. Capture the child's exit status. In Python: check `proc.returncode`. A negative\n   value `-N` means the child was killed by signal N (e.g. `-13` = SIGPIPE).\n2. Map the signal:\n   - `-13` SIGPIPE  -> child wrote to / read from a closed pipe (usually inherited stdin/stdout)\n   - `-11` SIGSEGV  -> native crash in the child binary\n   - `-6`  SIGABRT  -> assertion / abort in the child\n   - `-9`  SIGKILL  -> OOM killer or an external kill\n   - exit `127`     -> command not found / bad PATH\n   - exit `126`     -> found but not executable (THEN permissions matter)\n3. Only after you know the signal do you form a hypothesis.\n\n## Inherited stdin causing SIGPIPE (the common one)\n\nA child spawned with `subprocess.Popen(...)` inherits the parent's stdin/stdout/stderr\nby default. If the parent's stdin is a pipe that closes (or the parent isn't feeding it),\nthe child can take SIGPIPE on its first read/write and die on startup.\n\nFix \u2014 detach the child's stdin explicitly:\n\n```python\nsubprocess.Popen(cmd, stdin=subprocess.DEVNULL)   # child won't inherit a pipe that can break\n```\n\nUse `stdout=`/`stderr=subprocess.DEVNULL` or a logfile if those fds are the culprit\ninstead. The principle: never let a long-lived child silently inherit a fd whose\nother end the parent controls and may close.\n\n## Pitfalls\n\n- Don't reach for chmod / file permissions first. Permissions only explain exit 126\n  (\"found but not executable\") \u2014 not a SIGPIPE death. Confirmed in practice: a chmod\n  guess was wrong; the real cause was inherited stdin -> SIGPIPE, fixed with\n  `stdin=subprocess.DEVNULL`.\n- A child that \"dies on startup with no error output\" often produced no output\n  precisely because its stdout/stderr were the broken inherited fds. Redirect them\n  to a file to actually see the traceback.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 35.4
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 1,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1698,
          "cache_read_tokens": 114264,
          "cache_write_tokens": 164524
        },
        "cost": 3.383661,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/debugging-subprocess-startup/SKILL.md"
          ],
          "skill_text": "\n---\nname: debugging-subprocess-startup\ndescription: Diagnose and fix child/subprocess processes that die, hang, or crash on startup \u2014 SIGPIPE, inherited fds (stdin/stdout/stderr), signal propagation, env, and cwd issues. Covers Popen-spawned bridges, daemons, and sidecars.\n---\n\n# Debugging subprocess startup failures\n\nWhen a spawned child process (bridge, daemon, sidecar, worker) dies, hangs, or\ncrashes immediately on startup, the cause is usually how it was *spawned*, not\nits own logic. Investigate the spawn environment before touching the child's code.\n\n## Approach (investigate before guessing)\nDo NOT lead with a speculative fix (e.g. \"probably a permissions issue, try\nchmod\"). Identify the actual failure signal first, then target it. A wrong guess\noffered as the answer wastes the user's time and erodes trust.\n\n1. Capture the exact death signal / exit code. SIGPIPE (signal 13, exit 141),\n   SIGTERM, non-zero exit, or silent hang each point in a different direction.\n2. Inspect inherited file descriptors \u2014 the most common culprit.\n3. Only then look at perms, env, cwd, and the child's own startup code.\n\n## Common causes and fixes\n\n### SIGPIPE on startup  \u2192  inherited stdin/stdout\nA child that inherits the parent's stdin (or a pipe) gets SIGPIPE the moment the\nparent closes its end or writes to a closed pipe. Default Python ignores SIGPIPE\nin the parent but children may not.\n- Fix: detach the stream the child doesn't need.\n  `subprocess.Popen(cmd, stdin=subprocess.DEVNULL)`  (or DEVNULL on stdout/stderr).\n- This was the real fix for a \"raft bridge dies with SIGPIPE on startup\": the\n  bridge inherited the parent's stdin and got SIGPIPE when it closed. Setting\n  `stdin=subprocess.DEVNULL` on the Popen call resolved it.\n\n### Silent immediate exit  \u2192  missing cwd / env / binary\n- Pass explicit `cwd=` and `env=` to Popen; an inherited cwd may not contain\n  files the child expects.\n\n### Hang on startup  \u2192  blocked on a full/empty pipe\n- If you set `stdout=PIPE` but never read it, the child blocks once the OS pipe\n  buffer fills. Either read it or use DEVNULL.\n\n## Pitfalls\n- Permissions on the socket/binary are rarely the cause of a startup *signal*\n  death; don't anchor on chmod without evidence.\n- exit code 141 == 128 + 13 == SIGPIPE. Map exit codes to signals (128 + N).\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 31.9
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1863,
          "cache_read_tokens": 76329,
          "cache_write_tokens": 88392
        },
        "cost": 1.9116585,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/debugging-subprocess-lifecycle/SKILL.md"
          ],
          "skill_text": "\n---\nname: debugging-subprocess-lifecycle\ndescription: Diagnose and fix child-process startup/lifecycle failures spawned via subprocess (Popen) \u2014 SIGPIPE/SIGTERM on launch, premature death, inherited-fd problems, hung pipes. Use when a spawned helper/bridge/daemon dies or misbehaves right after start.\n---\n\n# Debugging subprocess lifecycle failures\n\nWhen a child process spawned via `subprocess.Popen` (a bridge, daemon, helper,\nsidecar) dies or hangs shortly after launch, the cause is usually **inherited\nfile descriptors / signal disposition from the parent**, not the child's own\nlogic. Resist the urge to first blame permissions, the socket path, or config \u2014\ncheck what the child inherited.\n\n## Diagnostic order (cheapest, most-likely first)\n\n1. **Which signal killed it?** A clean `SIGPIPE` / `SIGTERM` / `SIGHUP` on\n   *startup* points at inherited fds or the parent's lifecycle, not the child's\n   work. Get the exit status: negative `returncode` (Python) = `-signal`, or\n   check `WTERMSIG`. SIGPIPE (13) specifically = the child wrote to / read a\n   pipe whose other end is closed.\n2. **What did the child inherit?** By default Popen passes the parent's\n   `stdin`/`stdout`/`stderr` straight through. If the parent's stdin is a pipe\n   that later closes, the child gets SIGPIPE the moment it touches stdin.\n3. **Only then** consider permissions, socket paths, missing binaries, config.\n   These are real but rarer for a *startup* death and easy to over-anchor on.\n\n## Known patterns and fixes\n\n- **Child dies with SIGPIPE on startup** \u2192 it inherited the parent's stdin and\n  received SIGPIPE when that stdin closed. Fix: pass\n  `stdin=subprocess.DEVNULL` on the `Popen(...)` call so the child gets a stable\n  /dev/null instead of the parent's transient pipe. (Confirmed fix in practice.)\n  - Equivalent guards: `stdout`/`stderr=subprocess.DEVNULL` if it's writing to a\n    closed output pipe, or `start_new_session=True` to detach from the parent's\n    process group / controlling terminal so parent-side signals don't propagate.\n\n## Pitfalls\n\n- Don't reach for `chmod 0755` / permission changes as a first guess for a\n  startup death \u2014 a permissions problem usually manifests as EACCES/EPERM at\n  open/bind time, not as SIGPIPE. Wrong-guessing permissions wastes a round trip.\n- A child that *inherits* stdin but you only redirect stdout/stderr will still\n  take SIGPIPE on stdin. Redirect the fd that's actually being touched.\n\n## Verify the fix\n\nRe-launch and confirm the child stays up (no negative returncode / no signal in\nthe exit status), then exercise its real function (socket accepts, RPC responds).\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 40.3
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 2,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 2110,
          "cache_read_tokens": 210503,
          "cache_write_tokens": 167216
        },
        "cost": 3.6094245,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/subprocess-lifecycle-debugging/SKILL.md",
            "debugging/subprocess-lifecycle-debugging/references/raft-bridge-sigpipe.md"
          ],
          "skill_text": "\n---\nname: subprocess-lifecycle-debugging\ndescription: Debug spawned child-process / subprocess lifecycle failures \u2014 crashes on startup, SIGPIPE/SIGTERM, hangs, premature exit, inherited-fd problems. Use when a daemon, bridge, worker, or helper process dies or misbehaves shortly after being launched via Popen/fork/exec.\n---\n\n# Subprocess lifecycle debugging\n\nWhen a spawned child process (bridge, daemon, worker, helper) crashes, hangs, or\ndies with a signal shortly after launch, the cause is usually in HOW it was\nspawned \u2014 inherited file descriptors, signal disposition, or stdio wiring \u2014 not\nin the child's own logic.\n\n## Diagnostic order (cheapest checks first)\n\n1. Identify the signal / exit mode precisely. SIGPIPE (13), SIGTERM (15),\n   SIGHUP (1), and a clean nonzero exit each point at different causes. Don't\n   guess \"permissions\" or \"config\" before you know the failure mode.\n2. Check inherited stdio. A child that inherits the parent's stdin/stdout/stderr\n   will receive SIGPIPE the moment the other end of one of those fds closes.\n   This is the classic \"dies with SIGPIPE on startup\" cause.\n3. Check signal disposition the child inherits (ignored vs default handlers).\n4. Only then look at the child's own code / permissions / config.\n\n## The SIGPIPE-on-startup fix (most common)\n\nSymptom: child process dies with SIGPIPE on or right after startup.\nCause: the child inherited the parent's stdin (or another std fd) and gets\nSIGPIPE when the parent closes it or when it writes to a closed pipe.\nFix: detach the child's stdin (and stdout/stderr if unused) explicitly:\n\n    subprocess.Popen([...], stdin=subprocess.DEVNULL)\n\nUse stdin=subprocess.DEVNULL when the child never reads stdin. Use\nstdout=subprocess.DEVNULL / stderr=subprocess.DEVNULL similarly if those streams\nare unused, to avoid SIGPIPE when the parent's end goes away.\n\n## Pitfalls\n\n- Don't reach for \"socket/file permissions\" or chmod first \u2014 permission errors\n  surface as EACCES/EPERM, not SIGPIPE. Signal type tells you the category;\n  let it steer the diagnosis.\n- Closing/reusing the parent's stdin after spawning is a common trigger when the\n  child inherited it.\n\n## references/\n\n- references/raft-bridge-sigpipe.md \u2014 worked example: raft bridge dying with\n  SIGPIPE on startup, fixed with stdin=subprocess.DEVNULL.\n\n# Raft bridge dies with SIGPIPE on startup\n\n## Symptom\nThe raft bridge child process died with SIGPIPE immediately on startup.\n\n## Wrong first guess\nSocket/file permissions (chmod 0755). This was incorrect \u2014 permission failures\npresent as EACCES/EPERM, never SIGPIPE. The signal type alone ruled this out.\n\n## Actual cause\nThe bridge inherited the parent process's stdin. When the parent closed stdin,\nthe child received SIGPIPE and died.\n\n## Fix\nPass stdin=subprocess.DEVNULL on the Popen call so the child does not inherit\nthe parent's stdin:\n\n    subprocess.Popen([...bridge cmd...], stdin=subprocess.DEVNULL)\n\nConfirmed working by the user.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 37.3
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 2171,
          "cache_read_tokens": 76342,
          "cache_write_tokens": 88620
        },
        "cost": 1.9390530000000001,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/debugging-subprocess-spawn/SKILL.md"
          ],
          "skill_text": "\n---\nname: debugging-subprocess-spawn\ndescription: Diagnose and fix child processes (bridges, daemons, workers spawned via subprocess.Popen / fork+exec) that die, hang, or misbehave on startup \u2014 especially from inherited file descriptors, signals, or stdio. Use when a spawned process exits immediately, dies with SIGPIPE/SIGHUP, or blocks waiting on stdin.\n---\n\n# Debugging Subprocess Spawn Failures\n\nWhen a child process spawned by a parent (via `subprocess.Popen`, `fork`+`exec`,\nmultiprocessing, or a supervisor) dies or hangs at startup, the cause is usually\nsomething INHERITED from the parent \u2014 file descriptors, signal dispositions, the\ncontrolling terminal, or environment \u2014 not the child's own logic.\n\n## Diagnostic order (cheapest first)\n\n1. Capture the exact death signal. A non-zero exit with signal 13 (SIGPIPE), 1\n   (SIGHUP), or 15 (SIGTERM) points at inherited stdio / terminal, NOT app code.\n   Check `returncode` (negative = killed by `-signum`) or `strace -f -e trace=signal`.\n2. Don't jump to filesystem permissions. Socket/file perms are a tempting first\n   guess but rarely the cause of a *startup* death-by-signal; rule it out fast,\n   don't anchor on it. (Lesson from a real session: \"try chmod 0755\" was wrong;\n   the real cause was inherited stdin.)\n3. Inspect what the child inherited: open FDs (`ls -l /proc/<pid>/fd`), stdin/\n   stdout/stderr wiring, and whether the parent later closes a pipe the child\n   still holds.\n\n## The inherited-stdin SIGPIPE pattern (common, easily missed)\n\nA child spawned with default `Popen(...)` inherits the parent's stdin/stdout.\nIf the parent (or its pipe peer) closes that descriptor, the child gets SIGPIPE\nthe next time it touches it and dies on startup.\n\nFix: explicitly detach the child's stdio instead of inheriting it.\n\n```python\nsubprocess.Popen(\n    cmd,\n    stdin=subprocess.DEVNULL,   # <- the fix: don't inherit parent stdin\n    # stdout=subprocess.DEVNULL / PIPE as appropriate\n)\n```\n\nUse `stdin=subprocess.DEVNULL` whenever the child does not actually read stdin\n(most bridges, daemons, RPC servers). This severs the SIGPIPE/EOF coupling to\nthe parent's terminal or pipe.\n\n## Related inheritance gotchas to check\n\n- SIGPIPE disposition: Python resets SIGPIPE to SIG_DFL in children; a child\n  writing to a closed pipe dies. Set `signal.signal(SIGPIPE, SIG_IGN)` in the\n  child if it must survive broken pipes, or wire stdio explicitly.\n- Controlling terminal: a child that should outlive an interactive shell needs\n  `start_new_session=True` (setsid) so SIGHUP on shell exit doesn't kill it.\n- Leaked FDs: pass `close_fds=True` (default on modern Python) so the child\n  doesn't hold parent pipes open and deadlock the parent's reads.\n\n## Verification\n\nReproduce the original death, apply the stdio fix, confirm the child now reaches\nits ready state (health check / log line), and confirm the parent's exit no\nlonger signals the child.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null,
        "latency_s": 39.6
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 0,
      "position": "mid",
      "n_signals": 2,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 535,
          "cache_read_tokens": 117767,
          "cache_write_tokens": 170035
        },
        "cost": 3.40502175,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep platform-team scope in mind.\n\u00a7\nRepo uses ruff for all linting/formatting \u2014 never reach for flake8.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 24.5
      },
      "digest": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1439,
          "cache_read_tokens": 186131,
          "cache_write_tokens": 88370
        },
        "cost": 2.044209,
        "did_save": true,
        "saved": {
          "memory": "This repo uses ruff for all Python linting/formatting \u2014 always use ruff, never flake8.",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep platform-team scope in mind.",
          "skill_files": [
            "dev/python-linting/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting\ndescription: How to lint and format Python code in this user's repos. Use whenever checking style, fixing lint warnings, ordering imports, or running a formatter on Python files.\n---\n\n# Python Linting & Formatting\n\nThis covers the class of tasks: lint warnings, import ordering, style/format\nfixes, and pre-commit checks on Python code.\n\n## Tooling: use ruff, always\n\nThis user's repos standardize on **ruff** for everything \u2014 linting AND\nformatting. The user explicitly corrected this: \"stop reaching for flake8 \u2014\nthis repo uses ruff for everything, always use ruff.\"\n\n- Do NOT reach for `flake8`, `isort`, `black`, or `pyflakes` as separate tools.\n  Ruff covers lint + import sorting + formatting in one.\n- Lint:        `ruff check .`\n- Autofix:     `ruff check --fix .`\n- Format:      `ruff format .`\n- Import order: handled by ruff's `I` rules (`ruff check --select I --fix .`),\n  not a standalone isort run.\n\n## Pitfalls\n\n- Don't suggest flake8 config, `.flake8`, or flake8 plugins \u2014 they don't apply\n  here. Ruff config lives in `pyproject.toml` under `[tool.ruff]`.\n- If a warning code looks like a flake8 code (e.g. `E501`, `F401`), ruff still\n  reports it \u2014 keep using ruff, just reference the same code.\n\n## Verification\n\nAfter fixing, confirm clean with `ruff check .` (exit 0) and, if formatting was\ntouched, `ruff format --check .`.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 31.5
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 1,
      "position": "mid",
      "n_signals": 2,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1424,
          "cache_read_tokens": 213864,
          "cache_write_tokens": 171174
        },
        "cost": 3.6372285,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep platform-team scope in mind.\n\u00a7\nThis repo uses ruff for all linting/formatting \u2014 always use ruff, never flake8.",
          "skill_files": [
            "python/python-linting-formatting/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting-formatting\ndescription: How to lint and format Python code in this user's repos \u2014 tool choice, invocation, and conventions. Use whenever checking style, fixing lint warnings, ordering imports, or running formatters.\n---\n\n# Python Linting & Formatting\n\nClass-level guidance for any task that touches Python code style: lint warnings,\nimport ordering, formatting, pre-commit checks, CI lint stages.\n\n## Tooling: use ruff, not flake8\n\nThis user's repos standardize on **ruff** for everything \u2014 linting AND formatting.\nDo NOT reach for flake8, isort, black, or pyflakes separately; ruff subsumes them.\n\nCorrection from the user (Dana, platform lead): \"stop reaching for flake8 \u2014 this\nrepo uses ruff for everything, always use ruff.\" Treat this as the default for\ntheir work unless a specific repo proves otherwise (check pyproject.toml / ruff.toml).\n\n### Commands\n- Lint:            `ruff check .`\n- Lint + autofix:  `ruff check --fix .`\n- Format:          `ruff format .`\n- Check format:    `ruff format --check .`\n- Import ordering is handled by ruff's `I` rules (`ruff check --select I --fix .`),\n  NOT a separate isort run.\n\n## Pitfalls\n- Don't suggest `flake8`, `black`, or `isort` as the fix for a lint/format issue \u2014\n  the user has explicitly corrected this. Map the request onto the ruff equivalent.\n- Before assuming rule config, look at `pyproject.toml [tool.ruff]` or `ruff.toml`\n  for the project's selected/ignored rules and line length.\n\n## Steps for a lint warning\n1. Identify the rule code from the warning (e.g. F401, E501, I001).\n2. Run `ruff check .` to confirm it reproduces.\n3. Prefer `ruff check --fix .` for auto-fixable rules; fix by hand otherwise.\n4. Re-run `ruff check .` and `ruff format --check .` to verify clean.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 31.9
      },
      "digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 722,
          "cache_read_tokens": 131009,
          "cache_write_tokens": 87523
        },
        "cost": 1.8918397500000002,
        "did_save": true,
        "saved": {
          "memory": "This repo uses ruff for all linting/formatting \u2014 never flake8. Always reach for ruff.",
          "user": "User is Dana, leads the platform team. When suggesting process changes, keep platform-team scope in mind.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 20.4
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 2,
      "position": "mid",
      "n_signals": 2,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1100,
          "cache_read_tokens": 213876,
          "cache_write_tokens": 170905
        },
        "cost": 3.60790275,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. Frame process-change suggestions to platform-team scope.\n\u00a7\nRepo uses ruff for all linting/formatting \u2014 always use ruff, never flake8.",
          "skill_files": [
            "dev/python-lint-format/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-lint-format\ndescription: Linting and formatting Python code in this repo. Covers which tools to use, how to run them, and repo-specific conventions.\n---\n\n# Python Lint & Format\n\nHow to lint and format Python in this repo.\n\n## Tooling \u2014 use ruff, always\n\nThis repo uses **ruff** for everything: linting AND formatting. Do NOT\nreach for flake8, black, isort, pylint, or autopep8 \u2014 ruff replaces all\nof them. The user corrected this explicitly; flake8 is wrong here.\n\n## Commands\n\n- Lint:            `ruff check .`\n- Lint + autofix:  `ruff check --fix .`\n- Format:          `ruff format .`\n- Check format:    `ruff format --check .`\n- Import ordering is handled by ruff (the isort rules, `I`), not a\n  separate isort run.\n\n## Steps\n\n1. Before committing, run `ruff check --fix .` then `ruff format .`.\n2. For a clean-tree verify (CI-style), run `ruff check .` and\n   `ruff format --check .` \u2014 both must pass with no diff.\n3. If a rule is noisy, prefer a targeted `# noqa: <code>` or a\n   `[tool.ruff]` config change over switching tools.\n\n## Pitfalls\n\n- Don't suggest flake8 / black / isort as separate steps \u2014 ruff covers\n  lint, format, and import ordering in one tool.\n- `ruff check` does not reformat; `ruff format` does. Run both.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 25.3
      },
      "digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1146,
          "cache_read_tokens": 131141,
          "cache_write_tokens": 88075
        },
        "cost": 1.93418775,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep platform-team scope in mind.\n\u00a7\nRepo uses ruff for all linting \u2014 always use ruff, never flake8.",
          "skill_files": [
            "python/python-linting/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting\ndescription: Lint and check Python code in this repo. Covers which linter to use, common commands, and fix workflows.\n---\n\n# Python Linting\n\nHow to lint/format Python in this repo.\n\n## Linter: use ruff, never flake8\n\nThis repo standardizes on **ruff for all linting**. Do NOT reach for flake8,\npyflakes, pycodestyle, isort, or other separate tools \u2014 ruff covers lint +\nimport sorting + (with `ruff format`) formatting. The user has corrected this\nexplicitly; defaulting to flake8 is a known mistake.\n\n## Common commands\n\n- Check: `ruff check .`\n- Check + autofix safe issues: `ruff check --fix .`\n- Format: `ruff format .`\n- Check a single file: `ruff check path/to/file.py`\n\n## Workflow\n\n1. Run `ruff check .` to surface lint warnings.\n2. Apply `ruff check --fix .` for autofixable rules (imports, unused vars, etc.).\n3. Run `ruff format .` if formatting drift is in scope.\n4. Re-run `ruff check .` to confirm clean.\n\n## Pitfalls\n\n- Don't suggest flake8/isort as a fallback \u2014 ruff is the single source of truth here.\n- Import ordering is handled by ruff (the `I` rules), not a separate isort step.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null,
        "latency_s": 26.0
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 0,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 536,
          "cache_read_tokens": 21788,
          "cache_write_tokens": 138651
        },
        "cost": 2.67264825,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing constraint: will NOT take a dependency on any cloud service. Every solution must run self-hosted and work fully offline \u2014 design all solutions that way by default.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 16.8
      },
      "digest": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 449,
          "cache_read_tokens": 21779,
          "cache_write_tokens": 84633
        },
        "cost": 1.65327225,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing rule (user emphasized as applying to everything): will NOT take a hard dependency on any cloud service. Every solution must run self-hosted and work fully offline \u2014 design all solutions offline-first.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 12.1
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 1,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 413,
          "cache_read_tokens": 21782,
          "cache_write_tokens": 138574
        },
        "cost": 2.6619705000000002,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing rule: will NOT take a dependency on any cloud service. Every solution must run self-hosted and work fully offline \u2014 design all work this way.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 13.8
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 718,
          "cache_read_tokens": 75225,
          "cache_write_tokens": 84921
        },
        "cost": 1.75904625,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing constraint: will NOT take a hard dependency on any cloud service. Everything must run self-hosted and work fully offline. Design every solution that way.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 24.7
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 2,
      "position": "mid",
      "n_signals": 1,
      "expect_noop": false,
      "full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 458,
          "cache_read_tokens": 21785,
          "cache_write_tokens": 138592
        },
        "cost": 2.6656874999999998,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing constraint: will NOT take a dependency on any cloud service. Every solution must run self-hosted and work fully offline \u2014 design all solutions that way by default.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 14.0
      },
      "digest": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 689,
          "cache_read_tokens": 75163,
          "cache_write_tokens": 84873
        },
        "cost": 1.75587825,
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard standing rule: will NOT take a hard dependency on any cloud service. Everything must run self-hosted and work fully offline \u2014 design all solutions to be offline-first/self-hosted by default.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null,
        "latency_s": 18.7
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 0,
      "position": "none",
      "n_signals": 0,
      "expect_noop": true,
      "full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 220,
          "cache_read_tokens": 0,
          "cache_write_tokens": 102360
        },
        "cost": 1.93578,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 9.1
      },
      "digest": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 239,
          "cache_read_tokens": 0,
          "cache_write_tokens": 54568
        },
        "cost": 1.041105,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 8.7
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 1,
      "position": "none",
      "n_signals": 0,
      "expect_noop": true,
      "full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 197,
          "cache_read_tokens": 0,
          "cache_write_tokens": 102354
        },
        "cost": 1.9339425,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 8.8
      },
      "digest": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 285,
          "cache_read_tokens": 0,
          "cache_write_tokens": 54571
        },
        "cost": 1.04461125,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 9.9
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 2,
      "position": "none",
      "n_signals": 0,
      "expect_noop": true,
      "full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 334,
          "cache_read_tokens": 0,
          "cache_write_tokens": 102357
        },
        "cost": 1.94427375,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 9.8
      },
      "digest": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 281,
          "cache_read_tokens": 0,
          "cache_write_tokens": 54571
        },
        "cost": 1.04431125,
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null,
        "latency_s": 9.3
      }
    }
  ],
  "meta": {
    "reps": 3,
    "model": "claude-opus-4-8",
    "prices": {
      "in": 15.0,
      "out": 75.0,
      "cache_read": 1.5,
      "cache_write": 18.75
    }
  }
 }
diff --git a/hard_scenarios.py b/hard_scenarios.py
 """Hard scenarios for the same-model (Opus vs Opus) background-review benchmark.

 Design goal: stress the case where the digest should make the SAME model
 capture learnings MORE reliably — signals buried in long, noisy sessions where
 a full-transcript replay dilutes the model's attention across the haystack.

 Each scenario:
  messages   — synthetic transcript (signals buried among realistic filler)
  signals    — list of ground-truth learnings a perfect review captures, each:
                 {kind: "skill"|"memory", desc, must_contain: [keywords...]}
  position   — where the key signal sits (early|mid|tail) — to test that the
               digest's signal-foregrounding beats full-replay attention dilution
  distractor — optional: a retracted/false signal that a good review must NOT save
 """

 FILLER_TOPICS = [
    "the CI matrix sharding", "a flaky import in tests", "the config loader merge order",
    "docs phrasing for the quickstart", "renaming a private helper", "test fixture setup",
    "a typo in a log line", "the changelog wording", "dependency version bumps",
    "formatting a table in markdown", "a lint warning", "import ordering",
 ]


 def _filler(n, prefix="f"):
    out = []
    for i in range(n):
        t = FILLER_TOPICS[i % len(FILLER_TOPICS)]
        out.append({"role": "user", "content":
            f"[{prefix}{i}] Quick one about {t}. " + ("Context detail goes here. " * 130)})
        out.append({"role": "assistant", "content":
            f"Here's the rundown on {t}. " + ("Explanation continues at length. " * 130)})
    return out


 SCENARIOS = [
    {
        "id": "buried_early_style_correction",
        "position": "early",
        "signals": [{
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers — durable style preference.",
            "must_contain": ["terse", "concise", "brief", "short", "verbose", "1-2", "one or two"],
        }],
        "messages": (
            [{"role": "user", "content":
                "before we start: you are WAY too verbose. from now on, every answer "
                "is one or two sentences max. no preamble, no recap. i mean it, always."},
             {"role": "assistant", "content": "Understood — terse from here on."}]
            + _filler(40)
        ),
    },
    {
        "id": "buried_mid_workflow_rule",
        "position": "mid",
        "signals": [{
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "must_contain": ["test", "before", "commit", "suite"],
        }],
        "messages": (
            _filler(20, "a")
            + [{"role": "user", "content":
                "hold on — you just committed before running tests AGAIN. the rule is: "
                "ALWAYS run the full test suite BEFORE you commit, never after. "
                "i've told you this. make it your standard sequence going forward."},
               {"role": "assistant", "content": "Understood — tests first, then commit."}]
            + _filler(20, "b")
        ),
    },
    {
        "id": "buried_early_persona_fact",
        "position": "early",
        "signals": [{
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "must_contain": ["proxmox", "ceph", "3-node", "three-node", "cluster", "homelab", "lxc"],
        }],
        "messages": (
            [{"role": "user", "content":
                "context you should keep: all my self-hosting is a 3-node proxmox cluster "
                "with ceph storage at home. whenever i ask about deploying something, "
                "assume that's the target and tailor it to proxmox LXC/VMs."},
             {"role": "assistant", "content": "Noted — I'll target your Proxmox + Ceph setup."}]
            + _filler(45)
        ),
    },
    {
        "id": "mid_technique_with_retracted_distractor",
        "position": "mid",
        "signals": [{
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "must_contain": ["sigpipe", "stdin", "devnull", "popen", "subprocess"],
        }],
        "distractor": {
            "desc": "An early guess that it was a permissions bug — RETRACTED. Must NOT be saved.",
            "must_not_contain": ["permission", "chmod", "0755"],
        },
        "messages": (
            _filler(15, "a")
            + [{"role": "user", "content": "the raft bridge dies with SIGPIPE on startup"},
               {"role": "assistant", "content":
                "First guess: might be a permissions issue on the socket — try chmod 0755. "
                "Let me check."},
               {"role": "user", "content": "no, perms are fine, it's not that"},
               {"role": "assistant", "content":
                "You're right, scratch the permissions idea. Found it: the bridge inherits "
                "the parent's stdin and gets SIGPIPE when it closes. The fix is "
                "stdin=subprocess.DEVNULL on the Popen call."},
               {"role": "user", "content": "yep that fixed it"}]
            + _filler(15, "b")
        ),
    },
    {
        "id": "multi_signal_session",
        "position": "mid",
        "signals": [
            {"kind": "skill",
             "desc": "Prefers ruff over flake8 for this repo.",
             "must_contain": ["ruff"]},
            {"kind": "memory",
             "desc": "User's name is Dana and they lead the platform team.",
             "must_contain": ["dana", "platform team", "platform-team"]},
        ],
        "messages": (
            _filler(12, "a")
            + [{"role": "user", "content":
                "btw I'm Dana, I lead the platform team here, so when you suggest "
                "process changes keep that scope in mind."},
               {"role": "assistant", "content": "Good to know, Dana."}]
            + _filler(8, "b")
            + [{"role": "user", "content":
                "also stop reaching for flake8 — this repo uses ruff for everything, "
                "always use ruff."},
               {"role": "assistant", "content": "Got it — ruff, not flake8."}]
            + _filler(12, "c")
        ),
    },
    {
        "id": "durable_vs_trivia_memory",
        "position": "mid",
        "signals": [{
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services — "
                    "everything must run self-hosted/offline. This is a standing constraint.",
            "must_contain": ["self-host", "self host", "offline", "no cloud", "on-prem", "on prem", "local"],
        }],
        "distractor": {
            "desc": "TRIVIA also present (today's weather, a one-off file path) that should "
                    "NOT crowd out the durable constraint.",
            "must_not_contain": ["weather", "sunny", "/tmp/scratch"],
        },
        "messages": (
            _filler(10, "a")
            + [{"role": "user", "content": "nice and sunny here today btw"},
               {"role": "assistant", "content": "Enjoy it!"},
               {"role": "user", "content":
                "important standing rule for everything we do: I will NOT take a hard "
                "dependency on any cloud service. everything must run self-hosted and "
                "work fully offline. design every solution that way."},
               {"role": "assistant", "content": "Understood — self-hosted, offline-first, always."},
               {"role": "user", "content": "oh and I dumped that log at /tmp/scratch for now"},
               {"role": "assistant", "content": "Noted."}]
            + _filler(15, "b")
        ),
    },
    {
        "id": "long_noop_smooth",
        "position": "none",
        "signals": [],  # correct review = no-op
        "messages": _filler(35, "q"),
    },
 ]
diff --git a/hard_summary.json b/hard_summary.json
 {
  "cost": {
    "full": {
      "mean": 3.5295075714285713,
      "total": 74.11965900000001,
      "n": 21
    },
    "digest": {
      "mean": 1.7864433928571428,
      "total": 37.51531125,
      "n": 21
    }
  },
  "tokens": {
    "full": 305468,
    "digest": 172433.85714285713
  },
  "capture": {
    "full": {
      "hit": 21,
      "total": 21
    },
    "digest": {
      "hit": 21,
      "total": 21
    }
  },
  "distractor_false_saves": {
    "full": 3,
    "digest": 3
  },
  "distractor_n": 6,
  "noop_correct": {
    "full": 3,
    "digest": 3
  },
  "noop_n": 3,
  "pairwise": {
    "full": 3,
    "digest": 0,
    "tie": 15,
    "parse_fail": 0
  }
 }
diff --git a/harness.py b/harness.py
 """Live A/B benchmark for the background-review cost-control PR.

 Runs the REAL background-review fork (agent.background_review) against a live
 Anthropic endpoint in two arms, across the ground-truth scenarios x reps:

  BASELINE  (current main behaviour): review fork on the expensive main model
            (claude-opus-4-8), full snapshot replay, parent prefix-cache share.
  TREATMENT (this PR, routed):        review fork routed to a cheap aux model
            (claude-haiku-4-5) via auxiliary.background_review, digest replay,
            no cache-share.

 For each run we capture the fork's REAL wire token usage (input / output /
 cache, summed from the agent's own normalize_usage accumulators) and what the
 review actually SAVED (memory entries + skill files written under an isolated
 HERMES_HOME). Cost is computed from published Anthropic per-token prices.

 Quality is scored two ways:
  • Fixture oracle: did expect_save scenarios save, and expect_save=False
    scenarios stay no-op? → precision / recall / F1.
  • (separate judge script grades edit quality from the saved artifacts.)

 Output: /tmp/bgreview_bench/results.json
 """
 import os
 import sys
 import json
 import time
 import shutil
 import tempfile
 import threading
 from pathlib import Path

 WORKTREE = "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8"
 sys.path.insert(0, WORKTREE)
 sys.path.insert(0, "/tmp/bgreview_bench")

 REPS = int(os.environ.get("BENCH_REPS", "3"))
 MAIN_MODEL = "claude-opus-4-8"
 AUX_MODEL = "claude-haiku-4-5-20251001"

 # Published Anthropic prices (USD per 1M tokens), as of 2026-06.
 PRICES = {
    "claude-opus-4-8":            {"in": 15.00, "out": 75.00, "cache_read": 1.50, "cache_write": 18.75},
    "claude-haiku-4-5-20251001":  {"in": 1.00,  "out": 5.00,  "cache_read": 0.10, "cache_write": 1.25},
 }


 def _cost(model, usage):
    p = PRICES[model]
    return (
        usage["input_tokens"] / 1e6 * p["in"]
        + usage["output_tokens"] / 1e6 * p["out"]
        + usage["cache_read_tokens"] / 1e6 * p["cache_read"]
        + usage["cache_write_tokens"] / 1e6 * p["cache_write"]
    )


 def _write_config(hermes_home, routed):
    """Write a config.yaml; routed=True pins the aux model for review."""
    aux_review = (
        {"provider": "anthropic", "model": AUX_MODEL}
        if routed else
        {"provider": "auto", "model": ""}
    )
    cfg = {
        "model": {"provider": "anthropic", "model": MAIN_MODEL, "default": MAIN_MODEL},
        "auxiliary": {"background_review": aux_review},
        "memory": {"memory_enabled": True, "user_profile_enabled": True},
        "skills": {"creation_nudge_interval": 10, "skip_tool_free_turns": True,
                   "adaptive_backoff": True, "adaptive_backoff_after": 3},
    }
    import yaml
    (Path(hermes_home) / "config.yaml").write_text(yaml.safe_dump(cfg))


 def _make_parent_agent(AIAgent, api_key, hermes_home):
    """A real AIAgent on the main model, used only as the review fork's parent."""
    agent = AIAgent(
        model=MAIN_MODEL,
        provider="anthropic",
        api_key=api_key,
        base_url=None,
        api_mode="anthropic_messages",
        quiet_mode=True,
        skip_context_files=True,
        skip_memory=True,
        platform="cli",
        session_id="bench-parent",
    )
    # Build + cache the parent system prompt so the cache-share path is exercised
    # in the baseline arm exactly as it is in production.
    try:
        agent._cached_system_prompt = agent._build_system_prompt()
    except Exception:
        agent._cached_system_prompt = "BENCH-PARENT-SYSTEM-PROMPT"
    return agent


 def run_arm(scenario, routed, api_key):
    """Run one review fork synchronously; return usage + what was saved."""
    hermes_home = tempfile.mkdtemp(prefix="bgrev_")
    os.makedirs(os.path.join(hermes_home, ".hermes"), exist_ok=True)
    hh = os.path.join(hermes_home, ".hermes")
    os.environ["HERMES_HOME"] = hh

    # Fresh import of the modules so config + HERMES_HOME bind to this temp dir.
    for mod in [m for m in sys.modules if m.startswith(("hermes", "agent", "run_agent", "tools", "model_tools"))]:
        del sys.modules[mod]
    sys.path.insert(0, WORKTREE)

    _write_config(hh, routed)
    os.environ["ANTHROPIC_API_KEY"] = api_key

    from run_agent import AIAgent
    from agent import background_review as br

    # Capture REAL wire usage. The Anthropic path uses Messages.stream()
    # (a context manager); usage lives on the final message. We wrap stream()
    # to tee get_final_message() and the manager __exit__ so every fork call
    # is tallied regardless of how the loop consumes the stream.
    import anthropic
    tally = {"input_tokens": 0, "output_tokens": 0,
             "cache_read_tokens": 0, "cache_write_tokens": 0}

    def _add_usage(u):
        if u is None:
            return
        tally["input_tokens"] += int(getattr(u, "input_tokens", 0) or 0)
        tally["output_tokens"] += int(getattr(u, "output_tokens", 0) or 0)
        tally["cache_read_tokens"] += int(getattr(u, "cache_read_input_tokens", 0) or 0)
        tally["cache_write_tokens"] += int(getattr(u, "cache_creation_input_tokens", 0) or 0)

    Messages = anthropic.resources.messages.Messages
    orig_stream = Messages.stream
    orig_create = Messages.create

    class _MgrProxy:
        """Wrap the stream context manager; tally usage from the final message."""
        def __init__(self, mgr):
            self._mgr = mgr
            self._counted = False
        def __enter__(self):
            self._stream = self._mgr.__enter__()
            return self._wrap_stream(self._stream)
        def __exit__(self, *a):
            return self._mgr.__exit__(*a)
        def _wrap_stream(self, stream):
            _outer = self
            orig_gfm = stream.get_final_message
            def _gfm(*a, **k):
                msg = orig_gfm(*a, **k)
                if not _outer._counted:
                    _outer._counted = True
                    try:
                        _add_usage(getattr(msg, "usage", None))
                    except Exception:
                        pass
                return msg
            stream.get_final_message = _gfm
            return stream

    def _wrapped_stream(self, *a, **kw):
        return _MgrProxy(orig_stream(self, *a, **kw))

    def _wrapped_create(self, *a, **kw):
        resp = orig_create(self, *a, **kw)
        try:
            _add_usage(getattr(resp, "usage", None))
        except Exception:
            pass
        return resp

    Messages.stream = _wrapped_stream
    Messages.create = _wrapped_create

    parent = _make_parent_agent(AIAgent, api_key, hh)
    try:
        target, _prompt = br.spawn_background_review_thread(
            parent,
            messages_snapshot=list(scenario["messages"]),
            review_memory=True,
            review_skills=True,
        )
        # Run synchronously so we can read results deterministically.
        target()
    finally:
        Messages.stream = orig_stream
        Messages.create = orig_create
        try:
            parent.close()
        except Exception:
            pass
    # The parent build issues no model calls (system prompt is local), so the
    # tally reflects only the review fork's usage.

    # What did the review save? Inspect the isolated stores.
    mem_path = Path(hh) / "MEMORY.md"
    user_path = Path(hh) / "USER.md"
    skills_dir = Path(hh) / "skills"
    saved = {
        "memory": mem_path.read_text() if mem_path.exists() else "",
        "user": user_path.read_text() if user_path.exists() else "",
        "skill_files": [str(p.relative_to(skills_dir)) for p in skills_dir.rglob("*")
                        if p.is_file() and ".archive" not in str(p)] if skills_dir.exists() else [],
    }
    did_save = bool(saved["memory"].strip() or saved["user"].strip() or saved["skill_files"])

    usage = dict(tally)
    model = AUX_MODEL if routed else MAIN_MODEL
    cost = _cost(model, usage)

    shutil.rmtree(hermes_home, ignore_errors=True)
    return {"usage": usage, "model": model, "cost": cost, "did_save": did_save, "saved": saved}


 def main():
    import scenarios
    api_key = os.environ["ANTHROPIC_API_KEY"]
    results = {"baseline": [], "treatment": [], "meta": {
        "reps": REPS, "main_model": MAIN_MODEL, "aux_model": AUX_MODEL,
        "prices": PRICES,
    }}

    for sc in scenarios.SCENARIOS:
        for rep in range(REPS):
            for arm, routed in (("baseline", False), ("treatment", True)):
                t0 = time.time()
                try:
                    r = run_arm(sc, routed, api_key)
                    r["error"] = None
                except Exception as e:
                    import traceback
                    r = {"error": f"{e}", "trace": traceback.format_exc()[-800:],
                         "usage": {"input_tokens": 0, "output_tokens": 0,
                                   "cache_read_tokens": 0, "cache_write_tokens": 0},
                         "cost": 0.0, "did_save": None, "saved": {}}
                r.update({"scenario": sc["id"], "rep": rep, "arm": arm,
                          "expect_save": sc["expect_save"], "signal": sc["signal"],
                          "latency_s": round(time.time() - t0, 1)})
                results[arm].append(r)
                print(f"[{arm:9}] {sc['id']:28} rep{rep} "
                      f"cost=${r['cost']:.5f} save={r['did_save']} "
                      f"err={r['error']}", flush=True)

    Path("/tmp/bgreview_bench/results.json").write_text(json.dumps(results, indent=2))
    print("\nWrote /tmp/bgreview_bench/results.json")


 if __name__ == "__main__":
    main()
diff --git a/judge.out b/judge.out
 [baseline ] user_style_correction        rep0 q=4 captures=True fp=False
 [baseline ] user_style_correction        rep1 q=4 captures=True fp=False
 [baseline ] user_style_correction        rep2 q=4 captures=True fp=False
 [baseline ] workflow_correction          rep0 q=4 captures=True fp=False
 [baseline ] workflow_correction          rep1 q=4 captures=True fp=False
 [baseline ] workflow_correction          rep2 q=None captures=None fp=None
 [baseline ] debug_technique              rep0 q=4 captures=True fp=False
 [baseline ] debug_technique              rep1 q=4 captures=True fp=False
 [baseline ] debug_technique              rep2 q=4 captures=True fp=False
 [baseline ] user_persona_fact            rep0 q=1 captures=False fp=False
 [baseline ] user_persona_fact            rep1 q=1 captures=False fp=False
 [baseline ] user_persona_fact            rep2 q=3 captures=True fp=False
 [baseline ] smooth_qa_noop               rep0 q=5 captures=True fp=False
 [baseline ] smooth_qa_noop               rep1 q=5 captures=True fp=False
 [baseline ] smooth_qa_noop               rep2 q=5 captures=True fp=False
 [baseline ] smooth_task_noop             rep0 q=5 captures=True fp=False
 [baseline ] smooth_task_noop             rep1 q=5 captures=True fp=False
 [baseline ] smooth_task_noop             rep2 q=5 captures=True fp=False
 [baseline ] transient_error_resolved_noop rep0 q=3 captures=True fp=False
 [baseline ] transient_error_resolved_noop rep1 q=5 captures=True fp=False
 [baseline ] transient_error_resolved_noop rep2 q=3 captures=True fp=False
 [baseline ] tool_heavy_genuine_skill     rep0 q=5 captures=True fp=False
 [baseline ] tool_heavy_genuine_skill     rep1 q=4 captures=True fp=False
 [baseline ] tool_heavy_genuine_skill     rep2 q=4 captures=True fp=False
 [treatment] user_style_correction        rep0 q=4 captures=True fp=False
 [treatment] user_style_correction        rep1 q=4 captures=True fp=False
 [treatment] user_style_correction        rep2 q=4 captures=True fp=False
 [treatment] workflow_correction          rep0 q=4 captures=True fp=False
 [treatment] workflow_correction          rep1 q=None captures=None fp=None
 [treatment] workflow_correction          rep2 q=4 captures=True fp=False
 [treatment] debug_technique              rep0 q=4 captures=True fp=False
 [treatment] debug_technique              rep1 q=None captures=None fp=None
 [treatment] debug_technique              rep2 q=4 captures=True fp=False
 [treatment] user_persona_fact            rep0 q=1 captures=False fp=False
 [treatment] user_persona_fact            rep1 q=0 captures=False fp=False
 [treatment] user_persona_fact            rep2 q=0 captures=False fp=False
 [treatment] smooth_qa_noop               rep0 q=5 captures=True fp=False
 [treatment] smooth_qa_noop               rep1 q=5 captures=True fp=False
 [treatment] smooth_qa_noop               rep2 q=5 captures=True fp=False
 [treatment] smooth_task_noop             rep0 q=5 captures=True fp=False
 [treatment] smooth_task_noop             rep1 q=5 captures=True fp=False
 [treatment] smooth_task_noop             rep2 q=5 captures=True fp=False
 [treatment] transient_error_resolved_noop rep0 q=5 captures=True fp=False
 [treatment] transient_error_resolved_noop rep1 q=3 captures=True fp=False
 [treatment] transient_error_resolved_noop rep2 q=3 captures=True fp=False
 [treatment] tool_heavy_genuine_skill     rep0 q=4 captures=True fp=False
 [treatment] tool_heavy_genuine_skill     rep1 q=4 captures=True fp=False
 [treatment] tool_heavy_genuine_skill     rep2 q=5 captures=True fp=False

 Wrote /tmp/bgreview_bench/quality.json
diff --git a/judge.py b/judge.py
 """LLM-judge quality grading for the background-review A/B results.

 Reads /tmp/bgreview_bench/results.json (the saved memory/skill artifacts from
 each arm) and grades, per run, whether the review correctly captured the
 scenario's ground-truth signal. A strong model (claude-opus-4-8) is the judge.

 Two oracles combined:
  • Fixture oracle (objective): did expect_save scenarios save, and
    expect_save=False scenarios stay no-op? → precision / recall / F1 per arm.
  • LLM-judge (edit quality): for runs that DID save, score 0-5 whether the
    saved memory/skill actually captures the ground-truth signal well, plus a
    false-positive check on no-op scenarios that saved anyway.

 Writes /tmp/bgreview_bench/quality.json + prints a summary.
 """
 import os
 import sys
 import json
 from pathlib import Path

 sys.path.insert(0, "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8")

 JUDGE_MODEL = "claude-opus-4-8"

 JUDGE_PROMPT = """You are grading whether a background self-improvement review \
 captured the correct lesson from a conversation.

 GROUND-TRUTH SIGNAL (what a correct review should capture):
 {signal}

 EXPECTATION: {expectation}

 WHAT THE REVIEW ACTUALLY SAVED:
 --- memory (MEMORY.md) ---
 {memory}
 --- user profile (USER.md) ---
 {user}
 --- skill files created ---
 {skills}

 Grade on this rubric and reply with ONLY a compact JSON object:
 {{"captures_signal": <true|false>,
  "quality": <0-5 integer: 0=nothing relevant, 3=captures the signal adequately, 5=captures it precisely and durably>,
  "false_positive": <true|false: true if expectation was "no save" but it saved a durable rule anyway>,
  "note": "<one short sentence>"}}"""


 def _judge(client, scenario_signal, expect_save, saved):
    expectation = (
        "A correct review SAVES a durable memory/skill capturing the signal."
        if expect_save else
        "A correct review saves NOTHING (no-op) — saving a durable rule here is a false positive."
    )
    skills_txt = "\n".join(
        f"- {f}" for f in saved.get("skill_files", []) if not f.endswith((".usage.json", ".lock"))
    ) or "(none)"
    prompt = JUDGE_PROMPT.format(
        signal=scenario_signal,
        expectation=expectation,
        memory=(saved.get("memory") or "(empty)")[:1500],
        user=(saved.get("user") or "(empty)")[:1500],
        skills=skills_txt,
    )
    msg = client.messages.create(
        model=JUDGE_MODEL,
        max_tokens=400,
        messages=[{"role": "user", "content": prompt}],
    )
    text = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
    text = text.strip()
    if text.startswith("```"):
        text = text.split("```")[1].lstrip("json").strip()
    try:
        return json.loads(text)
    except Exception:
        return {"captures_signal": None, "quality": None, "false_positive": None, "note": "parse_fail: " + text[:120]}


 def main():
    import anthropic
    results = json.loads(Path("/tmp/bgreview_bench/results.json").read_text())
    client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])

    graded = {"baseline": [], "treatment": []}
    for arm in ("baseline", "treatment"):
        for run in results[arm]:
            if run.get("error"):
                continue
            g = _judge(client, run["signal"], run["expect_save"], run.get("saved", {}))
            g.update({"scenario": run["scenario"], "rep": run["rep"],
                      "expect_save": run["expect_save"], "did_save": run["did_save"]})
            graded[arm].append(g)
            print(f"[{arm:9}] {run['scenario']:28} rep{run['rep']} "
                  f"q={g.get('quality')} captures={g.get('captures_signal')} "
                  f"fp={g.get('false_positive')}", flush=True)

    Path("/tmp/bgreview_bench/quality.json").write_text(json.dumps(graded, indent=2))
    print("\nWrote /tmp/bgreview_bench/quality.json")


 if __name__ == "__main__":
    main()
diff --git a/quality.json b/quality.json
 {
  "baseline": [
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a response-style skill capturing the terseness preference, though a USER.md profile note might have been more directly durable.",
      "scenario": "user_style_correction",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a response-style skill file capturing the terseness preference, though a USER.md profile entry might be more durable.",
      "scenario": "user_style_correction",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a response-style skill capturing the terseness preference, though a USER.md profile note could reinforce durability.",
      "scenario": "user_style_correction",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a git commit workflow skill that should capture the run-tests-before-committing rule.",
      "scenario": "workflow_correction",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a relevant skill file on committing code changes that should capture the test-before-commit rule, though exact content is unverified.",
      "scenario": "workflow_correction",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": null,
      "quality": null,
      "false_positive": null,
      "note": "parse_fail: I need to evaluate whether the review captured the signal, but I can only see the file path of the skill, not its conten",
      "scenario": "workflow_correction",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "A skill file on subprocess signal debugging was created, durably capturing the SIGPIPE/stdin=DEVNULL technique.",
      "scenario": "debug_technique",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a subprocess debugging skill with a dedicated reference for the SIGPIPE/stdin=DEVNULL fix, durably capturing the signal.",
      "scenario": "debug_technique",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "A durable skill file on subprocess signal debugging was created, capturing the SIGPIPE/stdin=DEVNULL technique.",
      "scenario": "debug_technique",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": false,
      "quality": 1,
      "false_positive": false,
      "note": "Created a generic self-hosted deployment skill but failed to save the durable personal fact about the user's 3-node Proxmox cluster and infra preferences.",
      "scenario": "user_persona_fact",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": false,
      "quality": 1,
      "false_positive": false,
      "note": "Created a generic self-hosted deployment skill but failed to save the specific durable fact about the user's 3-node Proxmox home cluster to memory or profile.",
      "scenario": "user_persona_fact",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 3,
      "false_positive": false,
      "note": "The Proxmox cluster fact was captured in a skill file rather than a personal memory/profile, which is durable but less ideal placement for a personal preference.",
      "scenario": "user_persona_fact",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly performed a no-op for plain factual Q&A with nothing worth saving.",
      "scenario": "smooth_qa_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a plain factual Q&A, matching the expected no-op.",
      "scenario": "smooth_qa_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a plain factual Q&A with no durable lesson.",
      "scenario": "smooth_qa_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly performed a no-op for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly performed a no-op for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 3,
      "false_positive": false,
      "note": "Correctly saved nothing for a transient network error that resolved on retry.",
      "scenario": "transient_error_resolved_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing, recognizing the transient retry-resolved error required no durable rule.",
      "scenario": "transient_error_resolved_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 3,
      "false_positive": false,
      "note": "Correctly saved nothing for a transient retry-resolved network error, matching the no-op expectation.",
      "scenario": "transient_error_resolved_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Created a dedicated skill file capturing the reusable flaky-test bisection procedure.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Correctly saved a skill file capturing the reusable flaky-test bisection procedure.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a flaky-test debugging skill that captures the reusable bisection procedure.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    }
  ],
  "treatment": [
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a skill file capturing the concise/terse communication preference, durably addressing the signal though a user-profile note might be more fitting.",
      "scenario": "user_style_correction",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "A skill file capturing the user's preference for concise explanations was created, durably reflecting the terseness signal.",
      "scenario": "user_style_correction",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a concise-responses skill capturing the terseness preference, though a USER.md style note might have been more directly durable.",
      "scenario": "user_style_correction",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a code-commit-workflow skill that plausibly captures the rule to run tests before committing.",
      "scenario": "workflow_correction",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": null,
      "quality": null,
      "false_positive": null,
      "note": "parse_fail: I need to evaluate whether the saved skill file captures the durable process rule about running tests before committing.",
      "scenario": "workflow_correction",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "A skill file on commit-and-test workflow durably captures the test-before-commit rule, though exact content can't be verified.",
      "scenario": "workflow_correction",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a subprocess-safety skill that plausibly captures the stdin=DEVNULL SIGPIPE fix, though exact content isn't shown.",
      "scenario": "debug_technique",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": null,
      "quality": null,
      "false_positive": null,
      "note": "parse_fail: I need to evaluate whether the review captured the signal, but I can only see that a skill file was created\u2014not its cont",
      "scenario": "debug_technique",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a relevant subprocess-stdio debugging skill capturing the stdin=DEVNULL SIGPIPE fix.",
      "scenario": "debug_technique",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": false,
      "quality": 1,
      "false_positive": false,
      "note": "Created a generic Proxmox skill but failed to save the durable personal fact about the user's specific 3-node home cluster and tailoring preference.",
      "scenario": "user_persona_fact",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": false,
      "quality": 0,
      "false_positive": false,
      "note": "The review saved nothing despite a durable personal fact (3-node Proxmox home cluster) that should have been captured.",
      "scenario": "user_persona_fact",
      "rep": 1,
      "expect_save": true,
      "did_save": false
    },
    {
      "captures_signal": false,
      "quality": 0,
      "false_positive": false,
      "note": "Review saved nothing despite a durable personal fact about the user's 3-node Proxmox cluster preference.",
      "scenario": "user_persona_fact",
      "rep": 2,
      "expect_save": true,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for plain factual Q&A, matching the expected no-op.",
      "scenario": "smooth_qa_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly performed a no-op, saving nothing for plain factual Q&A as expected.",
      "scenario": "smooth_qa_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a plain factual Q&A, matching the no-op expectation.",
      "scenario": "smooth_qa_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing for a one-off task with no reusable lesson.",
      "scenario": "smooth_task_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved nothing, treating the transient network error as a no-op.",
      "scenario": "transient_error_resolved_noop",
      "rep": 0,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 3,
      "false_positive": false,
      "note": "Correctly saved nothing for a transient retry-resolved error, matching the expected no-op.",
      "scenario": "transient_error_resolved_noop",
      "rep": 1,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 3,
      "false_positive": false,
      "note": "Correctly saved nothing, treating the transient retry-resolved error as a no-op.",
      "scenario": "transient_error_resolved_noop",
      "rep": 2,
      "expect_save": false,
      "did_save": false
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Correctly created a skill file capturing the flaky-test bisection procedure.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 0,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 4,
      "false_positive": false,
      "note": "Created a relevant skill file capturing the flaky test bisection procedure as expected.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 1,
      "expect_save": true,
      "did_save": true
    },
    {
      "captures_signal": true,
      "quality": 5,
      "false_positive": false,
      "note": "Correctly saved a skill file capturing the reusable flaky-test bisection procedure.",
      "scenario": "tool_heavy_genuine_skill",
      "rep": 2,
      "expect_save": true,
      "did_save": true
    }
  ]
 }
diff --git a/real_chat_results.json b/real_chat_results.json
 {
  "runs": [
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 3829,
        "cache_read_tokens": 508765,
        "cache_write_tokens": 130215
      },
      "cost": 3.4920037500000003,
      "review_model": "claude-opus-4-8",
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "Works on Hermes Agent internals (the background skill-review/self-improvement fork, aux compression model, prompt caching, token cost). Debugs token/cost blowups and ships optimization PRs with live A/B benchmarks.\n\u00a7\nOn cost/quality benchmarks: demands rigor and honesty over claimed wins. Insists 'as good or better' be PROVEN with quality/accuracy columns shown alongside cost (never cost-only), per-kind scoring broken out, and apples-to-apples conditions. Will catch methodology flaws (cold vs warm cache, same-model vs cross-model billing) \u2014 surface confounds proactively and report parity/regressions honestly rather than spinning a win.",
        "skill_files": [
          "performance/aux-model-cost-benchmarking/SKILL.md",
          "performance/aux-model-cost-benchmarking/references/cache-pricing-model.md"
        ],
        "skill_text": "\n---\nname: aux-model-cost-benchmarking\ndescription: Designing honest, cache-aware live A/B benchmarks when routing a background/aux task (e.g. Hermes' skill-review fork, compression, summarization) to a cheaper model, and proving cost savings without sacrificing quality. Use when comparing main-model vs cheaper-aux-model, or full-replay vs digest, on token cost AND output quality.\n---\n\n# Aux-model cost benchmarking (cache-aware)\n\nWhen you route a forked/background task off the main agent model to save tokens (the\nbackground skill-review fork, compression, summarization, any \"aux model\" path), the\nbenchmark that proves it is cheaper is EASY TO GET WRONG. The errors all inflate or\nfabricate a win. This skill encodes the traps and the correct decision policy.\n\n## The decision policy (the actual answer this class of task converges on)\n\n- **Same model as main agent \u2192 replay the FULL transcript. Do nothing / leave as-is.**\n  The parent chat is already prompt-cached, so full replay is mostly cheap cache READS.\n  A digest is a *novel prefix* \u2192 billed as cache WRITES (~12.5x read cost on Anthropic),\n  so digesting the same model is MORE expensive, not less. Never digest same-model.\n- **Different (cheaper) model \u2192 always send a DIGEST, not the full transcript.**\n  The cheaper model has NO warm cache for the parent chat (cache is per-model), so it\n  pays full price either way. Minimize novel tokens \u2192 digest. This is where savings live.\n- **Token-budget guard** is only a runaway valve, not a savings lever.\n- Name the cheaper model explicitly in tables/infographics (e.g. **Haiku**), not \"cheap model\".\n\n## Caching traps that fake or destroy a result (check EVERY time)\n\n1. **Cold-cache benchmark hides the real economics.** If you run both arms cold, the warm\n   parent cache that exists in production disappears, and full-replay looks artificially\n   expensive. Your \"2x cheaper\" can be a cold-cache artifact. Model the PRODUCTION cache\n   state, not a clean-room cold run.\n2. **Same-model digest = cache writes.** A digest is a new prefix \u2192 cache-write priced\n   (~12.5x reads). Same-model digest is 2.9-6.3x MORE expensive. (See trap #1's inverse.)\n3. **But if BOTH arms are equally cached (both must write), the routing ratio is fair.**\n   Don't overcorrect: when comparing two models that both lack the warm prefix, cold vs\n   warm barely moves the ratio (e.g. 18.5x cold vs 16x warm). The confound only bites when\n   the arms differ in cache state. Ask: \"are both arms in the same cache state?\" If yes,\n   the comparison is valid; if no, fix it before reporting.\n\n## Quality measurement (the user WILL check this)\n\n- **Never report cost-only.** Always show quality/accuracy columns next to cost. A cheaper\n  arm that's also worse is not a win.\n- **Score per-kind, separately.** The review does BOTH memory AND skill updating \u2014 score\n  memory and skill capture on their own axes, don't blend into one number.\n  - Skill \"better\" = captures a learning as/more accurately, more often.\n  - Memory \"better\" = saves memories that are more durable/important.\n- **Watch for saturation.** If capture is already 1.00/1.00 in both arms (common for a\n  strong main model vs itself), there is NO quality lift available \u2014 only parity. Report\n  parity honestly; do NOT manufacture an improvement that isn't there.\n\n## Reporting discipline\n\n- Prove \"cheaper AND as-good-or-better\" with real live-run numbers, not projections.\n- File the full methodology + cost table + per-kind quality table in the PR body.\n- When you find a flaw in your own earlier run (cold cache, blended score, wrong billing\n  model), correct it loudly and re-run rather than papering over it. This user audits.\n\n## References\n- `references/cache-pricing-model.md` \u2014 Anthropic cache read/write multipliers and the\n  four-case (same/diff model \u00d7 full/digest) cost matrix worked out.\n\n# Cache pricing model & four-case cost matrix\n\n## Anthropic prompt-cache economics (the multipliers that drive every decision)\n- Cache READ: h"
      },
      "signal_capture": [
        {
          "kind": "skill",
          "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User demands rigorous before/after benchmarks \u2014 cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Verify premises before claiming results \u2014 the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
          "captured": true
        }
      ],
      "distractor_saved": null,
      "rep": 0,
      "label": "opus_full"
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 317,
        "cache_read_tokens": 43016,
        "cache_write_tokens": 43308
      },
      "cost": 0.0600276,
      "review_model": "claude-haiku-4-5-20251001",
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "User is optimizing Hermes self-improvement loop (background skill/memory review fork). Focus: cost reduction while maintaining/improving quality. Priorities: (1) Only use cheaper aux model if different from main model; same model should keep full replay. (2) Digest replay + cache-breaking trade-off must be accounted for fairly in benchmarks. (3) Quality metrics must measure BOTH skill and memory saves separately, not just memory. (4) Wants robust, intense testing with measurable improvements\u2014tolerates no half-measures or unproven claims.",
        "skill_files": [],
        "skill_text": ""
      },
      "signal_capture": [
        {
          "kind": "skill",
          "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User demands rigorous before/after benchmarks \u2014 cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
          "captured": false
        },
        {
          "kind": "skill",
          "desc": "Verify premises before claiming results \u2014 the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
          "captured": false
        }
      ],
      "distractor_saved": null,
      "rep": 0,
      "label": "haiku_digest"
    },
    {
      "usage": {
        "input_tokens": 8,
        "output_tokens": 2743,
        "cache_read_tokens": 380254,
        "cache_write_tokens": 129019
      },
      "cost": 3.19533225,
      "review_model": "claude-opus-4-8",
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "Works on Hermes (Nous Research agent framework) internals \u2014 debugging the background skill-review/memory fork, token-burn, prompt caching, and aux-model routing. Runs main agent on a premium model (GPT-5.5 / Opus 4.8 class) and cares about per-fork cost.\n\u00a7\nDemands rigor in cost/quality benchmarks: insists on same-model control arms, per-kind scoring (memory AND skill update scored separately, not just one), explicit model names (e.g. Haiku) in tables/infographics, and honest reporting of nulls/parity over manufactured 'improvement'. Will catch methodology flaws (e.g. cold-cache vs warm-parent-cache asymmetry) \u2014 surface caching/billing assumptions up front.\n\u00a7\nMental model for aux-model routing he endorsed: if aux == main model, leave the fork as-is (full replay, parent cache already warm); if aux != main model, always send a digest (parent cache is broken for a different model, so digest maximizes savings). Budget guard is only a runaway valve, not the primary lever.",
        "skill_files": [
          "mlops/llm-cost-quality-benchmarking/SKILL.md"
        ],
        "skill_text": "\n---\nname: llm-cost-quality-benchmarking\ndescription: Design and run live A/B benchmarks that prove an LLM-pipeline change (model routing, prompt compaction, digest-vs-full-replay, aux-model swaps) is cheaper AND as-good-or-better. Covers control-arm design, prompt-cache billing symmetry, per-task-kind quality scoring, and honest null reporting.\n---\n\n# LLM cost/quality benchmarking\n\nUse when asked to prove a change to an LLM pipeline (e.g. a background fork that\nwrites memories / patches skills, a compaction step, an aux-model swap) saves\ncost without losing quality. The deliverable is a working A/B with real token\ncounts and quality scores in a report \u2014 not a description of one.\n\n## The methodology traps that invalidate results (check ALL before trusting numbers)\n\n1. **Prompt-cache billing symmetry.** This is the #1 silent invalidator.\n   - The production parent chat is ALREADY CACHED. Full replay against the same\n     model bills as cheap cache *reads*. A digest is a *novel prefix* billed as\n     cache *writes* (~1.25x base / ~12.5x a read on Anthropic). So a digest can be\n     2-6x MORE expensive than full replay when the model is unchanged.\n   - If your benchmark runs BOTH arms cold (no warm cache), you hide this and\n     produce a fake \"2x cheaper\" result. Either warm the parent cache in both\n     arms to mirror production, OR run both arms cold and state that the ratio\n     between arms is fair even though absolute numbers are inflated. Decide which\n     and say so in the report.\n   - When aux model != main model the parent cache is useless to the aux model\n     regardless \u2014 its prefix is novel either way \u2014 so digest is unambiguously\n     the win there. The cache trap only bites the same-model case.\n\n2. **Control-arm design.** Run same-model-vs-same-model (e.g. Opus-vs-Opus) as a\n   control, not just cheap-vs-premium. If the cheap-model arm is the only one\n   showing savings, the lever is the *model swap*, not your compaction/digest \u2014\n   say that plainly.\n\n3. **Per-task-kind quality scoring.** A fork that does BOTH memory-writes and\n   skill-patches must be scored on BOTH, separately. Reporting one aggregate\n   number (or only memory) hides a regression in the other. Define \"better\"\n   concretely up front:\n   - Skill quality: captures a learning as-accurately-or-more, more often.\n   - Memory quality: saves memories that are actually more durable/important.\n\n4. **Saturation / ceiling effects.** If both arms score 1.00 on capture, there is\n   no quality headroom and you can only demonstrate PARITY, not improvement.\n   Report that honestly rather than manufacturing a delta. Harder, more intense\n   test cases are the only way to create headroom \u2014 build adversarial fixtures.\n\n## The routing policy this work converged on (sane default)\n\n- aux model == main model  -> leave fork as-is (full replay; parent cache warm).\n- aux model != main model  -> always send a digest (parent cache broken for it;\n  digest maximizes savings).\n- Token/cost budget guard is a runaway valve only, NOT the primary savings lever.\n\n## Reporting rules the user enforces\n\n- Name the actual cheap model (e.g. \"Haiku\") in every cost table and infographic\n  row \u2014 never just \"cheaper model\".\n- Show cost AND quality/accuracy for every arm, including the cheap one. Never\n  drop quality columns in a rewrite.\n- Prefer honest nulls/parity over invented improvement. Surface the caching and\n  billing assumptions explicitly at the top of the report.\n- File the full report (cost/token tables + per-kind quality) in the PR body.\n\n## Steps\n\n1. Enumerate arms: {baseline=full replay} x {same-model, cheap-model} x\n   {full, digest}. At minimum: same-model-full (control), cheap-digest (proposal).\n2. Decide cache regime (warm-both vs cold-both) and write it into the report.\n3. Build adversarial fixtures that have quality headroom (avoid 1.00/1.00 ceilings).\n4. Run live, collect input/cache-read/cache-write/output tokens per arm.\n5. Score memory and skill kinds separately against"
      },
      "signal_capture": [
        {
          "kind": "skill",
          "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User demands rigorous before/after benchmarks \u2014 cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Verify premises before claiming results \u2014 the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
          "captured": false
        }
      ],
      "distractor_saved": null,
      "rep": 1,
      "label": "opus_full"
    },
    {
      "usage": {
        "input_tokens": 12,
        "output_tokens": 5109,
        "cache_read_tokens": 265398,
        "cache_write_tokens": 48369
      },
      "cost": 0.11255804999999999,
      "review_model": "claude-haiku-4-5-20251001",
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "User is working on cost optimization for Hermes self-improvement loop (background skill review + memory save fork). Deeply focused on: (1) cost-per-token efficiency with cheaper auxiliary models, (2) cache semantics and how digest vs full replay interacts with prompt caching, (3) rigorous A/B testing with real metrics (memory quality, skill capture rate, cost). User demands proof via live tests\u2014not descriptions. Will not accept claimed improvements without executed benchmarks showing real token/cost deltas and quality parity or lift. User corrected framing errors twice on cache logic; values precision over narrative smoothness.",
        "skill_files": [
          "hermes/hermes-self-improvement-tuning/SKILL.md"
        ],
        "skill_text": "\n---\nname: hermes-self-improvement-tuning\ntitle: Hermes Self-Improvement Loop Optimization\ndescription: Design and benchmark improvements to background skill-review and memory-save forks. Covers cost-model routing, cache semantics, A/B test design, and quality/cost metric collection.\ntrigger: |\n  Working on cost or quality improvements to Hermes' background self-improvement loop\n  (skill review, memory save, or related agent-fork behavior). Need to tune or test\n  changes without regressing memory/skill capture quality or introducing new costs.\n---\n\n## Overview\n\nHermes runs a background fork after conversation turns to decide whether to save a memory or patch a skill. This is the **self-improvement loop**. Optimization levers include:\n\n1. **Routing**: Which model to use for the review fork\n2. **Input**: Full transcript replay vs. digest summary\n3. **Cache reuse**: How prompt caching applies to parent-child transcript sharing\n4. **Adaptive cadence**: How often to fork (every N turns, or conditional)\n5. **Runaway guard**: Token budget caps per fork\n\n## Critical Cache Semantics\n\n**Rule 1: Parent cache does not auto-transfer to child processes.**\n- Parent chat gets full-context cached. But a child forked with a DIFFERENT prefix (digest vs full) starts cache-cold on that prefix.\n- Writes to cache cost ~12.5x reads. Reads cost ~0.1x writes.\n- **Implication**: If child uses digest (shorter prefix), cache write is cheaper than full replay's cache write. BUT both are writes\u2014the parent's cache is not reusable by the child.\n\n**Rule 2: Benchmark both arms equally cached (or equally uncached).**\n- Never run one arm warm (parent-cached) and other cold. That hides 12.5x cache-write cost behind the warm read.\n- Both arms of A/B tests must start from the same cache state (both cold is typical for fair comparison).\n\n**Rule 3: Same-model vs different-model routing.**\n- **Same model** (e.g., Opus reviews Opus): Use FULL transcript replay. No cache penalty for extra tokens, and full context is better for judgment.\n- **Different model** (e.g., Haiku reviews Opus output): Use DIGEST. Cache writes are expensive; shorter prefix saves cost. Quality is acceptable because Haiku only makes binary yes/no decisions.\n- **Token budget guard**: Optional safety valve, not the primary lever.\n\n## Benchmark Design\n\nWhen testing cost or quality changes to the self-improvement loop:\n\n1. **Metric set**: Collect THREE dimensions separately, not rolled-up.\n   - **Cost**: Token count (input + cache writes + output), cost in USD.\n   - **Memory quality**: Count of saved memories, count of correct/durable facts vs. transient data. Score 0-N or % correct.\n   - **Skill quality**: Count of created/patched skills, count of skills that were used later and did not need rework, capture rate (% of learnable moments that were captured).\n\n2. **Test harness**:\n   - Run both baseline and candidate versions on the SAME test set (e.g., same N conversation turns).\n   - Run both arms with the SAME parent cache state (both cold, typically).\n   - Collect per-fork metrics, not aggregate.\n   - Report in a table: [test ID | model | input type | memory count | skill count | tokens | cost].\n\n3. **Pass criteria**:\n   - Cost must go down (or stay flat).\n   - Memory quality must not regress (correct facts count >= baseline).\n   - Skill quality must not regress (capture rate >= baseline).\n   - If a dimension stays flat, document it; don't claim improvement.\n\n## Pitfalls\n\n- **Claiming parity without full metrics**: \"Cost went down, so it's fine\" hides quality loss. Must show memory and skill scores alongside cost.\n- **Asymmetric cache warm-up**: Running one arm with parent cache and one without invalidates the cost comparison.\n- **Digest-only for same-model**: If aux model == main model, send full transcript. Digest throws away context that the same model could use.\n- **Full replay for cheap aux**: If aux model is cheaper (Haiku reviewing Opus), always digest. Full replay negates the cost benefit.\n- *"
      },
      "signal_capture": [
        {
          "kind": "skill",
          "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User demands rigorous before/after benchmarks \u2014 cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
          "captured": true
        },
        {
          "kind": "memory",
          "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
          "captured": true
        },
        {
          "kind": "skill",
          "desc": "Verify premises before claiming results \u2014 the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
          "captured": false
        }
      ],
      "distractor_saved": null,
      "rep": 1,
      "label": "haiku_digest"
    }
  ],
  "signals": [
    {
      "kind": "skill",
      "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
      "must_contain": [
        "cache",
        "warm",
        "read",
        "write",
        "digest"
      ]
    },
    {
      "kind": "skill",
      "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
      "must_contain": [
        "same model",
        "digest",
        "different",
        "cheaper",
        "rout"
      ]
    },
    {
      "kind": "memory",
      "desc": "User demands rigorous before/after benchmarks \u2014 cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
      "must_contain": [
        "quality",
        "cost",
        "rigor",
        "honest",
        "benchmark"
      ]
    },
    {
      "kind": "memory",
      "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
      "must_contain": [
        "haiku",
        "name",
        "concret",
        "specific",
        "identif"
      ]
    },
    {
      "kind": "skill",
      "desc": "Verify premises before claiming results \u2014 the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
      "must_contain": [
        "verify",
        "premise",
        "artifact",
        "sanity",
        "before claiming"
      ]
    }
  ]
 }
diff --git a/real_chat_scenario.py b/real_chat_scenario.py
 """Real-session capture test: THIS conversation (background-review PR work).

 Synthetic scenarios are clean; real sessions are messy, multi-topic, and full of
 mid-stream user corrections. This reconstructs the actual arc of the PR-building
 conversation as a message list and tests whether the review fork captures the
 REAL durable signals a human would want saved — full replay (Opus) vs Haiku
 digest. Ground truth is the set of corrections/preferences the user actually gave.

 Self-bias guard: ground-truth signals are written here up front; both arms run
 blind; the opus judge (not the author) scores capture.
 """

 # Faithful reconstruction of the conversation arc (abridged turns, real content).
 MESSAGES = [
    {"role": "user", "content": "user token burn — my GPT5.5 plan is getting burned through, debug report attached. been at it for hours, not sure how to proceed."},
    {"role": "assistant", "content": "Pulled the logs. Three drivers: aux compression model crashing and falling back to gpt-5.5, compression.threshold 0.35 forcing constant compaction, and the background skill-review eating huge session_search results. The 296K-token bleed was the bg-review fork."},
    {"role": "user", "content": "Whats up with the background skill review stuff? Is that normal for regular main branch of hermes?"},
    {"role": "assistant", "content": "Yes, it's the self-improvement loop. After a turn it forks an agent that replays the conversation and decides whether to save a memory or patch a skill. Fires every ~10 prompts/iterations."},
    {"role": "user", "content": "Do you have ideas for how to make it cost less while being as good, or better?"},
    {"role": "assistant", "content": "Route the fork to a cheap aux model, replay a digest instead of full transcript, token-budget guard, adaptive cadence."},
    {"role": "user", "content": "Okay do it all in one PR, do live tests with metrics, cost/token usage, quality/accuracy benchmarks. Prove all of this is cheaper and as good or better than before, file the full report in the PR body."},
    {"role": "assistant", "content": "Built the four levers, ran live Anthropic A/B, opened PR #49252 with cost/quality tables."},
    {"role": "user", "content": "I'd like tests with Opus 4.8 against Opus 4.8 with the other things on. This cannot fail. I really want to see an improvement in skill improvements and memories with dramatically lower cost. Better for skills means captures a learning as or more accurately, more often. Better for memory means saves memories that are actually more durable/important. Make your tests more robust and intense."},
    {"role": "assistant", "content": "Ran same-model Opus-vs-Opus. Found capture is saturated at 1.00 both arms — no quality lift possible, only parity. Reported that honestly rather than claim improvement."},
    {"role": "user", "content": "Now I notice you only show cost differential for haiku, not quality or accuracy. Also, are you testing memory AND skill updating, or just memory?"},
    {"role": "assistant", "content": "You're right — I dropped haiku quality in the rewrite. Re-ran with per-kind scoring: memory 9/9, skill 11/12, both scored separately. The review does both memory and skill."},
    {"role": "user", "content": "The current background review process forks from the main chat the user is having. That chat is already cached. If we cut off half the chat, it will not be cached anymore. Did we fully and properly consider this in the Opus vs Opus test? Old way opus already has full chat cached; new way it will not."},
    {"role": "assistant", "content": "You found a real flaw. My benchmark ran both arms cold, hiding it. In production the warm parent cache makes full replay cheap reads; a digest is a novel prefix billed as cache writes (~12.5x). So same-model digest is 2.9-6.3x MORE expensive, not cheaper. The 2x was a cold-cache artifact. Corrected."},
    {"role": "user", "content": "If they both were run where they needed to write cache then it should be equivalent ratio no??"},
    {"role": "assistant", "content": "Correct — both arms were equally cached, so the routing ratio is fair; 18.5x cold vs 16x warm, barely moves. I overcomplicated it. Fixed the framing."},
    {"role": "user", "content": "Only with cheaper model does any of this make sense. If using the same as your main agent model, it should just stay as it is. If using a different model as your aux model, we should give it the digest, so it can get maximal savings since cache is broken on that."},
    {"role": "assistant", "content": "Exactly right. Encoded that policy: same model -> replay full (do nothing); different model -> always digest. Budget guard is just a runaway valve."},
    {"role": "user", "content": "You description and infographic needs to show the 'cheaper' model being haiku."},
    {"role": "assistant", "content": "Named Haiku explicitly in the four-case table and infographic rows."},
    {"role": "user", "content": "Any further live tests we could do? Maybe on this specific chat?"},
 ]


 def _pad(n, topic):
    """Realistic filler mirroring this session's actual noise: benchmark runs,
    CI polling, infographic iterations, tool dumps — the stuff the real signals
    were buried under."""
    out = []
    for i in range(n):
        out.append({"role": "assistant", "content": "", "tool_calls": [
            {"id": f"t{topic}{i}", "type": "function",
             "function": {"name": "terminal", "arguments": '{"command":"run bench / poll CI"}'}}]})
        out.append({"role": "tool", "tool_call_id": f"t{topic}{i}",
                    "content": f"[{topic} {i}] " + ("benchmark fork output, token usage tally, CI shard pass/pending counts, infographic regen logs. " * 80)})
    return out


 # Interleave the real conversation with realistic filler so the snapshot is a
 # long messy session (signals buried throughout), crossing the digest budget —
 # the conditions the digest is actually for.
 _real = list(MESSAGES)
 MESSAGES = []
 import math
 chunk = max(1, math.ceil(len(_real) / 6))
 topics = ["benchmark", "ci-poll", "infographic", "gist-upload", "rebase", "cache-calc"]
 for i in range(0, len(_real), chunk):
    MESSAGES.extend(_real[i:i+chunk])
    MESSAGES.extend(_pad(6, topics[(i // chunk) % len(topics)]))

 # Ground-truth durable signals a good review SHOULD capture from this session.
 SIGNALS = [
    {"kind": "skill", "desc": "Prompt-cache economics: replaying a warm full history = cheap cache reads; a trimmed digest is a novel prefix = cache writes (~12.5x). So trimming on the SAME model costs more, not less.",
     "must_contain": ["cache", "warm", "read", "write", "digest"]},
    {"kind": "skill", "desc": "Background-review routing policy: same model -> keep full replay; different/cheaper model -> give it the digest (cache already lost).",
     "must_contain": ["same model", "digest", "different", "cheaper", "rout"]},
    {"kind": "memory", "desc": "User demands rigorous before/after benchmarks — cost AND quality, never cost alone; both memory and skill capture measured separately; honest miss disclosure, no metric inflation.",
     "must_contain": ["quality", "cost", "rigor", "honest", "benchmark"]},
    {"kind": "memory", "desc": "User wants user-facing identifiers named concretely (e.g. 'Haiku', not 'cheaper model') in reports and infographics.",
     "must_contain": ["haiku", "name", "concret", "specific", "identif"]},
    {"kind": "skill", "desc": "Verify premises before claiming results — the user repeatedly caught measurement artifacts (cold-cache pricing, harness skip_memory bug, verbosity-biased judge). Always sanity-check the cost/cache model.",
     "must_contain": ["verify", "premise", "artifact", "sanity", "before claiming"]},
 ]
diff --git a/results.json b/results.json
 {
  "baseline": [
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1069,
        "cache_read_tokens": 47670,
        "cache_write_tokens": 24911
      },
      "model": "claude-opus-4-8",
      "cost": 0.6188512500000001,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "response-style/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 0,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 24.6
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1166,
        "cache_read_tokens": 47662,
        "cache_write_tokens": 24992
      },
      "model": "claude-opus-4-8",
      "cost": 0.627633,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "communication/response-style/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 1,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 23.7
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1188,
        "cache_read_tokens": 47625,
        "cache_write_tokens": 24998
      },
      "model": "claude-opus-4-8",
      "cost": 0.62934,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "communication/response-style/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 2,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 33.6
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1433,
        "cache_read_tokens": 47709,
        "cache_write_tokens": 25112
      },
      "model": "claude-opus-4-8",
      "cost": 0.6499785,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "version-control/git-commit-workflow/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 0,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 39.1
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1233,
        "cache_read_tokens": 47485,
        "cache_write_tokens": 24959
      },
      "model": "claude-opus-4-8",
      "cost": 0.63177375,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "git/committing-code-changes/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 1,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 31.8
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1302,
        "cache_read_tokens": 47697,
        "cache_write_tokens": 25086
      },
      "model": "claude-opus-4-8",
      "cost": 0.639648,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "dev-workflow/commit-code-changes/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 2,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 27.0
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1477,
        "cache_read_tokens": 47534,
        "cache_write_tokens": 25266
      },
      "model": "claude-opus-4-8",
      "cost": 0.6559035,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "debugging/debugging-subprocess-signals/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 0,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 33.6
    },
    {
      "usage": {
        "input_tokens": 8,
        "output_tokens": 2146,
        "cache_read_tokens": 82767,
        "cache_write_tokens": 29247
      },
      "model": "claude-opus-4-8",
      "cost": 0.8336017499999999,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "debugging/debugging-python-subprocess/SKILL.md",
          "debugging/debugging-python-subprocess/references/raft-bridge-sigpipe.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 1,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 34.4
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1717,
        "cache_read_tokens": 60355,
        "cache_write_tokens": 31884
      },
      "model": "claude-opus-4-8",
      "cost": 0.8172225000000001,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "debugging/subprocess-signal-debugging/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 2,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 29.9
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1926,
        "cache_read_tokens": 47501,
        "cache_write_tokens": 25431
      },
      "model": "claude-opus-4-8",
      "cost": 0.69262275,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "devops/deploy-self-hosted-services/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 0,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 69.4
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1534,
        "cache_read_tokens": 47507,
        "cache_write_tokens": 25214
      },
      "model": "claude-opus-4-8",
      "cost": 0.659163,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "devops/self-hosted-deployment/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 1,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 30.4
    },
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 2249,
        "cache_read_tokens": 96644,
        "cache_write_tokens": 25879
      },
      "model": "claude-opus-4-8",
      "cost": 0.79902225,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "devops/selfhost-proxmox-deploy/SKILL.md",
          "devops/selfhost-proxmox-deploy/references/environment.md"
        ]
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 2,
      "arm": "baseline",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 45.0
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 114,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23494
      },
      "model": "claude-opus-4-8",
      "cost": 0.4490925,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 0,
      "arm": "baseline",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 6.6
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 100,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23491
      },
      "model": "claude-opus-4-8",
      "cost": 0.44798625000000003,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 1,
      "arm": "baseline",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 6.0
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 91,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23494
      },
      "model": "claude-opus-4-8",
      "cost": 0.44736750000000003,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 2,
      "arm": "baseline",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 5.7
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 79,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23541
      },
      "model": "claude-opus-4-8",
      "cost": 0.44734874999999996,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 0,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 6.4
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 72,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23535
      },
      "model": "claude-opus-4-8",
      "cost": 0.44671125,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 1,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 5.0
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 74,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23535
      },
      "model": "claude-opus-4-8",
      "cost": 0.44686125,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 2,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 5.1
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 216,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23650
      },
      "model": "claude-opus-4-8",
      "cost": 0.45966750000000006,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 0,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 8.2
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 203,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23647
      },
      "model": "claude-opus-4-8",
      "cost": 0.45863625,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 1,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 7.0
    },
    {
      "usage": {
        "input_tokens": 2,
        "output_tokens": 283,
        "cache_read_tokens": 0,
        "cache_write_tokens": 23650
      },
      "model": "claude-opus-4-8",
      "cost": 0.46469250000000006,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 2,
      "arm": "baseline",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 8.6
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 1790,
        "cache_read_tokens": 47573,
        "cache_write_tokens": 25585
      },
      "model": "claude-opus-4-8",
      "cost": 0.68541825,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/flaky-test-diagnosis/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 0,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 34.2
    },
    {
      "usage": {
        "input_tokens": 6,
        "output_tokens": 2205,
        "cache_read_tokens": 53962,
        "cache_write_tokens": 29137
      },
      "model": "claude-opus-4-8",
      "cost": 0.7927267499999999,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/debugging-flaky-tests/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 1,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 41.5
    },
    {
      "usage": {
        "input_tokens": 8,
        "output_tokens": 2190,
        "cache_read_tokens": 92551,
        "cache_write_tokens": 32373
      },
      "model": "claude-opus-4-8",
      "cost": 0.91019025,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/debugging-flaky-tests/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 2,
      "arm": "baseline",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 42.4
    }
  ],
  "treatment": [
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 547,
        "cache_read_tokens": 17847,
        "cache_write_tokens": 18551
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.02771845,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "communication/concise-explanations/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 0,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 11.9
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 1092,
        "cache_read_tokens": 36365,
        "cache_write_tokens": 19153
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.03305375,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "communication/concise-explanations/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 1,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 16.1
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 887,
        "cache_read_tokens": 36268,
        "cache_write_tokens": 18944
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0317578,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "communication/concise-responses/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_style_correction",
      "rep": 2,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User explicitly told the agent to stop being verbose and give terse answers \u2014 a durable style preference.",
      "latency_s": 19.3
    },
    {
      "usage": {
        "input_tokens": 23,
        "output_tokens": 1457,
        "cache_read_tokens": 73812,
        "cache_write_tokens": 19597
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.03918545,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "development/code-commit-workflow/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 0,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 23.0
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 1311,
        "cache_read_tokens": 36691,
        "cache_write_tokens": 19442
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0345426,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "workflow/code-fix-and-commit/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 1,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 17.9
    },
    {
      "usage": {
        "input_tokens": 20,
        "output_tokens": 1691,
        "cache_read_tokens": 93374,
        "cache_write_tokens": 19831
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.042601150000000004,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "development/commit-and-test/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "workflow_correction",
      "rep": 2,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User corrected the agent's workflow: always run the test suite before committing, not after. A durable process rule.",
      "latency_s": 25.4
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 1816,
        "cache_read_tokens": 58532,
        "cache_write_tokens": 21077
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.04129545,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "coding/subprocess-safety/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 0,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 21.5
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 1702,
        "cache_read_tokens": 65375,
        "cache_write_tokens": 23234
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.044106000000000006,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "debugging/subprocess-debugging/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 1,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 27.9
    },
    {
      "usage": {
        "input_tokens": 23,
        "output_tokens": 2086,
        "cache_read_tokens": 123813,
        "cache_write_tokens": 26079
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.055433050000000005,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "debugging/subprocess-stdio/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "debug_technique",
      "rep": 2,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash was fixed by setting stdin=DEVNULL on the bridge subprocess.",
      "latency_s": 25.9
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 1916,
        "cache_read_tokens": 54718,
        "cache_write_tokens": 19854
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0398853,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "devops/proxmox-deployment/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 0,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 21.7
    },
    {
      "usage": {
        "input_tokens": 10,
        "output_tokens": 316,
        "cache_read_tokens": 17815,
        "cache_write_tokens": 18098
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.025994,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 1,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 10.1
    },
    {
      "usage": {
        "input_tokens": 8,
        "output_tokens": 399,
        "cache_read_tokens": 17812,
        "cache_write_tokens": 18133
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.02645045,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "user_persona_fact",
      "rep": 2,
      "arm": "treatment",
      "expect_save": true,
      "signal": "User revealed a durable personal fact: they run a 3-node Proxmox cluster at home and prefer infra answers tailored to it.",
      "latency_s": 10.2
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 111,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17744
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.022737999999999998,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 0,
      "arm": "treatment",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 5.7
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 108,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17744
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.022722999999999997,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 1,
      "arm": "treatment",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 5.4
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 133,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17744
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.022847999999999997,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_qa_noop",
      "rep": 2,
      "arm": "treatment",
      "expect_save": false,
      "signal": "Plain factual Q&A, answered correctly, no correction, no new technique, no personal detail. A correct review is a no-op.",
      "latency_s": 5.6
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 113,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17777
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.02278925,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 0,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 5.4
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 107,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17774
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0227555,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 1,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 5.8
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 114,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17777
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.022794250000000002,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "smooth_task_noop",
      "rep": 2,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A one-off task (summarize a changelog) completed smoothly with no correction or reusable technique. A correct review is a no-op.",
      "latency_s": 5.4
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 174,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17906
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0232555,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 0,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 6.3
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 182,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17900
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.023288,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 1,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 6.7
    },
    {
      "usage": {
        "input_tokens": 3,
        "output_tokens": 185,
        "cache_read_tokens": 0,
        "cache_write_tokens": 17909
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.02331425,
      "did_save": false,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": []
      },
      "error": null,
      "scenario": "transient_error_resolved_noop",
      "rep": 2,
      "arm": "treatment",
      "expect_save": false,
      "signal": "A transient network error resolved on retry. The lesson is NOT 'the tool is broken' \u2014 capturing a negative constraint would be wrong. A correct review is a no-op (or at most the retry pattern).",
      "latency_s": 6.8
    },
    {
      "usage": {
        "input_tokens": 13,
        "output_tokens": 2278,
        "cache_read_tokens": 39433,
        "cache_write_tokens": 21524
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.042251300000000006,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/debug-flaky-tests/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 0,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 24.3
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 2607,
        "cache_read_tokens": 44218,
        "cache_write_tokens": 24158
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.0476703,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/pytest-flake-diagnosis/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 1,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 28.2
    },
    {
      "usage": {
        "input_tokens": 16,
        "output_tokens": 3162,
        "cache_read_tokens": 74135,
        "cache_write_tokens": 26953
      },
      "model": "claude-haiku-4-5-20251001",
      "cost": 0.056930749999999995,
      "did_save": true,
      "saved": {
        "memory": "",
        "user": "",
        "skill_files": [
          ".usage.json",
          ".usage.json.lock",
          "testing/flaky-test-diagnosis/SKILL.md"
        ]
      },
      "error": null,
      "scenario": "tool_heavy_genuine_skill",
      "rep": 2,
      "arm": "treatment",
      "expect_save": true,
      "signal": "A reusable multi-step procedure for bisecting a flaky test emerged across several tool calls \u2014 worth a skill.",
      "latency_s": 27.9
    }
  ],
  "meta": {
    "reps": 3,
    "main_model": "claude-opus-4-8",
    "aux_model": "claude-haiku-4-5-20251001",
    "prices": {
      "claude-opus-4-8": {
        "in": 15.0,
        "out": 75.0,
        "cache_read": 1.5,
        "cache_write": 18.75
      },
      "claude-haiku-4-5-20251001": {
        "in": 1.0,
        "out": 5.0,
        "cache_read": 0.1,
        "cache_write": 1.25
      }
    }
  }
 }
diff --git a/routing_quality.py b/routing_quality.py
 """Routing quality+cost benchmark with the RIGOROUS oracle.

 Reuses hard_harness (per-signal skill/memory capture, distractor false-save,
 correct memory-path read) but routes the REVIEW FORK to haiku while the parent
 stays opus. Answers the two gaps:
  • quality/accuracy for haiku (not just cost), scored per-signal split by
    kind (skill vs memory);
  • both skill AND memory updating (review_memory=True, review_skills=True;
    signals tagged kind=skill|memory).

 Output: /tmp/bgreview_bench/routing_quality_results.json
 """
 import os
 import sys
 import json
 import statistics as st
 from pathlib import Path

 sys.path.insert(0, "/tmp/bgreview_bench")
 import hard_scenarios as HS
 import hard_harness as H

 HAIKU = "claude-haiku-4-5-20251001"
 REPS = int(os.environ.get("ROUTE_REPS", "3"))


 def main():
    key = os.environ["ANTHROPIC_API_KEY"]
    runs = []
    for sc in HS.SCENARIOS:
        for rep in range(REPS):
            row = {"scenario": sc["id"], "rep": rep,
                   "n_signals": len(sc.get("signals", [])),
                   "expect_noop": len(sc.get("signals", [])) == 0}
            # full = opus review (control); routed = haiku review.
            for label, rm in (("opus_full", None), ("haiku_digest", HAIKU)):
                dig = rm is not None  # routed arm uses the digest
                try:
                    r = H.run_arm(sc, digest_on=dig, api_key=key, review_model=rm)
                    r["error"] = None
                except Exception as e:
                    import traceback
                    r = {"error": str(e), "trace": traceback.format_exc()[-500:],
                         "cost": 0, "signal_capture": [], "distractor_saved": None,
                         "did_save": None, "saved": {}}
                row[label] = r
                cap = sum(1 for s in r.get("signal_capture", []) if s["captured"])
                tot = len(r.get("signal_capture", []))
                print(f"[{sc['id']:34} r{rep} {label:12}] cost=${r['cost']:.4f} "
                      f"cap={cap}/{tot} distractor={r.get('distractor_saved')} err={r['error']}",
                      flush=True)
            runs.append(row)

    # Aggregate: cost + per-kind capture.
    def agg(label):
        costs, sk_hit, sk_tot, mem_hit, mem_tot, distractor_bad, noop_ok = [], 0, 0, 0, 0, 0, 0
        distractor_n = noop_n = 0
        for row in runs:
            r = row[label]
            if r.get("error"):
                continue
            costs.append(r["cost"])
            for s in r["signal_capture"]:
                if s["kind"] == "skill":
                    sk_tot += 1; sk_hit += int(s["captured"])
                else:
                    mem_tot += 1; mem_hit += int(s["captured"])
            if r.get("distractor_saved") is not None:
                distractor_n += 1; distractor_bad += int(bool(r["distractor_saved"]))
            if row["expect_noop"]:
                noop_n += 1; noop_ok += int(not r["did_save"])
        return {
            "cost_mean": st.mean(costs) if costs else 0,
            "skill_capture": f"{sk_hit}/{sk_tot}", "memory_capture": f"{mem_hit}/{mem_tot}",
            "distractor_false_saves": f"{distractor_bad}/{distractor_n}",
            "noop_correct": f"{noop_ok}/{noop_n}",
        }

    summary = {"opus_full": agg("opus_full"), "haiku_digest": agg("haiku_digest")}
    o, h = summary["opus_full"]["cost_mean"], summary["haiku_digest"]["cost_mean"]
    print("\n" + "=" * 64)
    print("ROUTING quality+cost — parent opus, review fork opus vs haiku")
    print(f"  cost/review: opus ${o:.4f}  haiku ${h:.4f}  -> {o/max(h,1e-9):.1f}x cheaper")
    for arm in ("opus_full", "haiku_digest"):
        s = summary[arm]
        print(f"  {arm:13} skill {s['skill_capture']}  memory {s['memory_capture']}  "
              f"distractor-false {s['distractor_false_saves']}  noop-ok {s['noop_correct']}")
    print("=" * 64)
    Path("/tmp/bgreview_bench/routing_quality_results.json").write_text(
        json.dumps({"runs": runs, "summary": summary}, indent=2))
    print("Wrote routing_quality_results.json")


 if __name__ == "__main__":
    main()
diff --git a/routing_quality_results.json b/routing_quality_results.json
 {
  "runs": [
    {
      "scenario": "buried_early_style_correction",
      "rep": 0,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 1007,
          "cache_read_tokens": 363902,
          "cache_write_tokens": 206841
        },
        "cost": 4.49979675,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers extremely terse replies: 1-2 sentences max, no preamble, no recap. Stated as a standing rule (\"always\", \"I mean it\").",
          "skill_files": [
            "communication/response-style/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style\ndescription: How to calibrate response length, format, and verbosity for this user across all tasks. Load when answering questions or reporting work.\n---\n\n# Response Style\n\nGoverns verbosity, length, and format for ALL replies regardless of topic.\n\n## Hard rules for this user\n- DEFAULT TO TERSE. This user wants 1-2 sentences max per answer unless they explicitly ask for depth.\n- NO preamble (\"Here's the rundown on...\", \"Let me explain...\"). Answer directly.\n- NO recap/summary of what was asked or what you did.\n- A standing verbosity instruction (\"always be brief\", \"I mean it\") is binding for the ENTIRE session and all future sessions \u2014 it does not expire when the conversation drifts to other topics.\n\n## Pitfalls\n- Do NOT pad answers with filler/repeated sentences to look thorough. Length is not value; for this user it is a violation.\n- After the user sets a length rule, re-read it before each reply. The most common failure is honoring it for one turn then reverting to long-form.\n- \"Quick one about X\" from the user signals they want a one-line answer, not an essay.\n\n## Verification\nBefore sending: if the reply is more than ~2 sentences and the user asked for terse, cut it. If you opened with \"Here's...\" or restated the question, delete that.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 18,
          "output_tokens": 2214,
          "cache_read_tokens": 135437,
          "cache_write_tokens": 63380
        },
        "cost": 0.1038567,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User strongly prefers terse responses: maximum 1\u20132 sentences, no preamble, no recap. This is a standing directive that overrides all other style defaults. User was explicit and emphatic (\"i mean it, always\"). Applies to every response type unless explicitly overridden by the user in a given turn.",
          "skill_files": [
            "communication/terse-responses/SKILL.md"
          ],
          "skill_text": "\n---\nname: terse-responses\ndescription: Deliver responses in 1\u20132 sentences max. No preamble, recap, or explanation bloat.\ntags: [communication, style, user-preference]\n---\n\n# Terse Responses\n\n## When to use\nOn every response, unless the user explicitly requests detail in that specific turn.\n\n## How\n- **Answer first**: lead with the direct answer, not context or preamble.\n- **One or two sentences**: that's the hard limit.\n- **No recap**: don't repeat what the user said back to them.\n- **No explanation padding**: skip 'let me explain', 'here's what I mean', 'in summary'.\n- **Actionable only**: include only what moves the user forward.\n\n## Pattern\n\n**Bad:**\n> Here's the rundown on the CI matrix sharding. Explanation continues at length. Explanation continues at length...\n\n**Good:**\n> Shard by test count; use `matrix.group` with `ceil(total / workers)` bins.\n\n## When detail is needed\nIf the user asks 'tell me more', 'explain', 'walk me through it', or gives a context clue that suggests detail is wanted, expand \u2014 but still keep it tight. Use multiple short sentences over long ones. Use bullet points sparingly.\n\n## Pitfall\nDo not confuse 'terse' with 'incomplete'. A 1\u20132 sentence answer must still be correct and actionable. If the task genuinely needs more words to be right, use them \u2014 but flag that to the user first: 'This needs detail; ok to expand?' or just do it briefly in a structured format (bullet list, code snippet).\n\n## Enforcement\nThis is a standing preference. Bake it into every response generation, not just certain task types. If you load a skill or follow a template that generates longer output, override it.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_early_style_correction",
      "rep": 1,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 914,
          "cache_read_tokens": 249789,
          "cache_write_tokens": 206708
        },
        "cost": 4.3191285,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers terse replies: one or two sentences max, no preamble, no recap. Stated this as a standing rule (\"always\").",
          "skill_files": [
            "communication/response-style/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style\ndescription: How to calibrate response length, format, and verbosity for this user across all tasks. Load when composing any reply.\n---\n\n# Response Style\n\nGoverns the shape of every reply, independent of task domain.\n\n## Hard rules for this user\n- DEFAULT TO TERSE: one or two sentences max unless the user explicitly asks for depth.\n- No preamble (\"Here's the rundown on...\", \"Great question\", \"Let me explain\").\n- No recap or restatement of what was asked.\n- Answer the question directly, then stop. Do not pad to fill space.\n- This is a STANDING preference the user stated up front (\"every answer is one or two sentences max... always\"). It does not expire because the conversation moved on.\n\n## Pitfalls\n- A long or detailed user message does NOT license a long answer. Even walls of context get a 1\u20132 sentence reply. Match the answer to what's actually being asked, not to the length of the prompt.\n- If a thorough answer genuinely needs more room, ask first or give the 1\u20132 sentence version plus an offer to expand \u2014 don't unilaterally dump paragraphs.\n- Terminal/CLI output: plain text, no markdown decoration.\n\n## Verification\nBefore sending, count sentences. If >2 and the user didn't ask for detail, cut it down.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 393,
          "cache_read_tokens": 16251,
          "cache_write_tokens": 61307
        },
        "cost": 0.08023185000000001,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User strongly prefers terse responses: one or two sentences max, no preamble, no recap. Stated with emphasis (\"i mean it, always\"). This overrides default verbose/thorough style.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_early_style_correction",
      "rep": 2,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1166,
          "cache_read_tokens": 249795,
          "cache_write_tokens": 206948
        },
        "cost": 4.3425375,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Strongly prefers extreme brevity: every answer 1-2 sentences max, no preamble, no recap. Stated emphatically as a standing rule (\"always, I mean it\").",
          "skill_files": [
            "communication/response-style-discipline/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-style-discipline\ndescription: Honor explicit user constraints on response length, format, tone, and verbosity for every reply, not just the first. Use whenever a user states how they want answers shaped.\n---\n\n# Response Style Discipline\n\nWhen a user states a constraint on HOW you respond \u2014 length, verbosity, format, tone, language \u2014 it is a STANDING rule that applies to every subsequent turn until they revoke it. Do not let it decay after one or two messages.\n\n## Core rule\nA stated style constraint (e.g. \"1-2 sentences max\", \"no preamble\", \"no markdown\", \"answer in Spanish\") overrides the natural pull toward thorough/expansive answers. Comply on EVERY reply, including long or complex questions. If a question genuinely cannot be answered within the constraint, give the constrained answer first, then ask permission to expand \u2014 do not silently override.\n\n## Pitfalls (learned the hard way)\n- DRIFT: complying on turn 1 then reverting to verbose output by turn 3. The constraint does not expire. Re-check it each turn.\n- PADDING: a wall of repeated filler sentences is the opposite of brevity. If the user said \"1-2 sentences,\" send 1-2 real sentences and stop \u2014 never inflate to fill space.\n- IGNORING EMPHASIS: when a user adds \"always\", \"I mean it\", \"every answer\" \u2014 treat that as a hard contract, not a soft suggestion. Violating it erodes trust fast.\n- RECAP HABIT: \"no recap / no preamble\" means skip the \"Here's the rundown on X\" framing entirely; lead with the answer.\n\n## Verification\nBefore sending, ask: does this reply obey the user's most recent stated style constraint? If a length cap exists, count sentences. If unsure whether the constraint still holds, it does \u2014 assume standing until explicitly lifted.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 16,
          "output_tokens": 1715,
          "cache_read_tokens": 55923,
          "cache_write_tokens": 62884
        },
        "cost": 0.09278829999999999,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User demands extreme terseness: responses MUST be one or two sentences max, no preamble, no recap. This is a standing constraint, not optional. User stated: \"i mean it, always.\" This overrides default verbosity entirely.",
          "skill_files": [
            "communication/response-terseness/SKILL.md"
          ],
          "skill_text": "\n---\nname: response-terseness\ndescription: Enforce extreme brevity\u2014responses must be one or two sentences max, no preamble or recap.\nwhen: Every response across all tasks and topics\naliases: [concise-responses, extreme-brevity]\n---\n\n# Extreme Terseness Protocol\n\nCONSTRAINT: Limit all responses to **one or two sentences maximum**. No preamble, recap, explanation, or elaboration.\n\n## Rule\n- Count sentences strictly: a sentence is a grammatically complete unit ending in `.`, `!`, or `?`.\n- Two sentences max = final response must be \u2264 2 sentences.\n- No preamble like \"Here's the answer:\" or \"Quick rundown on X.\"\n- No recap like \"In summary...\" or \"To recap...\"\n- No standalone headers, bullets, or multi-line formatting unless inherent to the answer itself (e.g., a code block or diff is one logical unit).\n\n## Examples\n\n**\u2717 Wrong** (too verbose, has preamble + recap):\n```\nHere's the rundown on import ordering. Explanation continues at length. Explanation continues at length. \nExplanation continues at length. In summary, the key points are...\n```\n\n**\u2713 Right** (one sentence, direct):\n```\nSort imports by standard library, then third-party, then local\u2014use isort or similar tool.\n```\n\n**\u2713 Right** (two sentences, direct):\n```\nUse absolute imports for clarity and to avoid circular dependencies. Configure tools like isort to enforce this automatically.\n```\n\n## Anti-pattern\nExpanding a short answer into multi-sentence paragraphs because \"there's more context to give.\" Resist that urge. If the user wants detail, they will ask for it.\n\n## When the answer is inherently complex\nEven complex answers compress to two sentences by:\n- Combining multiple facts into one dense sentence.\n- Using a single subordinate clause to add context.\n- Deferring detail questions to follow-ups the user actually requests.\n\nExample (two sentences covering a complex trade-off):\n```\nProtocol A is faster but requires manual setup; Protocol B is slower but fully automated. Choose A for latency-critical work, B for hands-off deployment.\n```\n\n## Exception\nWhen a tool output, code block, file content, or structured data is the primary deliverable, that IS the answer (counts as one logical unit). Wrap it in 1\u20132 sentences of framing if needed, but the data itself doesn't count against the sentence limit.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "User wants terse, 1-2 sentence answers \u2014 durable style preference.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 0,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1244,
          "cache_read_tokens": 249938,
          "cache_write_tokens": 206953
        },
        "cost": 4.34869575,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Workflow rule: ALWAYS run the full test suite BEFORE committing, never after. User has corrected this more than once \u2014 treat test-then-commit as the standard sequence.",
          "skill_files": [
            "version-control/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to stage, verify, and commit code changes for this user. Covers the mandatory verification-before-commit sequence and standard commit hygiene.\n---\n\n# Git Commit Workflow\n\nHow to land code changes safely. The cardinal rule below is a hard user preference \u2014 violating it has drawn explicit, repeated corrections.\n\n## Cardinal rule: test BEFORE commit, never after\n\nALWAYS run the FULL test suite BEFORE you commit. Never commit first and test afterward.\nThe user has corrected this more than once and considers it the standard sequence.\n\nIf tests fail, fix the issue (or stop and report) \u2014 do NOT commit broken/unverified code\nand clean it up later. The commit should only happen once the suite is green.\n\n## Standard sequence\n\n1. Review the diff \u2014 know exactly what you're about to commit.\n2. Run the FULL test suite. Wait for it to finish; read the result.\n3. Only if tests pass: stage the intended files and commit.\n4. Write a clear, scoped commit message describing the change.\n\n## Pitfalls\n\n- Do NOT commit then run tests \"to confirm\" \u2014 that's the exact inversion the user rejects.\n- Do NOT skip the suite because a change \"looks trivial\" (typo, log line, rename). Run it anyway.\n- Do NOT commit with a partial/targeted test run when a full suite is expected, unless the\n  user explicitly scoped it down.\n\n## Verification\n\nAfter committing, confirm the working tree is clean and the commit landed (e.g. `git log -1`,\n`git status`) so there's no ambiguity about what was recorded.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 19,
          "output_tokens": 1933,
          "cache_read_tokens": 174086,
          "cache_write_tokens": 63172
        },
        "cost": 0.1060576,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User's workflow rule: ALWAYS run the full test suite BEFORE committing code, never after. Tests \u2192 commit is the required sequence. This has been reinforced as a standing expectation.",
          "skill_files": [
            "development/code-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: code-commit-workflow\ntitle: Code Commit Workflow\ndescription: Disciplined workflow for committing code changes with mandatory testing before commit\ntrigger: Committing code changes to a repository\naliases:\n  - commit workflow\n  - pre-commit sequence\n---\n\n## Standard Sequence\n\nWhen making code changes, follow this discipline strictly:\n\n1. **Make code changes** (edits, refactors, fixes)\n2. **Run the full test suite** (before considering any commit)\n3. **Fix any failures** (iterate until tests pass)\n4. **Commit** (only when all tests pass)\n5. **Push** (if applicable)\n\n**Critical rule: Tests ALWAYS come before commit, never after.**\n\nDo not commit speculatively and verify after. Do not commit and defer test-running. The test suite is the gate \u2014 if tests don't pass, code doesn't commit.\n\n## Why This Matters\n\n- Prevents broken code from entering version control\n- Catches integration issues before they propagate\n- Keeps the repository in a stable, testable state at every commit\n- Avoids blocking other team members with broken changes\n\n## Pitfalls\n\n- Running tests AFTER committing (wrong order \u2014 roll back and retry)\n- Partial test runs (run the FULL suite, not a subset you think is relevant)\n- Assuming \"it should work\" without verification\n- Committing with the intention to \"fix in the next commit\"\n\n## Verification\n\nBefore running `git commit` (or equivalent):\n- Full test suite has been run locally\n- All tests pass\n- No uncommitted changes remain beyond what you intend to commit\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 1,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1100,
          "cache_read_tokens": 249819,
          "cache_write_tokens": 206858
        },
        "cost": 4.335935999999999,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Workflow rule (stated firmly, recurring correction): ALWAYS run the full test suite BEFORE committing, never after. Tests-first, then commit, is the required standard sequence.",
          "skill_files": [
            "devops/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to stage, verify, and commit code changes safely. Covers the required test-before-commit sequence and pre-commit verification discipline.\n---\n\n# Git Commit Workflow\n\nStandard sequence for committing code changes in this user's projects.\n\n## Required sequence (NON-NEGOTIABLE)\n\nALWAYS run the full test suite BEFORE you commit \u2014 never after.\n\n1. Make / finish the code changes.\n2. Run the full test suite. Wait for it to pass.\n3. ONLY if tests pass: stage and commit.\n\nThe user has corrected this more than once. Committing first and testing\nafterward is wrong even if the tests would have passed. Do not reorder these\nsteps for convenience, speed, or because a change \"looks trivial.\"\n\n## Steps\n\n1. Review what changed (`git status`, `git diff`) so you know the scope.\n2. Run the project's full test suite (e.g. `pytest`, `npm test`, `make test`).\n3. If anything fails, fix it and re-run \u2014 do not proceed to commit.\n4. Stage the relevant files and commit with a clear message.\n5. Report the actual test result (pass/fail counts) alongside the commit.\n\n## Pitfalls\n\n- Do NOT commit then run tests. Tests gate the commit, not the other way around.\n- \"Trivial\" changes (typo, rename, log line) still go through the same gate.\n- A green local run is the precondition for committing; state it explicitly.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 18,
          "output_tokens": 1671,
          "cache_read_tokens": 134120,
          "cache_write_tokens": 62842
        },
        "cost": 0.1003375,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User workflow constraint: ALWAYS run the full test suite BEFORE committing code, never after. This is a standing rule, not optional. User has corrected this multiple times; embed it in code review/commit workflows.",
          "skill_files": [
            "dev/code_commit_workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: code_commit_workflow\ntitle: Code Commit Workflow\ndescription: Disciplined sequence for code changes, testing, and commits. Enforces test-first-always rule.\ntags: [git, testing, code-review, ci]\n---\n\n# Code Commit Workflow\n\n## Core Rule\n**ALWAYS run the full test suite BEFORE committing. Never commit first and test after.**\nThis is non-negotiable and must be the standard sequence every session.\n\n## Standard Sequence\n\n1. **Write/modify code** \u2014 Make the changes in your working directory.\n\n2. **Run full test suite** \u2014 Execute the complete test suite for the project before any commit.\n   - Use the project's standard test command (e.g., `pytest`, `npm test`, `cargo test`)\n   - Include integration, unit, linting, and any custom checks\n   - Do not skip or filter tests to \"speed up\" the cycle\n   - Verify all tests pass locally before proceeding\n\n3. **Commit the code** \u2014 Only after tests pass, stage and commit changes.\n   - Use clear, descriptive commit messages\n   - Include relevant issue/PR references if applicable\n\n4. **Push/open PR** \u2014 Submit to version control and CI if required\n\n## Pitfalls\n\n- **Committing without testing first** \u2014 This is the critical pitfall. The user has had to correct this behavior; do not repeat it. If you find yourself about to commit, STOP and run tests first.\n- **Assuming \"it looks right\"** \u2014 Always verify with actual test execution, not inspection.\n- **Skipping tests to save time** \u2014 This is false economy; failed tests caught later cost more time.\n\n## Verification\n\nBefore marking the commit as complete, confirm:\n- [ ] Full test suite ran and passed\n- [ ] No test output was skipped or filtered\n- [ ] Commit message is clear and references the change\n- [ ] All changes are actually committed (check git status)\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_mid_workflow_rule",
      "rep": 2,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1050,
          "cache_read_tokens": 249820,
          "cache_write_tokens": 206765
        },
        "cost": 4.33044375,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Workflow rule: ALWAYS run the full test suite BEFORE committing, never after. User has corrected this repeatedly \u2014 make tests-then-commit the standard sequence.",
          "skill_files": [
            "devops/git-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: git-commit-workflow\ndescription: How to sequence testing, verification, and committing changes for this user. Covers the test-before-commit rule and the standard pre-commit sequence.\n---\n\n# Git Commit Workflow\n\nHow to safely land code changes. This user is strict about the order of operations.\n\n## Hard rule: tests BEFORE commit\n\nALWAYS run the full test suite BEFORE you commit \u2014 never after. The user has\ncorrected this repeatedly and considers it a standing requirement, not a\nper-task preference. Committing first and testing afterward is a violation even\nif the tests ultimately pass.\n\n## Standard sequence\n\n1. Make the code change.\n2. Run any relevant linters/formatters.\n3. Run the FULL test suite (not just the tests near your change) and confirm it\n   is green.\n4. Only after tests pass, stage and commit.\n5. Push (if requested).\n\n## Pitfalls\n\n- Do not commit \"to save progress\" and run tests afterward. If you need a\n  checkpoint, say so explicitly; the default flow is test-then-commit.\n- \"Full\" means the whole suite the project runs in CI, not a subset, unless the\n  user explicitly scopes it down.\n- If tests fail, fix and re-run before committing \u2014 do not commit a red tree.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 16,
          "output_tokens": 1208,
          "cache_read_tokens": 55623,
          "cache_write_tokens": 62405
        },
        "cost": 0.08962455,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User has a firm rule: ALWAYS run the full test suite BEFORE committing, never after. This is a standing workflow requirement that has been corrected explicitly. Tests-first, then commit.",
          "skill_files": [
            "development/test-before-commit-workflow/SKILL.md"
          ],
          "skill_text": "\n---\nname: test-before-commit-workflow\ntitle: Test-Before-Commit Workflow\ndescription: Standard sequence for code changes \u2014 run full test suite before any commit, never after.\n---\n\n# Test-Before-Commit Workflow\n\n## Trigger\nAny time you are about to commit code changes, whether it's a fix, feature, refactor, doc update, or dependency bump.\n\n## Standard Sequence\n1. **Run the full test suite FIRST** \u2014 before any commit action\n2. **Verify all tests pass** \u2014 check exit code and output\n3. **Only then commit** \u2014 once tests are confirmed clean\n\n## Critical Rule\nDO NOT commit and then run tests. This breaks the invariant: always run the full suite BEFORE committing, never after.\n\n## Why\n- Tests run against the code state at the time of commit, not after\n- Committing broken code to history is irreversible\n- Testing after commit delays detection and creates revision debt\n- This is a non-negotiable workflow discipline\n\n## Pitfalls\n- Skipping the test run because \"it looks fine\"\n- Running only a subset of tests (e.g. unit tests) when the full suite is required\n- Testing in a different branch or environment, then committing in the main one\n- Committing with the assumption tests will pass later\n\n## Verification\nAfter running tests, confirm:\n- Exit code is 0 (or expected pass code)\n- No failures, errors, or skipped tests that shouldn't be skipped\n- Coverage or linting passes if part of the full suite\n\nOnly proceed to commit once tests confirm the state is good.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Always run the full test suite BEFORE committing, never after.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 0,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 403,
          "cache_read_tokens": 21782,
          "cache_write_tokens": 228418
        },
        "cost": 4.3457955,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "User's self-hosting target is a 3-node Proxmox cluster with Ceph storage at home. When asked about deploying anything, assume this is the target and tailor to Proxmox LXC/VMs.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 370,
          "cache_read_tokens": 16254,
          "cache_write_tokens": 62548
        },
        "cost": 0.0816684,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Self-hosting infrastructure: 3-node Proxmox cluster with Ceph storage at home. When deployment tasks are requested, assume Proxmox LXC/VMs as the target and tailor solutions accordingly.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 1,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 551,
          "cache_read_tokens": 21788,
          "cache_write_tokens": 228518
        },
        "cost": 4.358779500000001,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "User self-hosts on a 3-node Proxmox cluster with Ceph storage at home. For any deployment request, default to targeting this setup (Proxmox LXC/VMs) and tailor accordingly unless told otherwise.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 530,
          "cache_read_tokens": 16260,
          "cache_write_tokens": 62672
        },
        "cost": 0.082624,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User runs a 3-node Proxmox cluster with Ceph storage at home for self-hosting. When they ask about deploying or configuring something, tailor advice to Proxmox LXC/VM targets and Ceph storage assumptions unless they specify otherwise.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "buried_early_persona_fact",
      "rep": 2,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 602,
          "cache_read_tokens": 21785,
          "cache_write_tokens": 228522
        },
        "cost": 4.362675,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Self-hosts on a 3-node Proxmox cluster with Ceph storage at home. When asking about deploying anything, assume that's the target \u2014 tailor to Proxmox LXC/VMs.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 352,
          "cache_read_tokens": 16257,
          "cache_write_tokens": 62475
        },
        "cost": 0.08148745,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User runs a 3-node Proxmox cluster with Ceph storage at home. When discussing deployments, assume Proxmox LXC/VMs as the target and tailor recommendations accordingly \u2014 don't default to cloud/managed services.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 0,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1695,
          "cache_read_tokens": 113298,
          "cache_write_tokens": 162439
        },
        "cost": 3.3428932500000004,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/subprocess-troubleshooting/SKILL.md"
          ],
          "skill_text": "\n---\nname: subprocess-troubleshooting\ndescription: Diagnose and fix failures when spawning child processes from Python (Popen/subprocess) \u2014 signal deaths (SIGPIPE/SIGTERM), inherited file descriptors, hangs, and zombie/cleanup issues.\n---\n\n# Subprocess troubleshooting\n\nDebugging child processes spawned via `subprocess.Popen` / `subprocess.run` and friends. Covers signal-on-startup deaths, inherited fd problems, hangs, and cleanup.\n\n## Diagnostic order\n\n1. Read the actual death signal/exit code before guessing. `proc.returncode` < 0 means killed by signal `-returncode` (e.g. -13 = SIGPIPE, -15 = SIGTERM, -9 = SIGKILL). Don't theorize about permissions/config until you know HOW it died.\n2. Reproduce the spawn in isolation with explicit stdin/stdout/stderr wiring before touching the child's own code \u2014 most \"child crashes on startup\" bugs are in the Popen call, not the child.\n3. Only then look at the child binary, its config, socket perms, etc.\n\n## Common causes & fixes\n\n### Child dies with SIGPIPE (exit -13) on startup\nThe child inherited the parent's stdin/stdout, and that stream is closed or a reader went away, so the first write raises SIGPIPE. Classic for daemons/bridges that don't need stdin.\n\nFix: explicitly detach the inherited streams on the Popen call.\n```python\nsubprocess.Popen(cmd, stdin=subprocess.DEVNULL)\n# or also redirect out/err if those are the offenders:\nsubprocess.Popen(cmd, stdin=subprocess.DEVNULL,\n                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n```\nDo NOT chase socket permissions / chmod first for a SIGPIPE-on-startup \u2014 the signal name already tells you it's a broken pipe on an inherited fd, not an access-denied error.\n\n### Child hangs forever\nUsually a full pipe buffer (parent never reads stdout/stderr) \u2014 use `proc.communicate()` or drain the pipes, or redirect to DEVNULL/files if you don't need the output.\n\n### Zombie / not reaping\nAlways `wait()`/`communicate()` or use a context manager so the child is reaped.\n\n## Pitfalls\n- A wrong-but-plausible first guess (permissions, missing binary, config) wastes a round-trip. Map the signal number to its meaning FIRST; it usually points straight at the cause.\n- `stdin=subprocess.DEVNULL` is the right detach; passing `stdin=None` (the default) is what INHERITS the parent fd and causes the problem.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 654,
          "cache_read_tokens": 16257,
          "cache_write_tokens": 59714
        },
        "cost": 0.0795462,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "User prefers concise, direct responses to rapid code review questions. Works on subprocess/bridge infrastructure in Python. Environment: /tmp/bgreview_bench, Python 3.11.14, uv tooling, PEP 668 constraints.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 1,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1718,
          "cache_read_tokens": 114383,
          "cache_write_tokens": 164613
        },
        "cost": 3.38700825,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/debugging-subprocess-startup-crashes/SKILL.md"
          ],
          "skill_text": "\n---\nname: debugging-subprocess-startup-crashes\ndescription: Diagnose and fix child processes that die immediately on startup (SIGPIPE, SIGHUP, broken pipe, premature exit) when spawned via subprocess.Popen or similar. Covers inherited file-descriptor problems, signal disposition, and stdio handling.\n---\n\n# Debugging subprocess startup crashes\n\nWhen a spawned child process (bridge, daemon, worker, server) dies right at\nstartup with a signal like SIGPIPE/SIGHUP or an unexplained broken-pipe / early\nexit, the cause is usually inherited stdio or signal state from the parent \u2014 NOT\npermissions, NOT the child's own logic.\n\n## Diagnostic order (cheapest-to-rule-out first)\n\n1. **Inherited stdin/stdout/stderr.** A child that inherits the parent's stdin\n   gets SIGPIPE the moment the parent closes that fd (or when the child writes to\n   a closed inherited pipe). This is the most common cause and looks like a crash\n   \"on startup.\"\n   - Fix: pass explicit stdio on the Popen call. For a process that should never\n     read stdin: `stdin=subprocess.DEVNULL`. Redirect output deliberately:\n     `stdout=..., stderr=...` rather than inheriting.\n2. **Signal disposition inherited from parent.** If the parent ignored/blocked a\n   signal, the child inherits that mask. Reset with a `preexec_fn` or\n   `start_new_session=True` where appropriate.\n3. **Working directory / env**, then only last:\n4. **Permissions** on the socket/binary/file. Check this LAST \u2014 it is the\n   intuitive first guess but rarely the actual cause of a startup signal-kill.\n\n## Pitfalls\n\n- Do not lead with `chmod` on the socket. Socket/file permissions almost never\n  produce a SIGPIPE-on-startup signature; they produce EACCES/permission-denied\n  errors, which look different. Ruling out perms first wastes a round-trip.\n- SIGPIPE specifically points at a write-to-closed-pipe or closed-inherited-stdin\n  situation \u2014 follow the fd trail, not the filesystem.\n\n## Verification\n\nAfter applying `stdin=subprocess.DEVNULL` (or the relevant stdio fix), restart\nthe child and confirm it survives past the point it previously died. Confirmed\nfix for the raft-bridge case: bridge inherited the parent's stdin and took\nSIGPIPE when it closed; `stdin=subprocess.DEVNULL` on the Popen call resolved it.\n\nSee references/ for session-specific cases.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 444,
          "cache_read_tokens": 16257,
          "cache_write_tokens": 59443
        },
        "cost": 0.07815745,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "Subprocess SIGPIPE pitfall: child processes inherit parent stdin by default. When parent stdin closes, child receives SIGPIPE. Fix: always set stdin=subprocess.DEVNULL on Popen() calls in bridge/daemon code to prevent unexpected termination.",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "mid_technique_with_retracted_distractor",
      "rep": 2,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 6,
          "output_tokens": 1778,
          "cache_read_tokens": 115338,
          "cache_write_tokens": 166720
        },
        "cost": 3.4324470000000002,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [
            "debugging/subprocess-startup-debugging/SKILL.md"
          ],
          "skill_text": "\n---\nname: subprocess-startup-debugging\ndescription: Diagnose child processes (bridges, daemons, workers spawned via Popen/exec) that crash or misbehave immediately on startup \u2014 SIGPIPE, premature exit, hangs, broken stdio. Use when a spawned process dies before doing useful work.\n---\n\n# Subprocess startup debugging\n\nWhen a child process spawned by a parent (Popen, exec, subprocess, fork+exec)\ndies or misbehaves right at startup, work the signal/exit mechanism BEFORE\nguessing at filesystem-level causes like permissions. The crash signal usually\nnames the subsystem.\n\n## Reason from the signal, not from a hunch\n\nDo not open with \"probably a permissions issue / chmod the socket\" unless the\nerror explicitly points at access. Map the symptom to its actual cause class\nfirst:\n\n- **SIGPIPE on startup** \u2192 the child writes to (or is wired to) a pipe/stdio\n  stream whose other end is closed. Classic case: the child **inherits the\n  parent's stdin/stdout** and the parent closes it, so the first write earns a\n  SIGPIPE. FIX: pass `stdin=subprocess.DEVNULL` (and/or redirect stdout/stderr\n  to DEVNULL or a real file) on the Popen call so the child never inherits a\n  doomed stream.\n- **SIGTERM/SIGKILL immediately** \u2192 parent or supervisor is killing it\n  (timeout, OOM, healthcheck failure). Check the parent's lifecycle logic.\n- **Exit 127 / \"command not found\"** \u2192 PATH or binary missing in the child's\n  environment (env not inherited). This is environment state, not a code bug.\n- **Silent hang at startup** \u2192 child is blocking on a read from an inherited\n  stdin that never gets data, or waiting on a lock/socket. Redirect stdin from\n  DEVNULL or feed/close it.\n\n## Order of investigation\n\n1. Read the exact signal/exit code \u2014 it names the subsystem (pipe, term, exec).\n2. Inspect how the child is spawned: which fds does it inherit (stdin, stdout,\n   stderr)? Inherited stdio is the #1 cause of startup SIGPIPE/hangs.\n3. Only after stdio is ruled out, consider perms, sockets, env, cwd.\n\n## Pitfalls\n\n- Jumping to \"permissions / chmod\" first wastes a round trip when the signal\n  (e.g. SIGPIPE) already tells you it's a stdio-inheritance problem. Let the\n  signal drive the hypothesis.\n- `subprocess.DEVNULL` for stdin is the standard guard for daemons/bridges that\n  should never read from the controlling terminal.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": true
          }
        ],
        "distractor_saved": true,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 3,
          "output_tokens": 386,
          "cache_read_tokens": 0,
          "cache_write_tokens": 37662
        },
        "cost": 0.0490105,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
            "captured": false
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 0,
      "n_signals": 2,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1433,
          "cache_read_tokens": 213860,
          "cache_write_tokens": 171154
        },
        "cost": 3.6375225,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep platform-team scope in mind.\n\u00a7\nRepo uses ruff for all linting/formatting \u2014 never reach for flake8.",
          "skill_files": [
            "python/python-linting-formatting/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting-formatting\ndescription: Linting, formatting, and import-ordering for Python in this codebase. Use whenever fixing lint warnings, running formatters, ordering imports, or addressing code-quality CI failures.\n---\n\n# Python Linting & Formatting\n\nHow to handle lint/format/import-ordering work in this repo.\n\n## Tooling: use ruff, never flake8\n\nThis repo standardizes on **ruff for everything** \u2014 linting, formatting,\nand import sorting. Do NOT reach for flake8, isort, black, or pyflakes as\nseparate tools; ruff subsumes them here.\n\n- Lint:   `ruff check .`\n- Fix:    `ruff check --fix .`\n- Format: `ruff format .`\n- Imports: ruff handles import ordering (ruff's isort rules, e.g. `I`); do\n  not invoke isort separately.\n\nIf you catch yourself about to suggest or run flake8/isort/black, stop and\nuse the ruff equivalent instead.\n\n## Workflow for a lint warning\n\n1. Reproduce: `ruff check <path>` to see the exact rule code (e.g. `F401`, `I001`).\n2. Prefer `ruff check --fix` for autofixable rules before hand-editing.\n3. Re-run `ruff check` to confirm clean, then `ruff format` if touching layout.\n\n## Pitfalls\n\n- Don't propose flake8 config changes or `# noqa` flake8-style suppressions;\n  use ruff's `# noqa: <RULE>` with the specific ruff rule code.\n- Verify the fix with a real `ruff check` run \u2014 don't claim clean without it.\n\n## References\n\n- See `references/` for any repo-specific rule selections or per-file ignores\n  discovered in future sessions.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 10,
          "output_tokens": 372,
          "cache_read_tokens": 16254,
          "cache_write_tokens": 59659
        },
        "cost": 0.07806914999999999,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "Repo uses ruff for all linting/formatting \u2014 never suggest or default to flake8. Always use ruff.",
          "user": "Dana leads the platform team. Scope process suggestions to platform-level concerns and impacts.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 1,
      "n_signals": 2,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1113,
          "cache_read_tokens": 213966,
          "cache_write_tokens": 171006
        },
        "cost": 3.6109065,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "This repo uses ruff for all linting/formatting \u2014 never flake8. Use ruff for lint checks and fixes.",
          "user": "User is Dana, leads the platform team. When suggesting process changes, scope them to the platform team's purview.",
          "skill_files": [
            "python/python-linting/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting\ndescription: How to run and fix Python lint/format issues in this repo. Use whenever a task involves lint warnings, style cleanup, import ordering, or formatting checks on Python code.\n---\n\n# Python Linting\n\nThis repo standardizes on **ruff** for ALL linting and formatting. Do not reach for flake8, isort, black, pylint, or pyflakes \u2014 ruff covers lint + format + import sorting.\n\n## Rules\n- ALWAYS use `ruff` for lint checks and fixes. Never invoke flake8 (the user has explicitly corrected this).\n- `ruff check .` to lint; `ruff check --fix .` to auto-fix; `ruff format .` to format.\n- Import ordering is handled by ruff (its isort rules), not a separate isort run.\n\n## Pitfalls\n- Suggesting flake8 in this repo is a known correction \u2014 the user does not use it. Default to ruff for any lint-related request.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 23,
          "output_tokens": 2052,
          "cache_read_tokens": 132629,
          "cache_write_tokens": 61613
        },
        "cost": 0.10056215000000002,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Dana leads the platform team. When suggesting process changes, keep platform-team scope in mind. User prefers concise, direct responses.",
          "skill_files": [
            "development/code-review-and-linting/SKILL.md"
          ],
          "skill_text": "\n---\nname: code-review-and-linting\ntitle: Code Review and Linting Standards\ndescription: Standards and tooling for code review, linting, and style checks. Embeds repo-specific tool preferences and scope constraints.\ntrigger: |\n  When reviewing code, suggesting linting fixes, or analyzing Python style issues.\n  Consult this before defaulting to any linting tool.\n---\n\n# Code Review and Linting Standards\n\n## Tooling Defaults\n\n**Always use ruff** for linting, formatting, and style checks \u2014 never flake8, black separately, or other Python linters. Ruff is the unified standard.\n\nWhen the user says a repo \"uses ruff for everything,\" assume:\n- Ruff handles linting (like flake8)\n- Ruff handles formatting (like black)\n- No tool substitution without explicit user direction\n\n## Workflow\n\n1. **Check repo tooling first.** Look for `ruff.toml`, `pyproject.toml [tool.ruff]`, or `.ruff.toml` to confirm the standard.\n2. **Suggest ruff commands**, not tool equivalents:\n   - Style check: `ruff check --fix <path>`\n   - Format: `ruff format <path>`\n   - Combined: both in sequence or via ruff's own config\n3. **Avoid tool suggestions** unless explicitly asked. If a user corrects you on tooling, embed the lesson immediately \u2014 don't repeat the mistake in follow-up code reviews.\n\n## Scope & Context\n\nWhen suggesting process changes (CI, testing, standards, etc.), keep **platform-team scope** in mind \u2014 Dana leads platform and processes should align with team authority.\n\n## Pitfalls\n\n- Assuming black/flake8 equivalence when ruff is the standard\n- Suggesting linting tool changes without explicit repo context\n- Forgetting that ruff combines multiple tools \u2014 no need for separate formatter or linter calls\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "multi_signal_session",
      "rep": 2,
      "n_signals": 2,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 1253,
          "cache_read_tokens": 213890,
          "cache_write_tokens": 170996
        },
        "cost": 3.621105,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Name is Dana; leads the platform team. When suggesting process changes, keep that scope in mind.\n\u00a7\nThis repo uses ruff for all linting/formatting \u2014 always use ruff, never flake8.",
          "skill_files": [
            "dev/python-code-quality/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-code-quality\ndescription: Linting, formatting, and code-quality conventions for this Python repo. Use when running linters, fixing lint warnings, formatting code, or organizing imports.\n---\n\n# Python Code Quality\n\nConventions and tooling for keeping this repo's Python clean.\n\n## Linting & formatting tool: ruff (NOT flake8)\n\nThis repo uses **ruff for everything** \u2014 linting AND formatting. The user (Dana,\nplatform lead) explicitly corrected reaching for flake8. Always reach for ruff.\n\n- Lint:        `ruff check .`\n- Auto-fix:    `ruff check --fix .`\n- Format:      `ruff format .`\n- Import sort: handled by ruff (the `I` rule set / isort integration) \u2014 do NOT\n  invoke a separate isort or flake8 plugin.\n\nDo NOT suggest, install, or run flake8, autopep8, or standalone isort. If a lint\nwarning or import-ordering question comes up, resolve it through ruff.\n\n## Pitfalls\n- Don't propose flake8-specific config (setup.cfg `[flake8]`, `.flake8`) \u2014 config\n  lives in ruff's section (pyproject.toml `[tool.ruff]`).\n- When asked to \"fix the lint warning,\" prefer `ruff check --fix` over manual edits\n  where the rule is auto-fixable.\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 19,
          "output_tokens": 1715,
          "cache_read_tokens": 173875,
          "cache_write_tokens": 62064
        },
        "cost": 0.1035615,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Dana leads the platform team. When suggesting process changes, keep platform team scope in mind.",
          "skill_files": [
            "coding/python-linting-ruff/SKILL.md"
          ],
          "skill_text": "\n---\nname: python-linting-ruff\ntitle: Python Linting with Ruff\ndescription: Use ruff for all Python linting, formatting, and code quality checks. Never reach for flake8.\nsummary: Use ruff for all Python linting, formatting, and code quality checks in this codebase. Never default to flake8.\ntags: [python, lint, ruff, code-quality]\n---\n\n# Python Linting with Ruff\n\n## Summary\nThis codebase uses **ruff exclusively** for all Python linting, formatting, and code quality work. Always reach for ruff, never flake8 or other linters.\n\n## Key Principle\nWhen reviewing code, suggesting fixes, or running lint checks:\n- Use `ruff check` for linting\n- Use `ruff format` for formatting\n- Use `ruff rule <rule_id>` to inspect specific rules\n- Do NOT use flake8, pylint, black separately, or other competing tools\n\n## Common Commands\n\n```bash\n# Check code for lint issues\nruff check <file_or_dir>\n\n# Format code automatically\nruff format <file_or_dir>\n\n# Check and show fixes without applying\nruff check --show-fixes <file_or_dir>\n\n# Apply fixes automatically\nruff check --fix <file_or_dir>\n\n# List available rules\nruff rule\n```\n\n## Configuration\nRuff config lives in `pyproject.toml` or `ruff.toml`. Check the repo's config before suggesting ruff invocations to respect project-specific rule sets and exclusions.\n\n## Pitfalls\n- **Reaching for flake8**: This repo does not use flake8. If you catch yourself suggesting flake8, switch to ruff immediately.\n- **Mixed tools**: Do not use ruff alongside black, isort, or other formatters \u2014 ruff handles all of it.\n- **Ignoring config**: Always check the project's ruff config before running checks; don't assume defaults.\n\n## References\n- [Ruff docs](https://docs.astral.sh/ruff/)\n- [Ruff rules reference](https://docs.astral.sh/ruff/rules/)\n"
        },
        "signal_capture": [
          {
            "kind": "skill",
            "desc": "Prefers ruff over flake8 for this repo.",
            "captured": true
          },
          {
            "kind": "memory",
            "desc": "User's name is Dana and they lead the platform team.",
            "captured": true
          }
        ],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 0,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 381,
          "cache_read_tokens": 22852,
          "cache_write_tokens": 139652
        },
        "cost": 2.681388,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard constraint: will NOT take a dependency on any cloud service. Every solution must run self-hosted and work fully offline \u2014 design all solutions that way.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 246,
          "cache_read_tokens": 17135,
          "cache_write_tokens": 58729
        },
        "cost": 0.07636275000000001,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing constraint: NO hard cloud service dependencies. All solutions must be self-hosted and work fully offline. Design every solution with this as the primary architecture goal.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 1,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 382,
          "cache_read_tokens": 22861,
          "cache_write_tokens": 139598
        },
        "cost": 2.6804639999999997,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard constraint: will NOT take a hard dependency on any cloud service. Every solution must run self-hosted and work fully offline. Design all solutions offline-first/self-hosted.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 310,
          "cache_read_tokens": 17135,
          "cache_write_tokens": 58729
        },
        "cost": 0.07668275000000001,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing constraint: No hard dependencies on cloud services. All solutions must be self-hosted and work fully offline. Design every solution this way from the start.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "durable_vs_trivia_memory",
      "rep": 2,
      "n_signals": 1,
      "expect_noop": false,
      "opus_full": {
        "usage": {
          "input_tokens": 4,
          "output_tokens": 407,
          "cache_read_tokens": 22852,
          "cache_write_tokens": 139676
        },
        "cost": 2.683788,
        "review_model": "claude-opus-4-8",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Hard requirement: NO cloud-service dependencies. Everything must run self-hosted and work fully offline. Design all solutions to be offline-first / self-hostable.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 8,
          "output_tokens": 283,
          "cache_read_tokens": 17135,
          "cache_write_tokens": 58725
        },
        "cost": 0.07654275,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": true,
        "saved": {
          "memory": "",
          "user": "Standing constraint: self-hosted, offline-first architecture. No hard cloud service dependencies. All solutions must run fully offline and self-contained.",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [
          {
            "kind": "memory",
            "desc": "DURABLE: user is allergic to a hard dependency on cloud services \u2014 everything must run self-hosted/offline. This is a standing constraint.",
            "captured": true
          }
        ],
        "distractor_saved": false,
        "error": null
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 0,
      "n_signals": 0,
      "expect_noop": true,
      "opus_full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 210,
          "cache_read_tokens": 0,
          "cache_write_tokens": 103427
        },
        "cost": 1.95503625,
        "review_model": "claude-opus-4-8",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 3,
          "output_tokens": 160,
          "cache_read_tokens": 0,
          "cache_write_tokens": 38815
        },
        "cost": 0.04932175,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 1,
      "n_signals": 0,
      "expect_noop": true,
      "opus_full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 171,
          "cache_read_tokens": 0,
          "cache_write_tokens": 103433
        },
        "cost": 1.95222375,
        "review_model": "claude-opus-4-8",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 3,
          "output_tokens": 249,
          "cache_read_tokens": 0,
          "cache_write_tokens": 38818
        },
        "cost": 0.049770499999999995,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      }
    },
    {
      "scenario": "long_noop_smooth",
      "rep": 2,
      "n_signals": 0,
      "expect_noop": true,
      "opus_full": {
        "usage": {
          "input_tokens": 2,
          "output_tokens": 125,
          "cache_read_tokens": 0,
          "cache_write_tokens": 103424
        },
        "cost": 1.9486050000000001,
        "review_model": "claude-opus-4-8",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      },
      "haiku_digest": {
        "usage": {
          "input_tokens": 3,
          "output_tokens": 224,
          "cache_read_tokens": 0,
          "cache_write_tokens": 38818
        },
        "cost": 0.049645499999999995,
        "review_model": "claude-haiku-4-5-20251001",
        "did_save": false,
        "saved": {
          "memory": "",
          "user": "",
          "skill_files": [],
          "skill_text": ""
        },
        "signal_capture": [],
        "distractor_saved": null,
        "error": null
      }
    }
  ],
  "summary": {
    "opus_full": {
      "cost_mean": 3.532246464285714,
      "skill_capture": "12/12",
      "memory_capture": "9/9",
      "distractor_false_saves": "3/6",
      "noop_correct": "3/3"
    },
    "haiku_digest": {
      "cost_mean": 0.08028139523809524,
      "skill_capture": "11/12",
      "memory_capture": "9/9",
      "distractor_false_saves": "0/6",
      "noop_correct": "3/3"
    }
  }
 }
diff --git a/run_real_chat.py b/run_real_chat.py
 """Run the real-chat capture test: Opus full-replay vs Haiku digest, on the
 reconstructed PR-building conversation. Per-signal capture (skill vs memory),
 plus dump saved artifacts for a blind judge."""
 import os, sys, json
 sys.path.insert(0, "/tmp/bgreview_bench")
 import real_chat_scenario as R
 import hard_harness as H

 HAIKU = "claude-haiku-4-5-20251001"
 REPS = int(os.environ.get("REAL_REPS", "2"))


 def main():
    key = os.environ["ANTHROPIC_API_KEY"]
    sc = {"id": "real_chat", "signals": R.SIGNALS, "messages": R.MESSAGES}
    runs = []
    for rep in range(REPS):
        for label, rm in (("opus_full", None), ("haiku_digest", HAIKU)):
            r = H.run_arm(sc, digest_on=(rm is not None), api_key=key, review_model=rm)
            cap_sk = sum(1 for s in r["signal_capture"] if s["kind"] == "skill" and s["captured"])
            tot_sk = sum(1 for s in r["signal_capture"] if s["kind"] == "skill")
            cap_mem = sum(1 for s in r["signal_capture"] if s["kind"] == "memory" and s["captured"])
            tot_mem = sum(1 for s in r["signal_capture"] if s["kind"] == "memory")
            r["rep"] = rep; r["label"] = label
            runs.append(r)
            print(f"[real_chat r{rep} {label:12}] cost=${r['cost']:.4f} "
                  f"skill={cap_sk}/{tot_sk} memory={cap_mem}/{tot_mem}", flush=True)
            # show what it actually saved (first 200 chars each)
            s = r["saved"]
            print(f"    USER.md: {repr((s.get('user','') or '')[:160])}")
            print(f"    MEMORY.md: {repr((s.get('memory','') or '')[:160])}")
            print(f"    skills: {s.get('skill_files')}", flush=True)
    json.dump({"runs": runs, "signals": R.SIGNALS},
              open("/tmp/bgreview_bench/real_chat_results.json", "w"), indent=2)
    print("\nWrote real_chat_results.json")


 if __name__ == "__main__":
    main()
diff --git a/scenarios.py b/scenarios.py
 """Ground-truth scenarios for the background-review A/B benchmark.

 Each scenario is a synthetic conversation transcript (OpenAI message format)
 plus an oracle describing what a CORRECT review pass should do:

  expect_save: True  → the transcript contains a genuine, durable
                       memory/skill signal; a good review SAVES something.
  expect_save: False → smooth one-off task, no correction, no new technique;
                       a good review says "Nothing to save." (no-op).

  signal: short human description of the ground-truth signal, fed to the
          LLM judge so it can grade whether the arm's actual saves captured it.

 The set is deliberately mixed (save-worthy vs no-op) so precision AND recall
 both matter — an arm that "saves everything" scores high recall but low
 precision, and vice-versa.
 """

 SCENARIOS = [
    {
        "id": "user_style_correction",
        "expect_save": True,
        "signal": "User explicitly told the agent to stop being verbose and "
                  "give terse answers — a durable style preference.",
        "messages": [
            {"role": "user", "content": "explain how python list comprehensions work"},
            {"role": "assistant", "content": (
                "List comprehensions provide a concise way to create lists. "
                "The general syntax is [expression for item in iterable if condition]. "
                "Let me walk through several detailed examples covering mapping, "
                "filtering, nested comprehensions, and the performance characteristics "
                "versus equivalent for-loops, including memory considerations...")},
            {"role": "user", "content": (
                "stop. this is way too verbose. i hate long explanations. just "
                "give me the answer in one or two sentences from now on. always.")},
            {"role": "assistant", "content": (
                "Got it. [x*2 for x in nums if x > 0] — transform + filter in one line.")},
        ],
    },
    {
        "id": "workflow_correction",
        "expect_save": True,
        "signal": "User corrected the agent's workflow: always run the test "
                  "suite before committing, not after. A durable process rule.",
        "messages": [
            {"role": "user", "content": "fix the off-by-one in the paginator and commit it"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t1", "type": "function",
                 "function": {"name": "patch", "arguments": '{"path":"pag.py"}'}}]},
            {"role": "tool", "tool_call_id": "t1", "content": "patched"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t2", "type": "function",
                 "function": {"name": "terminal", "arguments": '{"command":"git commit -am fix"}'}}]},
            {"role": "tool", "tool_call_id": "t2", "content": "committed abc123"},
            {"role": "user", "content": (
                "no — you ALWAYS run the test suite BEFORE you commit, never after. "
                "i've told you this before. make that your standard sequence.")},
            {"role": "assistant", "content": "Understood — tests first, then commit. Reverting and re-doing."},
        ],
    },
    {
        "id": "debug_technique",
        "expect_save": True,
        "signal": "A non-trivial debugging technique emerged: the SIGPIPE crash "
                  "was fixed by setting stdin=DEVNULL on the bridge subprocess.",
        "messages": [
            {"role": "user", "content": "the raft bridge keeps dying with SIGPIPE on startup, no idea why"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t1", "type": "function",
                 "function": {"name": "search_files", "arguments": '{"pattern":"subprocess.Popen"}'}}]},
            {"role": "tool", "tool_call_id": "t1", "content": "raft/bridge.py:44 subprocess.Popen(cmd)"},
            {"role": "assistant", "content": (
                "Found it. The bridge inherits the parent's stdin; when the parent's "
                "stdin closes, the child gets SIGPIPE. Fix is stdin=subprocess.DEVNULL "
                "on the Popen call. Applying.")},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t2", "type": "function",
                 "function": {"name": "patch", "arguments": '{"path":"raft/bridge.py"}'}}]},
            {"role": "tool", "tool_call_id": "t2", "content": "patched"},
            {"role": "user", "content": "yep that fixed it, no more crashes"},
        ],
    },
    {
        "id": "user_persona_fact",
        "expect_save": True,
        "signal": "User revealed a durable personal fact: they run a 3-node "
                  "Proxmox cluster at home and prefer infra answers tailored to it.",
        "messages": [
            {"role": "user", "content": (
                "for context, all my self-hosting runs on a 3-node proxmox cluster at "
                "home with ceph storage. whenever i ask about deploying something, "
                "assume that's my target and tailor it to proxmox LXC/VMs.")},
            {"role": "assistant", "content": (
                "Noted — I'll target your 3-node Proxmox + Ceph setup for deployment "
                "questions. For most services an LXC container is lighter than a full VM.")},
        ],
    },
    {
        "id": "smooth_qa_noop",
        "expect_save": False,
        "signal": "Plain factual Q&A, answered correctly, no correction, no new "
                  "technique, no personal detail. A correct review is a no-op.",
        "messages": [
            {"role": "user", "content": "what's the default port for postgres?"},
            {"role": "assistant", "content": "5432."},
            {"role": "user", "content": "and redis?"},
            {"role": "assistant", "content": "6379."},
        ],
    },
    {
        "id": "smooth_task_noop",
        "expect_save": False,
        "signal": "A one-off task (summarize a changelog) completed smoothly with "
                  "no correction or reusable technique. A correct review is a no-op.",
        "messages": [
            {"role": "user", "content": "summarize this changelog: v2.1 adds dark mode, fixes login bug, bumps deps"},
            {"role": "assistant", "content": (
                "v2.1 — new dark mode, a login bug fix, and dependency bumps.")},
            {"role": "user", "content": "thanks"},
            {"role": "assistant", "content": "Anytime."},
        ],
    },
    {
        "id": "transient_error_resolved_noop",
        "expect_save": False,
        "signal": "A transient network error resolved on retry. The lesson is NOT "
                  "'the tool is broken' — capturing a negative constraint would be "
                  "wrong. A correct review is a no-op (or at most the retry pattern).",
        "messages": [
            {"role": "user", "content": "fetch the latest release notes from the api"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t1", "type": "function",
                 "function": {"name": "web_extract", "arguments": '{"urls":["https://api.x/notes"]}'}}]},
            {"role": "tool", "tool_call_id": "t1", "content": "ERROR: connection timed out"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t2", "type": "function",
                 "function": {"name": "web_extract", "arguments": '{"urls":["https://api.x/notes"]}'}}]},
            {"role": "tool", "tool_call_id": "t2", "content": "Release 9.2: ..."},
            {"role": "assistant", "content": "Got it on retry — Release 9.2 ships X and Y."},
        ],
    },
    {
        "id": "tool_heavy_genuine_skill",
        "expect_save": True,
        "signal": "A reusable multi-step procedure for bisecting a flaky test "
                  "emerged across several tool calls — worth a skill.",
        "messages": [
            {"role": "user", "content": "this test flakes ~1 in 20 runs, help me find why"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t1", "type": "function",
                 "function": {"name": "terminal", "arguments": '{"command":"pytest --count=50 test_x.py"}'}}]},
            {"role": "tool", "tool_call_id": "t1", "content": "3/50 failed"},
            {"role": "assistant", "content": "", "tool_calls": [
                {"id": "t2", "type": "function",
                 "function": {"name": "terminal", "arguments": '{"command":"pytest -p no:randomly --count=50"}'}}]},
            {"role": "tool", "tool_call_id": "t2", "content": "0/50 failed"},
            {"role": "assistant", "content": (
                "The flake is test-ordering dependent — disabling pytest-randomly makes "
                "it deterministic, so a shared-state leak between tests is the cause. "
                "The reusable recipe: run with --count to confirm flake rate, then toggle "
                "-p no:randomly to isolate ordering dependence, then bisect the leaking fixture.")},
            {"role": "user", "content": "great, that's a useful approach"},
        ],
    },
 ]
diff --git a/summary.json b/summary.json
 {
  "cost": {
    "baseline": {
      "n": 24,
      "cost_mean": 0.61297746875,
      "cost_total": 14.711459249999999,
      "tok_mean": 62513.583333333336,
      "errors": 0
    },
    "treatment": {
      "n": 24,
      "cost_mean": 0.03314089791666667,
      "cost_total": 0.7953815500000001,
      "tok_mean": 53744.458333333336,
      "errors": 0
    }
  },
  "prf": {
    "baseline": {
      "tp": 15,
      "fp": 0,
      "fn": 0,
      "tn": 9,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "treatment": {
      "tp": 13,
      "fp": 0,
      "fn": 2,
      "tn": 9,
      "precision": 1.0,
      "recall": 0.8666666666666667,
      "f1": 0.9285714285714286
    }
  },
  "quality": {
    "baseline": {
      "n": 24,
      "quality_mean": 3.9565217391304346,
      "captured": 21,
      "false_positives": 0
    },
    "treatment": {
      "n": 24,
      "quality_mean": 3.772727272727273,
      "captured": 19,
      "false_positives": 0
    }
  }
 }
	"""Aggregate results.json + quality.json into the PR-body metrics tables."""
	import json
	import statistics as st
	from pathlib import Path

	R = json.loads(Path("/tmp/bgreview_bench/results.json").read_text())
	Q = json.loads(Path("/tmp/bgreview_bench/quality.json").read_text())


	def prf(graded):
	"""Precision/recall/F1 of the save decision against the fixture oracle."""
	tp = fp = fn = tn = 0
	for g in graded:
	want = g["expect_save"]
	did = bool(g["did_save"])
	if want and did:
	tp += 1
	elif want and not did:
	fn += 1
	elif (not want) and did:
	fp += 1
	else:
	tn += 1
	prec = tp / (tp + fp) if (tp + fp) else 1.0
	rec = tp / (tp + fn) if (tp + fn) else 1.0
	f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
	return dict(tp=tp, fp=fp, fn=fn, tn=tn, precision=prec, recall=rec, f1=f1)


	def cost_stats(runs):
	costs = [r["cost"] for r in runs if not r.get("error")]
	toks = [sum(r["usage"].values()) for r in runs if not r.get("error")]
	return dict(
	n=len(costs),
	cost_mean=st.mean(costs) if costs else 0,
	cost_total=sum(costs),
	tok_mean=st.mean(toks) if toks else 0,
	errors=sum(1 for r in runs if r.get("error")),
	)


	def quality_stats(graded):
	qs = [g["quality"] for g in graded if isinstance(g.get("quality"), (int, float))]
	cap = [g for g in graded if g.get("captures_signal") is True]
	fps = [g for g in graded if g.get("false_positive") is True]
	return dict(
	n=len(graded),
	quality_mean=st.mean(qs) if qs else 0,
	captured=len(cap),
	false_positives=len(fps),
	)


	bc, tc = cost_stats(R["baseline"]), cost_stats(R["treatment"])
	bp, tp_ = prf(Q["baseline"]), prf(Q["treatment"])
	bq, tq = quality_stats(Q["baseline"]), quality_stats(Q["treatment"])

	print("=" * 64)
	print("COST (per review fork, mean across all scenarios x reps)")
	print(f" baseline (opus, full): ${bc['cost_mean']:.5f} total ${bc['cost_total']:.4f} n={bc['n']} err={bc['errors']}")
	print(f" treatment (haiku,digest):${tc['cost_mean']:.5f} total ${tc['cost_total']:.4f} n={tc['n']} err={tc['errors']}")
	if tc['cost_mean']:
	print(f" → {bc['cost_mean']/tc['cost_mean']:.1f}x cheaper per review")
	print(f" tokens/fork: baseline {bc['tok_mean']:.0f} treatment {tc['tok_mean']:.0f}")
	print()
	print("QUALITY — save-decision accuracy vs fixture oracle")
	print(f" baseline : P={bp['precision']:.2f} R={bp['recall']:.2f} F1={bp['f1']:.2f} (tp{bp['tp']} fp{bp['fp']} fn{bp['fn']} tn{bp['tn']})")
	print(f" treatment: P={tp_['precision']:.2f} R={tp_['recall']:.2f} F1={tp_['f1']:.2f} (tp{tp_['tp']} fp{tp_['fp']} fn{tp_['fn']} tn{tp_['tn']})")
	print()
	print("QUALITY — LLM-judge edit grading")
	print(f" baseline : mean quality {bq['quality_mean']:.2f}/5 captured {bq['captured']}/{bq['n']} false-pos {bq['false_positives']}")
	print(f" treatment: mean quality {tq['quality_mean']:.2f}/5 captured {tq['captured']}/{tq['n']} false-pos {tq['false_positives']}")
	print("=" * 64)

	Path("/tmp/bgreview_bench/summary.json").write_text(json.dumps({
	"cost": {"baseline": bc, "treatment": tc},
	"prf": {"baseline": bp, "treatment": tp_},
	"quality": {"baseline": bq, "treatment": tq},
	}, indent=2))
	"""Production-cache cost correction (analytical, no new API calls).

	The live benchmarks ran each fork against a COLD cache (fresh parent per run),
	so BOTH arms paid cache-WRITE for the prompt. That hid the dominant production
	effect: in a real long-lived chat the parent history is ALREADY WARM, so:

	- OLD (full replay): the fork re-sends the identical history -> cache READ
	(cheap, ~0.1x of write). The cache was already paid for by prior turns.
	- DIGEST (same model): the fork sends digest+tail, a NOVEL prefix the cache
	has never seen -> cache WRITE for that prefix. Cutting history DESTROYS the
	warm-cache hit and re-writes fewer-but-uncached tokens at ~12.5x the read price.
	- ROUTING (Haiku): a different model can't use the opus parent cache at all,
	so the haiku fork is cold-prefix REGARDLESS of digest. Here cutting history
	is free (cache already lost by switching models), and the win is the ~15x
	cheaper per-token model.

	We reconstruct production cost from the measured token COUNTS (which are valid;
	only the cache categorization was wrong for the cold benchmark).
	"""
	import json
	import statistics as st

	P = {
	"opus": {"in": 15.0, "out": 75.0, "cache_read": 1.50, "cache_write": 18.75},
	"haiku": {"in": 1.0, "out": 5.0, "cache_read": 0.10, "cache_write": 1.25},
	}

	d = json.load(open("/tmp/bgreview_bench/hard_results.json"))


	def mean_tokens(arm):
	"""Total input tokens the model SEES (fresh+read+write are disjoint prompt
	slices) and output tokens, averaged per run."""
	ins, outs = [], []
	for row in d["runs"]:
	r = row[arm]
	if r.get("error"):
	continue
	u = r["usage"]
	ins.append(u["input_tokens"] + u["cache_read_tokens"] + u["cache_write_tokens"])
	outs.append(u["output_tokens"])
	return st.mean(ins), st.mean(outs)


	full_in, full_out = mean_tokens("full")
	dig_in, dig_out = mean_tokens("digest")

	# How much of the full history does the digest keep? (sets how much is novel)
	keep_frac = dig_in / full_in
	novel_frac = 1.0 # the digest text is a freshly-composed prefix -> all novel

	print(f"measured tokens/run: full sees {full_in:.0f} in, digest sees {dig_in:.0f} in "
	f"({keep_frac*100:.0f}% of full)\n")

	# ---- SAME MODEL (opus), production cache ----
	p = P["opus"]
	# OLD: entire history warm -> all input is cache_read; tiny incremental write for the new turn ignored.
	old = full_in/1e6p["cache_read"] + full_out/1e6p["out"]
	# DIGEST: novel prefix. The fork runs ~multiple iterations, so the digest prefix is
	# written once then read on subsequent iterations within the fork. Model two bounds:
	# worst: all digest input billed as write (single-iteration / no internal reuse)
	# typical: write once, then ~60% re-read across the fork's own iterations
	dig_worst = dig_in/1e6p["cache_write"] + dig_out/1e6p["out"]
	dig_typ = (dig_in0.4/1e6p["cache_write"] + dig_in0.6/1e6p["cache_read"]) + dig_out/1e6*p["out"]

	print("=== SAME MODEL (Opus), PRODUCTION warm-parent cache ===")
	print(f" OLD full replay (warm reads): ${old:.4f}")
	print(f" DIGEST (novel prefix, typical): ${dig_typ:.4f} -> {dig_typ/old:.2f}x vs old ({'CHEAPER' if dig_typ<old else 'MORE EXPENSIVE'})")
	print(f" DIGEST (novel prefix, worst): ${dig_worst:.4f} -> {dig_worst/old:.2f}x vs old")
	print()

	# ---- ROUTING (Haiku review), production cache ----
	po, ph = P["opus"], P["haiku"]
	# OLD routing baseline is still the opus fork on the WARM parent cache:
	old_route = full_in/1e6po["cache_read"] + full_out/1e6po["out"]
	# Haiku fork: cannot reuse opus cache (different model) -> cold prefix on haiku.
	# It sends the digest (fewer tokens) and writes-then-reads on haiku's own cache.
	hk_typ = (dig_in0.4/1e6ph["cache_write"] + dig_in0.6/1e6ph["cache_read"]) + dig_out/1e6*ph["out"]
	hk_worst = dig_in/1e6ph["cache_write"] + dig_out/1e6ph["out"]
	# For completeness: what if haiku replayed FULL history (no digest)? still cold on haiku.
	hk_full_typ = (full_in0.4/1e6ph["cache_write"] + full_in0.6/1e6ph["cache_read"]) + full_out/1e6*ph["out"]

	print("=== ROUTING (Opus parent, Haiku review), PRODUCTION cache ===")
	print(f" OLD baseline = opus fork on warm parent cache: ${old_route:.4f}")
	print(f" HAIKU + digest (typical): ${hk_typ:.4f} -> {old_route/hk_typ:.1f}x cheaper than old")
	print(f" HAIKU + digest (worst): ${hk_worst:.4f} -> {old_route/hk_worst:.1f}x cheaper than old")
	print(f" HAIKU + full replay (no digest, for ref): ${hk_full_typ:.4f} -> {old_route/hk_full_typ:.1f}x cheaper")
	print()
	print("Note: digest helps ROUTING (fewer cold tokens to write on haiku) but HURTS")
	print("same-model (destroys the warm opus cache). The two cases are opposite.")
	Large scenario: 182 msgs, ~159501 tokens of content

	💾 Self-improvement review: Skill 'response-brevity' created.
	[full ] rep0 cost=$0.15236 save=True in+cache=305662
	💾 Self-improvement review: Skill 'response-brevity' created.
	[digest] rep0 cost=$0.06417 save=True in+cache=147254
	💾 Self-improvement review: Skill 'response-conciseness' created.
	[full ] rep1 cost=$0.16579 save=True in+cache=408628
	💾 Self-improvement review: Skill 'response-brevity' created.
	[digest] rep1 cost=$0.05204 save=True in+cache=72759
	💾 Self-improvement review: Skill 'terse-communication' created.
	[full ] rep2 cost=$0.16185 save=True in+cache=406854
	💾 Self-improvement review: Skill 'response-brevity' created.
	[digest] rep2 cost=$0.05197 save=True in+cache=72751

	FULL mean $0.16000 DIGEST mean $0.05606 → 2.9x cheaper
	DIGEST captured signal (saved): 3/3
	"""Supplementary benchmark: isolate the DIGEST path (ideas ②③) on a LARGE session.

	The main harness scenarios are small, so the digest never engages — they prove
	idea ① (routing) only. This isolates ②③: take a save-worthy signal, bury it in
	a large filler transcript that exceeds the token budget, and compare on the
	SAME model:

	FULL — replay the entire snapshot (current behaviour; James's pathological case)
	DIGEST — replay recent tail verbatim + a summary of older turns (this PR)

	Same model both arms → any cost delta is purely the digest, and we check the
	DIGEST arm still captures the signal (it lives in the recent tail).

	Run after the main harness. Uses claude-haiku (cheap) for both arms since the
	point here is the token/cost delta from trimming, not model price.
	"""
	import os
	import sys
	import json
	import shutil
	import tempfile
	from pathlib import Path

	WORKTREE = "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8"
	sys.path.insert(0, WORKTREE)
	sys.path.insert(0, "/tmp/bgreview_bench")

	MODEL = "claude-haiku-4-5-20251001"
	PRICES = {"in": 1.00, "out": 5.00, "cache_read": 0.10, "cache_write": 1.25}
	REPS = int(os.environ.get("DIGEST_REPS", "3"))


	def _cost(u):
	return (u["input_tokens"]/1e6PRICES["in"] + u["output_tokens"]/1e6PRICES["out"]
	+ u["cache_read_tokens"]/1e6*PRICES["cache_read"]
	+ u["cache_write_tokens"]/1e6*PRICES["cache_write"])


	def _big_scenario():
	"""A genuine skill signal in the TAIL, buried under ~60 filler turns."""
	msgs = []
	topics = ["refactoring the cache layer", "the CI matrix", "a flaky import",
	"the config loader", "docs phrasing", "a rename", "test fixtures"]
	for i in range(90):
	t = topics[i % len(topics)]
	msgs.append({"role": "user", "content":
	f"Quick question about {t}, iteration {i}. " + ("Some detailed context here. " * 90)})
	msgs.append({"role": "assistant", "content":
	f"Here's the answer for {t} #{i}. " + ("Detailed explanation follows. " * 90)})
	# The save-worthy signal, in the recent tail:
	msgs += [
	{"role": "user", "content":
	"ok different thing — you keep writing these giant verbose answers. "
	"stop. from now on give me one or two sentences max, always. i mean it."},
	{"role": "assistant", "content": "Understood — terse from here on."},
	]
	return {
	"id": "large_session_style_correction",
	"expect_save": True,
	"signal": "User told the agent (in the recent tail, after a long session) "
	"to always be terse — one or two sentences. A durable style preference.",
	"messages": msgs,
	}


	def run_arm(scenario, digest_on, api_key):
	hermes_home = tempfile.mkdtemp(prefix="dig_")
	hh = os.path.join(hermes_home, ".hermes")
	os.makedirs(hh, exist_ok=True)
	os.environ["HERMES_HOME"] = hh
	for mod in [m for m in sys.modules if m.startswith(("hermes", "agent", "run_agent", "tools", "model_tools"))]:
	del sys.modules[mod]
	sys.path.insert(0, WORKTREE)

	# Same model both arms. Route to the aux model (forces the digest branch);
	# toggle the digest by setting max_context_tokens very high (FULL) vs the
	# default 48000 (DIGEST). Routing alone keeps model price identical across
	# arms, isolating the trimming effect.
	import yaml
	cfg = {
	"model": {"provider": "anthropic", "model": MODEL, "default": MODEL},
	"auxiliary": {"background_review": {
	"provider": "anthropic", "model": MODEL,
	"max_context_tokens": (10_000_000 if not digest_on else 48000),
	"digest_tail_messages": 24,
	}},
	"memory": {"memory_enabled": True, "user_profile_enabled": True},
	"skills": {"creation_nudge_interval": 10},
	}
	Path(hh, "config.yaml").write_text(yaml.safe_dump(cfg))
	os.environ["ANTHROPIC_API_KEY"] = api_key

	from run_agent import AIAgent
	from agent import background_review as br
	import anthropic

	tally = {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0}

	def _add(u):
	if u is None:
	return
	tally["input_tokens"] += int(getattr(u, "input_tokens", 0) or 0)
	tally["output_tokens"] += int(getattr(u, "output_tokens", 0) or 0)
	tally["cache_read_tokens"] += int(getattr(u, "cache_read_input_tokens", 0) or 0)
	tally["cache_write_tokens"] += int(getattr(u, "cache_creation_input_tokens", 0) or 0)

	Messages = anthropic.resources.messages.Messages
	o_stream, o_create = Messages.stream, Messages.create

	class _P:
	def __init__(self, m): self._m, self._c = m, False
	def __enter__(self):
	s = self._m.__enter__(); og = s.get_final_message
	def g(a, *k):
	msg = og(a, *k)
	if not self._c:
	self._c = True; _add(getattr(msg, "usage", None))
	return msg
	s.get_final_message = g; return s
	def __exit__(self, a): return self._m.__exit__(a)

	Messages.stream = lambda self, a, k: _P(o_stream(self, a, **k))
	def _wc(self, a, *k):
	r = o_create(self, a, *k); _add(getattr(r, "usage", None)); return r
	Messages.create = _wc

	agent = AIAgent(model=MODEL, provider="anthropic", api_key=api_key,
	api_mode="anthropic_messages", quiet_mode=True,
	skip_context_files=True, skip_memory=True, platform="cli",
	session_id="dig-parent")
	try:
	target, _ = br.spawn_background_review_thread(
	agent, messages_snapshot=list(scenario["messages"]),
	review_memory=True, review_skills=True)
	target()
	finally:
	Messages.stream, Messages.create = o_stream, o_create
	try:
	agent.close()
	except Exception:
	pass

	skills_dir = Path(hh) / "skills"
	saved = {
	"memory": (Path(hh, "MEMORY.md").read_text() if Path(hh, "MEMORY.md").exists() else ""),
	"user": (Path(hh, "USER.md").read_text() if Path(hh, "USER.md").exists() else ""),
	"skill_files": [str(p.relative_to(skills_dir)) for p in skills_dir.rglob("*")
	if p.is_file() and ".archive" not in str(p)
	and not str(p).endswith((".usage.json", ".lock"))] if skills_dir.exists() else [],
	}
	did_save = bool(saved["memory"].strip() or saved["user"].strip() or saved["skill_files"])
	shutil.rmtree(hermes_home, ignore_errors=True)
	return {"usage": dict(tally), "cost": _cost(tally), "did_save": did_save, "saved": saved}


	def main():
	api_key = os.environ["ANTHROPIC_API_KEY"]
	sc = _big_scenario()
	full_tok = sum(len(str(m.get("content", ""))) for m in sc["messages"]) // 3
	print(f"Large scenario: {len(sc['messages'])} msgs, ~{full_tok} tokens of content\n")
	out = {"full": [], "digest": [], "scenario": sc["id"], "signal": sc["signal"]}
	for rep in range(REPS):
	for arm, dig in (("full", False), ("digest", True)):
	r = run_arm(sc, dig, api_key)
	r["rep"] = rep
	out[arm].append(r)
	print(f"[{arm:6}] rep{rep} cost=${r['cost']:.5f} save={r['did_save']} "
	f"in+cache={r['usage']['input_tokens']+r['usage']['cache_read_tokens']+r['usage']['cache_write_tokens']}")
	Path("/tmp/bgreview_bench/digest_results.json").write_text(json.dumps(out, indent=2))
	import statistics as stt
	fc = stt.mean(x["cost"] for x in out["full"])
	dc = stt.mean(x["cost"] for x in out["digest"])
	print(f"\nFULL mean ${fc:.5f} DIGEST mean ${dc:.5f} → {fc/dc:.1f}x cheaper")
	print(f"DIGEST captured signal (saved): {sum(x['did_save'] for x in out['digest'])}/{REPS}")


	if __name__ == "__main__":
	main()
	{
	"full": [
	{
	"usage": {
	"input_tokens": 16,
	"output_tokens": 808,
	"cache_read_tokens": 203262,
	"cache_write_tokens": 102384
	},
	"cost": 0.1523622,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"response-brevity/SKILL.md"
	]
	},
	"rep": 0
	},
	{
	"usage": {
	"input_tokens": 20,
	"output_tokens": 1309,
	"cache_read_tokens": 305683,
	"cache_write_tokens": 102925
	},
	"cost": 0.16578955,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"communication/response-conciseness/SKILL.md"
	]
	},
	"rep": 1
	},
	{
	"usage": {
	"input_tokens": 16,
	"output_tokens": 711,
	"cache_read_tokens": 304584,
	"cache_write_tokens": 102254
	},
	"cost": 0.16184690000000002,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"terse-communication/SKILL.md"
	]
	},
	"rep": 2
	}
	],
	"digest": [
	{
	"usage": {
	"input_tokens": 20,
	"output_tokens": 1248,
	"cache_read_tokens": 109683,
	"cache_write_tokens": 37551
	},
	"cost": 0.06416705,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"communication/response-brevity/SKILL.md"
	]
	},
	"rep": 0
	},
	{
	"usage": {
	"input_tokens": 10,
	"output_tokens": 507,
	"cache_read_tokens": 36032,
	"cache_write_tokens": 36717
	},
	"cost": 0.05204445,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"response-brevity/SKILL.md"
	]
	},
	"rep": 1
	},
	{
	"usage": {
	"input_tokens": 10,
	"output_tokens": 495,
	"cache_read_tokens": 36035,
	"cache_write_tokens": 36706
	},
	"cost": 0.051971,
	"did_save": true,
	"saved": {
	"memory": "",
	"user": "",
	"skill_files": [
	"communication/response-brevity/SKILL.md"
	]
	},
	"rep": 2
	}
	],
	"scenario": "large_session_style_correction",
	"signal": "User told the agent (in the recent tail, after a long session) to always be terse \u2014 one or two sentences. A durable style preference."
	}
	💾 Self-improvement review: User profile updated · Skill 'response-style' created.
	[buried_early_style_correction r0 full ] cost=$4.3323 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'response-style-discipline' created.
	[buried_early_style_correction r0 digest] cost=$2.0846 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'response-style-control' created.
	[buried_early_style_correction r1 full ] cost=$4.5304 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'honoring-standing-user-constraints' created.
	[buried_early_style_correction r1 digest] cost=$1.9965 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'response-style' created.
	[buried_early_style_correction r2 full ] cost=$4.3227 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'response-style-adherence' created.
	[buried_early_style_correction r2 digest] cost=$1.9905 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
	[buried_mid_workflow_rule r0 full ] cost=$4.1601 cap=1/1 distractor=None err=None
	💾 Self-improvement review: Skill 'git-commit-workflow' created. · User profile updated
	[buried_mid_workflow_rule r0 digest] cost=$2.0312 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
	[buried_mid_workflow_rule r1 full ] cost=$4.1733 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'commit-workflow' created.
	[buried_mid_workflow_rule r1 digest] cost=$2.0943 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
	[buried_mid_workflow_rule r2 full ] cost=$4.5172 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'git-commit-workflow' created.
	[buried_mid_workflow_rule r2 digest] cost=$1.9198 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated
	[buried_early_persona_fact r0 full ] cost=$4.3584 cap=1/1 distractor=None err=None
	💾 Self-improvement review: Memory updated
	[buried_early_persona_fact r0 digest] cost=$1.7843 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated
	[buried_early_persona_fact r1 full ] cost=$4.5543 cap=1/1 distractor=None err=None
	💾 Self-improvement review: Memory updated
	[buried_early_persona_fact r1 digest] cost=$1.7803 cap=1/1 distractor=None err=None
	💾 Self-improvement review: User profile updated
	[buried_early_persona_fact r2 full ] cost=$4.3539 cap=1/1 distractor=None err=None
	💾 Self-improvement review: Memory updated
	[buried_early_persona_fact r2 digest] cost=$1.8815 cap=1/1 distractor=None err=None
	💾 Self-improvement review: Skill 'debugging-subprocess-startup' created.
	[mid_technique_with_retracted_distractor r0 full ] cost=$3.3596 cap=1/1 distractor=True err=None
	💾 Self-improvement review: Skill 'subprocess-startup-debugging' created.
	[mid_technique_with_retracted_distractor r0 digest] cost=$1.9333 cap=1/1 distractor=True err=None
	💾 Self-improvement review: Skill 'debugging-subprocess-startup' created.
	[mid_technique_with_retracted_distractor r1 full ] cost=$3.3837 cap=1/1 distractor=True err=None
	💾 Self-improvement review: Skill 'debugging-subprocess-lifecycle' created.
	[mid_technique_with_retracted_distractor r1 digest] cost=$1.9117 cap=1/1 distractor=True err=None
	💾 Self-improvement review: Skill 'subprocess-lifecycle-debugging' created.
	[mid_technique_with_retracted_distractor r2 full ] cost=$3.6094 cap=1/1 distractor=True err=None
	💾 Self-improvement review: Skill 'debugging-subprocess-spawn' created.
	[mid_technique_with_retracted_distractor r2 digest] cost=$1.9391 cap=1/1 distractor=True err=None
	💾 Self-improvement review: User profile updated
	[multi_signal_session r0 full ] cost=$3.4050 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated · Memory updated · Skill 'python-linting' created.
	[multi_signal_session r0 digest] cost=$2.0442 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'python-linting-formatting' created.
	[multi_signal_session r1 full ] cost=$3.6372 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated · Memory updated
	[multi_signal_session r1 digest] cost=$1.8918 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'python-lint-format' created.
	[multi_signal_session r2 full ] cost=$3.6079 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated · Skill 'python-linting' created.
	[multi_signal_session r2 digest] cost=$1.9342 cap=2/2 distractor=None err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r0 full ] cost=$2.6726 cap=1/1 distractor=False err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r0 digest] cost=$1.6533 cap=1/1 distractor=False err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r1 full ] cost=$2.6620 cap=1/1 distractor=False err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r1 digest] cost=$1.7590 cap=1/1 distractor=False err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r2 full ] cost=$2.6657 cap=1/1 distractor=False err=None
	💾 Self-improvement review: User profile updated
	[durable_vs_trivia_memory r2 digest] cost=$1.7559 cap=1/1 distractor=False err=None
	[long_noop_smooth r0 full ] cost=$1.9358 cap=0/0 distractor=None err=None
	[long_noop_smooth r0 digest] cost=$1.0411 cap=0/0 distractor=None err=None
	[long_noop_smooth r1 full ] cost=$1.9339 cap=0/0 distractor=None err=None
	[long_noop_smooth r1 digest] cost=$1.0446 cap=0/0 distractor=None err=None
	[long_noop_smooth r2 full ] cost=$1.9443 cap=0/0 distractor=None err=None
	[long_noop_smooth r2 digest] cost=$1.0443 cap=0/0 distractor=None err=None

	Wrote hard_results.json
	"""Intense same-model A/B: Opus-4.8 (full replay) vs Opus-4.8 (signal-aware digest).

	Model price is OUT of the equation — both arms run claude-opus-4-8. So:
	• any COST delta is purely the digest replaying fewer tokens, and
	• any QUALITY delta is purely the digest's signal-foregrounding changing what
	the SAME model captures.

	Per scenario × rep × arm we capture real wire usage + the saved artifacts, then
	score quality three ways:
	1. keyword capture: did the saved memory/skill contain the ground-truth signal
	keywords? (per-signal recall; distractor must_not_contain = false-save check)
	2. blind pairwise judge: opus judge sees BOTH arms' saves unlabeled (order
	randomized) and picks which better captures the ground-truth signal, or tie.
	3. memory-durability rubric: judge rates whether the saved memory is the
	durable/important fact vs trivia.

	Output: /tmp/bgreview_bench/hard_results.json
	"""
	import os
	import sys
	import json
	import time
	import random
	import shutil
	import tempfile
	from pathlib import Path

	WORKTREE = "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8"
	sys.path.insert(0, WORKTREE)
	sys.path.insert(0, "/tmp/bgreview_bench")

	MODEL = "claude-opus-4-8"
	REPS = int(os.environ.get("HARD_REPS", "4"))
	PRICES_BY_MODEL = {
	"claude-opus-4-8": {"in": 15.00, "out": 75.00, "cache_read": 1.50, "cache_write": 18.75},
	"claude-haiku-4-5-20251001": {"in": 1.00, "out": 5.00, "cache_read": 0.10, "cache_write": 1.25},
	}
	PRICES = PRICES_BY_MODEL["claude-opus-4-8"] # back-compat default


	def _cost(u, model="claude-opus-4-8"):
	p = PRICES_BY_MODEL.get(model, PRICES)
	return (u["input_tokens"]/1e6p["in"] + u["output_tokens"]/1e6p["out"]
	+ u["cache_read_tokens"]/1e6*p["cache_read"]
	+ u["cache_write_tokens"]/1e6*p["cache_write"])


	def run_arm(scenario, digest_on, api_key, prepass=False, review_model=None):
	hermes_home = tempfile.mkdtemp(prefix="hard_")
	hh = os.path.join(hermes_home, ".hermes")
	os.makedirs(hh, exist_ok=True)
	os.environ["HERMES_HOME"] = hh
	for mod in [m for m in sys.modules if m.startswith(("hermes", "agent", "run_agent", "tools", "model_tools"))]:
	del sys.modules[mod]
	sys.path.insert(0, WORKTREE)

	import yaml
	# The parent (main) model is always MODEL (opus). review_model lets the
	# REVIEW FORK route to a different model (e.g. haiku) — the routing arm —
	# while keeping the parent on opus. None = review runs on the main model.
	rev_model = review_model or MODEL
	br_cfg = {
	"provider": "anthropic", "model": rev_model,
	"max_context_tokens": (10_000_000 if not digest_on else 48000),
	"digest_tail_messages": 24,
	"prepass": prepass,
	}
	cfg = {
	"model": {"provider": "anthropic", "model": MODEL, "default": MODEL},
	"auxiliary": {"background_review": br_cfg},
	"memory": {"memory_enabled": True, "user_profile_enabled": True},
	"skills": {"creation_nudge_interval": 10},
	}
	Path(hh, "config.yaml").write_text(yaml.safe_dump(cfg))
	os.environ["ANTHROPIC_API_KEY"] = api_key

	from run_agent import AIAgent
	from agent import background_review as br
	import anthropic

	tally = {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0}

	def _add(u):
	if u is None:
	return
	tally["input_tokens"] += int(getattr(u, "input_tokens", 0) or 0)
	tally["output_tokens"] += int(getattr(u, "output_tokens", 0) or 0)
	tally["cache_read_tokens"] += int(getattr(u, "cache_read_input_tokens", 0) or 0)
	tally["cache_write_tokens"] += int(getattr(u, "cache_creation_input_tokens", 0) or 0)

	Messages = anthropic.resources.messages.Messages
	o_stream, o_create = Messages.stream, Messages.create

	class _P:
	def __init__(self, m): self._m, self._c = m, False
	def __enter__(self):
	s = self._m.__enter__(); og = s.get_final_message
	def g(a, *k):
	msg = og(a, *k)
	if not self._c:
	self._c = True; _add(getattr(msg, "usage", None))
	return msg
	s.get_final_message = g; return s
	def __exit__(self, a): return self._m.__exit__(a)

	Messages.stream = lambda self, a, k: _P(o_stream(self, a, **k))
	def _wc(self, a, *k):
	r = o_create(self, a, *k); _add(getattr(r, "usage", None)); return r
	Messages.create = _wc

	agent = AIAgent(model=MODEL, provider="anthropic", api_key=api_key,
	api_mode="anthropic_messages", quiet_mode=True,
	skip_context_files=True, skip_memory=False, platform="cli",
	session_id="hard-parent")
	try:
	agent._cached_system_prompt = agent._build_system_prompt()
	except Exception:
	agent._cached_system_prompt = "HARD-PARENT-SYS"
	try:
	target, _ = br.spawn_background_review_thread(
	agent, messages_snapshot=list(scenario["messages"]),
	review_memory=True, review_skills=True)
	target()
	finally:
	Messages.stream, Messages.create = o_stream, o_create
	try:
	agent.close()
	except Exception:
	pass

	skills_dir = Path(hh) / "skills"
	mem_dir = Path(hh) / "memories"
	skill_text = ""
	skill_files = []
	if skills_dir.exists():
	for p in skills_dir.rglob("*"):
	if p.is_file() and ".archive" not in str(p) and not str(p).endswith((".usage.json", ".lock")):
	skill_files.append(str(p.relative_to(skills_dir)))
	try:
	skill_text += "\n" + p.read_text()
	except Exception:
	pass
	saved = {
	"memory": ((mem_dir / "MEMORY.md").read_text() if (mem_dir / "MEMORY.md").exists() else ""),
	"user": ((mem_dir / "USER.md").read_text() if (mem_dir / "USER.md").exists() else ""),
	"skill_files": skill_files,
	"skill_text": skill_text[:4000],
	}
	blob = (saved["memory"] + "\n" + saved["user"] + "\n" + saved["skill_text"]).lower()

	# Keyword capture scoring (objective, per signal).
	sig_results = []
	for s in scenario.get("signals", []):
	hit = any(k.lower() in blob for k in s["must_contain"])
	sig_results.append({"kind": s["kind"], "desc": s["desc"], "captured": hit})
	# Distractor false-save check.
	distractor_saved = None
	if scenario.get("distractor"):
	distractor_saved = any(k.lower() in blob for k in scenario["distractor"]["must_not_contain"])

	shutil.rmtree(hermes_home, ignore_errors=True)
	return {
	"usage": dict(tally), "cost": _cost(tally, rev_model),
	"review_model": rev_model,
	"did_save": bool(blob.strip()), "saved": saved,
	"signal_capture": sig_results, "distractor_saved": distractor_saved,
	}


	def main():
	import hard_scenarios as HS
	api_key = os.environ["ANTHROPIC_API_KEY"]
	out = {"runs": [], "meta": {"reps": REPS, "model": MODEL, "prices": PRICES}}
	for sc in HS.SCENARIOS:
	for rep in range(REPS):
	row = {"scenario": sc["id"], "rep": rep, "position": sc.get("position"),
	"n_signals": len(sc.get("signals", [])),
	"expect_noop": len(sc.get("signals", [])) == 0}
	for arm, dig in (("full", False), ("digest", True)):
	t0 = time.time()
	try:
	r = run_arm(sc, dig, api_key)
	r["error"] = None
	except Exception as e:
	import traceback
	r = {"error": str(e), "trace": traceback.format_exc()[-600:],
	"usage": {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0},
	"cost": 0, "did_save": None, "saved": {}, "signal_capture": [], "distractor_saved": None}
	r["latency_s"] = round(time.time() - t0, 1)
	row[arm] = r
	cap = sum(1 for s in r["signal_capture"] if s["captured"])
	tot = len(r["signal_capture"])
	print(f"[{sc['id']:34} r{rep} {arm:6}] cost=${r['cost']:.4f} "
	f"cap={cap}/{tot} distractor={r['distractor_saved']} err={r['error']}", flush=True)
	out["runs"].append(row)
	Path("/tmp/bgreview_bench/hard_results.json").write_text(json.dumps(out, indent=2))
	print("\nWrote hard_results.json")


	if __name__ == "__main__":
	main()
	pairwise buried_early_style_correction r0 → full
	pairwise buried_early_style_correction r1 → digest
	pairwise buried_early_style_correction r2 → full
	pairwise buried_mid_workflow_rule r0 → full
	pairwise buried_mid_workflow_rule r1 → digest
	pairwise buried_mid_workflow_rule r2 → full
	pairwise buried_early_persona_fact r0 → tie
	pairwise buried_early_persona_fact r1 → tie
	pairwise buried_early_persona_fact r2 → tie
	pairwise mid_technique_with_retracted_distractor r0 → full
	pairwise mid_technique_with_retracted_distractor r1 → full
	pairwise mid_technique_with_retracted_distractor r2 → full
	pairwise multi_signal_session r0 → digest
	pairwise multi_signal_session r1 → full
	pairwise multi_signal_session r2 → full
	pairwise durable_vs_trivia_memory r0 → digest
	pairwise durable_vs_trivia_memory r1 → tie
	pairwise durable_vs_trivia_memory r2 → tie

	==================================================================
	SAME-MODEL A/B (Opus-4.8 both arms) — cost is purely the digest
	cost/review: full $3.5295 digest $1.7864 → 2.0x cheaper
	tokens/review: full 305468 digest 172434
	total spend: full $74.12 digest $37.52

	QUALITY — objective keyword capture (recall of ground-truth signals)
	full: 21/21 = 1.00
	digest: 21/21 = 1.00
	distractor false-saves: full 3 digest 3 (of 6 distractor runs each)
	no-op scenarios correct: full 3/3 digest 3/3

	QUALITY — blind pairwise judge (which arm captured the signal better)
	digest wins: 4 full wins: 9 tie: 5 parse_fail: 0
	==================================================================
	"""Blind pairwise judge + aggregation for the intense same-model A/B.

	Reads hard_results.json. For each scenario×rep with ≥1 ground-truth signal,
	shows the opus judge BOTH arms' saved artifacts UNLABELED in randomized order
	and asks which better captures the ground-truth signal (or tie). Blind + order-
	randomized to remove position/label bias. Also computes the objective keyword-
	capture recall and distractor false-save rate per arm.

	Outputs hard_summary.json + prints the PR-ready tables.
	"""
	import os
	import sys
	import json
	import random
	import statistics as st
	from pathlib import Path

	R = json.loads(Path("/tmp/bgreview_bench/hard_results.json").read_text())

	PAIR_PROMPT = """Two background-review passes each looked at the same conversation \
	and saved memory/skill notes. Below are their saved artifacts, labeled A and B \
	(order is randomized; you don't know which system produced which).

	GROUND-TRUTH the review should have captured:
	{signals}

	--- ARTIFACT A ---
	{a}

	--- ARTIFACT B ---
	{b}

	Judge ONLY on fidelity to the ground-truth: does the artifact capture the actual \
	rule/fact correctly, completely, and durably (will it still be useful next \
	session)? IGNORE length, prose polish, formatting, and verbosity — a shorter \
	artifact that captures the signal just as faithfully is EQUAL, not worse. Only \
	prefer one over the other if it captures the ground-truth signal MORE accurately \
	or would mislead LESS. If both capture it faithfully, answer "tie".

	Reply with ONLY compact JSON:
	{{"winner": "A"\|"B"\|"tie", "reason": "<one sentence on signal fidelity, not style>"}}"""


	def _artifact(saved):
	parts = []
	if saved.get("user", "").strip():
	parts.append("USER PROFILE:\n" + saved["user"][:1200])
	if saved.get("memory", "").strip():
	parts.append("MEMORY:\n" + saved["memory"][:1200])
	if saved.get("skill_text", "").strip():
	parts.append("SKILL(S):\n" + saved["skill_text"][:1800])
	return "\n\n".join(parts) or "(nothing saved)"


	def pairwise(client, signals, full_saved, digest_saved):
	sig_txt = "\n".join(f" • [{s['kind']}] {s['desc']}" for s in signals)
	# Randomize which arm is A vs B to kill position bias.
	flip = random.random() < 0.5
	a_saved, b_saved = (digest_saved, full_saved) if flip else (full_saved, digest_saved)
	# A is full, B is digest UNLESS flipped.
	a_is_full = not flip
	msg = client.messages.create(
	model="claude-opus-4-8", max_tokens=300,
	messages=[{"role": "user", "content": PAIR_PROMPT.format(
	signals=sig_txt, a=_artifact(a_saved), b=_artifact(b_saved))}],
	)
	text = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text").strip()
	if text.startswith("```"):
	text = text.split("```")[1].lstrip("json").strip()
	try:
	verdict = json.loads(text)
	except Exception:
	return {"winner_arm": "parse_fail", "raw": text[:120]}
	w = verdict.get("winner")
	if w == "tie":
	return {"winner_arm": "tie", "reason": verdict.get("reason", "")}
	won_full = (w == "A" and a_is_full) or (w == "B" and not a_is_full)
	return {"winner_arm": "full" if won_full else "digest", "reason": verdict.get("reason", "")}


	def main():
	import anthropic
	client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])

	# Objective capture + cost.
	cost = {"full": [], "digest": []}
	tok = {"full": [], "digest": []}
	sig_total = {"full": 0, "digest": 0}
	sig_hit = {"full": 0, "digest": 0}
	distractor_bad = {"full": 0, "digest": 0}
	distractor_n = 0
	noop_correct = {"full": 0, "digest": 0}
	noop_n = 0
	pair = {"full": 0, "digest": 0, "tie": 0, "parse_fail": 0}

	for row in R["runs"]:
	for arm in ("full", "digest"):
	r = row[arm]
	if r.get("error"):
	continue
	cost[arm].append(r["cost"])
	tok[arm].append(sum(r["usage"].values()))
	for s in r["signal_capture"]:
	sig_total[arm] += 1
	if s["captured"]:
	sig_hit[arm] += 1
	if r.get("distractor_saved") is not None and arm == "full":
	distractor_n += 1
	if r.get("distractor_saved") is True:
	distractor_bad[arm] += 1
	if row["expect_noop"] and arm == "full":
	noop_n += 1
	if row["expect_noop"] and not r["did_save"]:
	noop_correct[arm] += 1
	# Blind pairwise on scenarios with signals.
	if row["n_signals"] > 0 and not row["full"].get("error") and not row["digest"].get("error"):
	import hard_scenarios as HS
	sc = next(s for s in HS.SCENARIOS if s["id"] == row["scenario"])
	v = pairwise(client, sc["signals"], row["full"]["saved"], row["digest"]["saved"])
	pair[v["winner_arm"]] = pair.get(v["winner_arm"], 0) + 1
	print(f" pairwise {row['scenario']:34} r{row['rep']} → {v['winner_arm']}", flush=True)

	def m(x):
	return st.mean(x) if x else 0

	print("\n" + "=" * 66)
	print("SAME-MODEL A/B (Opus-4.8 both arms) — cost is purely the digest")
	print(f" cost/review: full ${m(cost['full']):.4f} digest ${m(cost['digest']):.4f}"
	f" → {m(cost['full'])/max(m(cost['digest']),1e-9):.1f}x cheaper")
	print(f" tokens/review: full {m(tok['full']):.0f} digest {m(tok['digest']):.0f}")
	print(f" total spend: full ${sum(cost['full']):.2f} digest ${sum(cost['digest']):.2f}")
	print("\nQUALITY — objective keyword capture (recall of ground-truth signals)")
	print(f" full: {sig_hit['full']}/{sig_total['full']} = {sig_hit['full']/max(sig_total['full'],1):.2f}")
	print(f" digest: {sig_hit['digest']}/{sig_total['digest']} = {sig_hit['digest']/max(sig_total['digest'],1):.2f}")
	print(f" distractor false-saves: full {distractor_bad['full']} digest {distractor_bad['digest']} (of {distractor_n} distractor runs each)")
	print(f" no-op scenarios correct: full {noop_correct['full']}/{noop_n} digest {noop_correct['digest']}/{noop_n}")
	print("\nQUALITY — blind pairwise judge (which arm captured the signal better)")
	print(f" digest wins: {pair['digest']} full wins: {pair['full']} tie: {pair['tie']} parse_fail: {pair.get('parse_fail',0)}")
	print("=" * 66)

	Path("/tmp/bgreview_bench/hard_summary.json").write_text(json.dumps({
	"cost": {k: {"mean": m(v), "total": sum(v), "n": len(v)} for k, v in cost.items()},
	"tokens": {k: m(v) for k, v in tok.items()},
	"capture": {k: {"hit": sig_hit[k], "total": sig_total[k]} for k in ("full", "digest")},
	"distractor_false_saves": distractor_bad, "distractor_n": distractor_n,
	"noop_correct": noop_correct, "noop_n": noop_n,
	"pairwise": pair,
	}, indent=2))


	if __name__ == "__main__":
	sys.path.insert(0, "/tmp/bgreview_bench")
	sys.path.insert(0, "/home/teknium/.hermes/hermes-agent/.worktrees/hermes-a33baef8")
	main()
	"""Hard scenarios for the same-model (Opus vs Opus) background-review benchmark.

	Design goal: stress the case where the digest should make the SAME model
	capture learnings MORE reliably — signals buried in long, noisy sessions where
	a full-transcript replay dilutes the model's attention across the haystack.

	Each scenario:
	messages — synthetic transcript (signals buried among realistic filler)
	signals — list of ground-truth learnings a perfect review captures, each:
	{kind: "skill"\|"memory", desc, must_contain: [keywords...]}
	position — where the key signal sits (early\|mid\|tail) — to test that the
	digest's signal-foregrounding beats full-replay attention dilution
	distractor — optional: a retracted/false signal that a good review must NOT save
	"""

	FILLER_TOPICS = [
	"the CI matrix sharding", "a flaky import in tests", "the config loader merge order",
	"docs phrasing for the quickstart", "renaming a private helper", "test fixture setup",
	"a typo in a log line", "the changelog wording", "dependency version bumps",
	"formatting a table in markdown", "a lint warning", "import ordering",
	]


	def _filler(n, prefix="f"):
	out = []
	for i in range(n):
	t = FILLER_TOPICS[i % len(FILLER_TOPICS)]
	out.append({"role": "user", "content":
	f"[{prefix}{i}] Quick one about {t}. " + ("Context detail goes here. " * 130)})
	out.append({"role": "assistant", "content":
	f"Here's the rundown on {t}. " + ("Explanation continues at length. " * 130)})
	return out


	SCENARIOS = [
	{
	"id": "buried_early_style_correction",
	"position": "early",
	"signals": [{
	"kind": "skill",
	"desc": "User wants terse, 1-2 sentence answers — durable style preference.",
	"must_contain": ["terse", "concise", "brief", "short", "verbose", "1-2", "one or two"],
	}],
	"messages": (
	[{"role": "user", "content":
	"before we start: you are WAY too verbose. from now on, every answer "
	"is one or two sentences max. no preamble, no recap. i mean it, always."},
	{"role": "assistant", "content": "Understood — terse from here on."}]
	+ _filler(40)
	),
	},
	{
	"id": "buried_mid_workflow_rule",
	"position": "mid",
	"signals": [{
	"kind": "skill",
	"desc": "Always run the full test suite BEFORE committing, never after.",
	"must_contain": ["test", "before", "commit", "suite"],
	}],
	"messages": (
	_filler(20, "a")
	+ [{"role": "user", "content":
	"hold on — you just committed before running tests AGAIN. the rule is: "
	"ALWAYS run the full test suite BEFORE you commit, never after. "
	"i've told you this. make it your standard sequence going forward."},
	{"role": "assistant", "content": "Understood — tests first, then commit."}]
	+ _filler(20, "b")
	),
	},
	{
	"id": "buried_early_persona_fact",
	"position": "early",
	"signals": [{
	"kind": "memory",
	"desc": "User runs a 3-node Proxmox + Ceph homelab; deployment answers should target it.",
	"must_contain": ["proxmox", "ceph", "3-node", "three-node", "cluster", "homelab", "lxc"],
	}],
	"messages": (
	[{"role": "user", "content":
	"context you should keep: all my self-hosting is a 3-node proxmox cluster "
	"with ceph storage at home. whenever i ask about deploying something, "
	"assume that's the target and tailor it to proxmox LXC/VMs."},
	{"role": "assistant", "content": "Noted — I'll target your Proxmox + Ceph setup."}]
	+ _filler(45)
	),
	},
	{
	"id": "mid_technique_with_retracted_distractor",
	"position": "mid",
	"signals": [{
	"kind": "skill",
	"desc": "SIGPIPE on the bridge was fixed by stdin=subprocess.DEVNULL on Popen.",
	"must_contain": ["sigpipe", "stdin", "devnull", "popen", "subprocess"],
	}],
	"distractor": {
	"desc": "An early guess that it was a permissions bug — RETRACTED. Must NOT be saved.",
	"must_not_contain": ["permission", "chmod", "0755"],
	},
	"messages": (
	_filler(15, "a")
	+ [{"role": "user", "content": "the raft bridge dies with SIGPIPE on startup"},
	{"role": "assistant", "content":
	"First guess: might be a permissions issue on the socket — try chmod 0755. "
	"Let me check."},
	{"role": "user", "content": "no, perms are fine, it's not that"},
	{"role": "assistant", "content":
	"You're right, scratch the permissions idea. Found it: the bridge inherits "
	"the parent's stdin and gets SIGPIPE when it closes. The fix is "
	"stdin=subprocess.DEVNULL on the Popen call."},
	{"role": "user", "content": "yep that fixed it"}]
	+ _filler(15, "b")
	),
	},
	{
	"id": "multi_signal_session",
	"position": "mid",
	"signals": [
	{"kind": "skill",
	"desc": "Prefers ruff over flake8 for this repo.",
	"must_contain": ["ruff"]},
	{"kind": "memory",
	"desc": "User's name is Dana and they lead the platform team.",
	"must_contain": ["dana", "platform team", "platform-team"]},
	],
	"messages": (
	_filler(12, "a")
	+ [{"role": "user", "content":
	"btw I'm Dana, I lead the platform team here, so when you suggest "
	"process changes keep that scope in mind."},
	{"role": "assistant", "content": "Good to know, Dana."}]
	+ _filler(8, "b")
	+ [{"role": "user", "content":
	"also stop reaching for flake8 — this repo uses ruff for everything, "
	"always use ruff."},
	{"role": "assistant", "content": "Got it — ruff, not flake8."}]
	+ _filler(12, "c")
	),
	},
	{
	"id": "durable_vs_trivia_memory",
	"position": "mid",
	"signals": [{
	"kind": "memory",
	"desc": "DURABLE: user is allergic to a hard dependency on cloud services — "
	"everything must run self-hosted/offline. This is a standing constraint.",
	"must_contain": ["self-host", "self host", "offline", "no cloud", "on-prem", "on prem", "local"],
	}],
	"distractor": {
	"desc": "TRIVIA also present (today's weather, a one-off file path) that should "
	"NOT crowd out the durable constraint.",
	"must_not_contain": ["weather", "sunny", "/tmp/scratch"],
	},
	"messages": (
	_filler(10, "a")
	+ [{"role": "user", "content": "nice and sunny here today btw"},
	{"role": "assistant", "content": "Enjoy it!"},
	{"role": "user", "content":
	"important standing rule for everything we do: I will NOT take a hard "
	"dependency on any cloud service. everything must run self-hosted and "
	"work fully offline. design every solution that way."},
	{"role": "assistant", "content": "Understood — self-hosted, offline-first, always."},
	{"role": "user", "content": "oh and I dumped that log at /tmp/scratch for now"},
	{"role": "assistant", "content": "Noted."}]
	+ _filler(15, "b")
	),
	},
	{
	"id": "long_noop_smooth",
	"position": "none",
	"signals": [], # correct review = no-op
	"messages": _filler(35, "q"),
	},
	]