|
#!/usr/bin/env python3 |
|
"""Matrix for bd-notes-clobber-guard. |
|
|
|
WHY THIS FILE EXISTS, WRITTEN 2026-08-28. It did not, for the whole life of |
|
the guard. The only thing exercising this hook was a FOUR-ROW inline matrix |
|
inside check-notes-clobber.sh — real clobber denies, --append-notes allows, |
|
the override works, and it is wired. That is a liveness check, not a matrix, |
|
and it could not see either of the two defects found on 2026-08-28: an |
|
additive `gc bd comment` REFUSED because its message contained a semicolon, |
|
and a real `echo "$(bd update x --notes y)"` ALLOWED because a raw split |
|
cannot see into a substitution. The sibling had 260 rows and found neither in |
|
its own copy of the same code, for the same reason. |
|
|
|
Asserts the DENY you expect, never the absence of one: an ALLOW row asserts |
|
empty output, and a DENY row asserts the guard's own reason text. A suite that |
|
only checked "not deny" would pass against a hook that never fires at all — |
|
this city has shipped a selector mutated to match NOTHING that still passed |
|
36/36. |
|
""" |
|
import json, os, re, subprocess, sys, tempfile |
|
|
|
# Overridable so --self-check can point the whole matrix at a MUTANT without |
|
# ever writing over the live guard, which is the file every Bash call in this |
|
# city goes through. |
|
HOOK = os.environ.get("NOTES_GUARD_UNDER_TEST") or os.path.join( |
|
os.path.dirname(os.path.abspath(__file__)), "bd-notes-clobber-guard") |
|
|
|
# PRECONDITION, NOT A SCORED ROW. run() EXECS the hook the way the harness |
|
# does, so a stripped +x bit means nothing below was measured. |
|
if not os.access(HOOK, os.X_OK): |
|
print("PRECONDITION FAILED: %s is not executable, so the harness could not\n" |
|
"run it either. Nothing below would have been measured.\n" |
|
"Fix with: chmod +x %s" % (HOOK, HOOK)) |
|
sys.exit(2) |
|
|
|
_FX = tempfile.mkdtemp(prefix="notes-guard-test-") |
|
# NEVER THE LIVE LOG. The guard appends a record on every DENY and OVERRIDE, |
|
# and this suite drives dozens; without this the audit trail our adoption |
|
# tells a reader to read would be mostly this suite's fixtures. |
|
_LOG = os.path.join(_FX, "guard.log") |
|
|
|
|
|
def run(cmd, hook=None, log=None): |
|
env = dict(os.environ, BD_NOTES_CLOBBER_GUARD_LOG=(log or _LOG)) |
|
p = subprocess.run([hook or HOOK], |
|
input=json.dumps({"tool_name": "Bash", |
|
"tool_input": {"command": cmd}}), |
|
capture_output=True, text=True, env=env) |
|
return p.stdout |
|
|
|
|
|
def is_deny(out): |
|
"""Parse it. A guard whose permissionDecision is flipped to "allow" still |
|
emits JSON containing the word "deny" in its reason text, so a substring |
|
test would score a neutered guard as working.""" |
|
out = out.strip() |
|
if not out: |
|
return False |
|
try: |
|
d = json.loads(out) |
|
except ValueError: |
|
return False |
|
return ((d.get("hookSpecificOutput") or {}).get("permissionDecision") == "deny") |
|
|
|
|
|
# A distinctive span of REASON, so a deny that is somehow some other text |
|
# scores as a failure rather than a pass. |
|
MARK = "REPLACES the notes field" |
|
|
|
|
|
# ─── SELF-CHECK: PROVE THIS MATRIX CAN GO RED ─────────────────────────────── |
|
# |
|
# Without this the rows below are unfalsified. Four properties, each learned by |
|
# getting it wrong first in this city (toolkit/patterns/self-check-harness.sh |
|
# carries the general form): |
|
# 1. one mutant must expect GREEN — a table where every row expects RED |
|
# cannot tell a working guard from a suite that stopped running; |
|
# 2. a mutation that did not apply is a FAILURE, not a pass; |
|
# 3. a missing tally is its own failure, never scored — that is how a |
|
# vacuous RED hides; |
|
# 4. the live file is never written; the mutant arrives by env var. |
|
_SELF_CHECK_MUTANTS = [ |
|
("deny-neutered", "red", |
|
'"permissionDecision": "deny",', |
|
'"permissionDecision": "allow",'), |
|
("offending-blinded", "red", |
|
" found = offending(cmd)\n", |
|
" found = None\n"), |
|
# THE GREEN CONTROL. Perturbs the file enough to satisfy the did-it-apply |
|
# check, changes no behaviour, and MUST stay green. A control that goes red |
|
# under every treatment is not a flaky control, it is a dead harness. |
|
("harmless-docstring-edit", "green", |
|
'A crash here must not wedge every Bash call in the city', |
|
'A crash here must NOT wedge every Bash call in the city'), |
|
] |
|
|
|
SELF_CHECK_RC = 0 |
|
if "--self-check" in sys.argv: |
|
src = open(HOOK).read() |
|
for name, expect, frm, to in _SELF_CHECK_MUTANTS: |
|
if src.count(frm) != 1: |
|
print(" self-check FAIL %-24s anchor matched %d times, not 1 — " |
|
"the mutation is vacuous" % (name, src.count(frm))) |
|
SELF_CHECK_RC = 1 |
|
continue |
|
mp = os.path.join(_FX, "selfcheck-" + name) |
|
with open(mp, "w") as fh: |
|
fh.write(src.replace(frm, to)) |
|
os.chmod(mp, 0o755) |
|
env = dict(os.environ, NOTES_GUARD_UNDER_TEST=mp) |
|
r = subprocess.run([sys.executable, os.path.abspath(__file__)], |
|
capture_output=True, text=True, env=env) |
|
m = re.search(r"(\d+)/(\d+) passed", r.stdout) |
|
if not m: |
|
print(" self-check FAIL %-24s no tally line — the mutant did not " |
|
"RUN, so nothing was measured" % name) |
|
SELF_CHECK_RC = 1 |
|
continue |
|
red = r.returncode != 0 |
|
want_red = (expect == "red") |
|
ok = (red == want_red) |
|
if not ok: |
|
SELF_CHECK_RC = 1 |
|
print(" self-check %-4s %-24s suite went %s as expected (%s passed)" |
|
% ("ok" if ok else "FAIL", name, |
|
"red" if red else "green", m.group(0).split()[0])) |
|
print() |
|
|
|
|
|
total = fails = 0 |
|
|
|
|
|
def check(want_deny, cmd, label): |
|
global total, fails |
|
total += 1 |
|
out = run(cmd) |
|
denied = is_deny(out) |
|
if denied and MARK not in out: |
|
got, ok = "deny:WRONG-TEXT", False |
|
else: |
|
got = "DENY " if denied else "allow" |
|
ok = (denied == want_deny) |
|
if not ok: |
|
fails += 1 |
|
print("%-4s %-52s %s" % ("ok" if ok else "FAIL", label, got)) |
|
|
|
|
|
for want, cmd, label in ( |
|
# ---- the thing the guard exists for ---- |
|
(True, 'bd update bl-x --notes "y"', "bare bd update --notes"), |
|
(True, 'gc bd update bl-x --notes "y"', "gc bd passthrough"), |
|
(True, 'bd update bl-x --notes=y', "--notes= joined form"), |
|
(True, 'bd -C /home/gascity/rigs/examplerig update ta-x --notes y', "bd -C <dir>"), |
|
(True, 'env -u BEADS_DIR bd update bl-x --notes y', "behind env -u"), |
|
(True, 'timeout 60 bd update bl-x --notes y', "behind a wrapper"), |
|
(True, 'ls && bd update bl-x --notes y', "second statement in a chain"), |
|
|
|
# ---- the safe spellings, which are the whole reason the rule is narrow ---- |
|
(False, 'bd update bl-x --append-notes "y"', "ctrl: --append-notes"), |
|
(False, 'gc bd note bl-x --file /tmp/n.md', "ctrl: bd note --file"), |
|
(False, 'gc bd update bl-x --description "y"', "ctrl: --description"), |
|
(False, 'bd note bl-x -- "---"', "ctrl: bd note"), |
|
(False, 'gc bd comment bl-x "hello"', "ctrl: bd comment"), |
|
(False, 'bd show bl-x', "ctrl: an unrelated bd call"), |
|
|
|
# ---- INVOCATION, NOT MENTION. This file, CLAUDE.md and every commit |
|
# message about the ban carry the literal string. |
|
(False, 'git commit -m "never use bd update --notes"', "ctrl: prose in a commit message"), |
|
(False, 'grep -n "bd update --notes" CLAUDE.md', "ctrl: grepping for the ban"), |
|
(False, 'echo bd update bl-x --notes y', "ctrl: an UNQUOTED mention after echo"), |
|
(False, "cat > f <<'EOF'\nbd update x --notes y\nEOF", "ctrl: heredoc body is data"), |
|
|
|
# ---- THE STATEMENT SCANNER, BOTH DIRECTIONS. Added 2026-08-28 with the |
|
# quote-aware scan that fixed them; every row below was wrong before. |
|
# |
|
# FALSE POSITIVES: a separator inside a quoted argument is prose. The live |
|
# instance was a `gc bd comment` on the zero-day rehearsal city refused at |
|
# 2026-08-28T20:38:34Z for explaining this very ban. |
|
(False, "gc bd comment bl-x 'foo; bd update y --notes z'", "scan fp: `;` in a single-quoted arg"), |
|
(False, 'gc bd comment bl-x "foo; bd update y --notes z"', "scan fp: `;` in a double-quoted arg"), |
|
(False, 'gc bd comment bl-x "a && bd update y --notes z"', "scan fp: `&&` in a quoted arg"), |
|
(False, 'gc bd comment bl-x "a | bd update y --notes z"', "scan fp: `|` in a quoted arg"), |
|
(False, 'gc bd comment bl-x "a\nbd update y --notes z"', "scan fp: newline in a quoted arg"), |
|
(False, "gc bd comment bl-x $'a; bd update y --notes z'", "scan fp: ANSI-C $'...' quoting"), |
|
# AND THIS ONE IS A CONTROL, NOT A DEMONSTRATION, WHICH IS WORTH SAYING. |
|
# It is the closest reconstruction of the live 2026-08-28 refusal, and it |
|
# was ALLOWED BY THE OLD GUARD TOO — a bare `--notes` after a separator is |
|
# not enough on its own, so the live line must have presented a bd-initial |
|
# fragment in the remainder, and the remainder is long and was truncated at |
|
# 400 characters. The CLASS reproduces in the six rows above; this exact |
|
# byte sequence does not, and the bead says so rather than claiming it. |
|
(False, 'gc bd comment bl-x "close it; --notes replaces existing notes"', |
|
"scan ctrl: the live shape, still not reproduced"), |
|
# |
|
# FALSE NEGATIVES, the worse half: a command SUBSTITUTION is command text |
|
# wearing an argument's costume. Every one of these really does clobber. |
|
(True, 'echo "$(bd update bl-x --notes y)"', "scan fn: $( ) inside double quotes"), |
|
(True, 'echo $(bd update bl-x --notes y)', "scan fn: bare $( )"), |
|
(True, 'echo "`bd update bl-x --notes y`"', "scan fn: backticks"), |
|
(True, 'X=$(ls; bd update bl-x --notes y)', "scan ctrl: separator inside $( ) was already caught"), |
|
(True, 'echo "$(echo "$(bd update bl-x --notes y)")"', "scan fn: nested substitution"), |
|
# |
|
# EVERY ROW ABOVE WAS CHECKED AGAINST BOTH GUARDS, and three labels were |
|
# wrong until it was. Each `fp`/`fn` row must give a DIFFERENT verdict on |
|
# the pre-fix guard than on this one; each `ctrl` row must give the SAME. |
|
# Run it before adding a row here — a row that scores identically on both |
|
# is not testing the change, however well it reads: |
|
# |
|
# git show <pre-fix-rev>:assets/claude-hooks/bd-notes-clobber-guard > /tmp/old |
|
# |
|
# It found: a "live shape" row allowed by both, a substitution row the raw |
|
# split already caught, and — the useful direction — a row filed as a |
|
# control that turned out to be a real FALSE NEGATIVE the fix closes. |
|
# |
|
# AND THE SCAN MUST NOT COST THE OLD ANSWERS. |
|
(True, 'ls; bd update bl-x --notes y', "scan ctrl: real `;` still splits"), |
|
(True, 'echo "quoted" && bd update bl-x --notes y', "scan ctrl: separator after a closed quote"), |
|
(True, 'bd update bl-x --notes "a; b"', "scan ctrl: separator INSIDE the note text"), |
|
(True, 'bd update bl-x 2>&1 --notes y', "scan fn: a redirect is not a separator"), |
|
# |
|
# UNBALANCED QUOTING FALLS BACK TO THE RAW SPLIT — the conservative |
|
# direction: it can over-refuse, loudly, and never under-refuses. |
|
(True, "echo 'unclosed && bd update bl-x --notes y", "scan ctrl: unclosed quote still catches a real one"), |
|
(True, "echo it's fine; bd update bl-x --notes y", "scan ctrl: bare apostrophe unbalances, raw split refuses"), |
|
(False, 'echo "it\'s fine; not a bd update --notes call"', "scan ctrl: apostrophe INSIDE double quotes is fine"), |
|
|
|
# ---- THE ESCAPE HATCH IS PART OF THE CONTRACT. If it stops working, |
|
# agents are hard-walled and the block message is lying to them. |
|
(False, 'BD_DESTROY_ALL_PRIOR_NOTES=1 bd update bl-x --notes y', "override: named on the same line"), |
|
(False, 'BD_DESTROY_ALL_PRIOR_NOTES=true gc bd update bl-x --notes y', "override: =true"), |
|
(True, 'BD_DESTROY_ALL_PRIOR_NOTES=0 bd update bl-x --notes y', "override: =0 is not truthy"), |
|
(True, 'BD_DESTROY_ALL_PRIOR_NOTES=1 ls; bd update bl-x --notes y', |
|
"override is scoped to ITS statement"), |
|
(False, 'env BD_DESTROY_ALL_PRIOR_NOTES=1 bd update bl-x --notes y', "override through env"), |
|
): |
|
check(want, cmd, label) |
|
|
|
|
|
# ─── THE AUDIT RECORD ─────────────────────────────────────────────────────── |
|
# |
|
# Our adoption checklist tells a reader to audit this log for false refusals, |
|
# so the log's SHAPE is part of the contract, not decoration. All three |
|
# properties below were absent until 2026-08-28, and the first is what cost |
|
# the bl-l88il investigation a session: the one refusal anybody ever questioned |
|
# was cut at 400 characters, mid-sentence, and had to be recovered from an |
|
# agent's transcript instead. |
|
_ALOG = os.path.join(_FX, "audit.log") |
|
long_note = "z" * 12000 |
|
for cmd in ('bd update bl-x --notes "line one\nline two"', |
|
'bd update bl-y --notes "%s"' % long_note, |
|
'BD_DESTROY_ALL_PRIOR_NOTES=1 bd update bl-z --notes "ok"'): |
|
run(cmd, log=_ALOG) |
|
# READ IT DEFENSIVELY, BECAUSE A BLINDED GUARD WRITES NOTHING AT ALL. The |
|
# self-check's `offending-blinded` mutant refuses nothing, so this file does |
|
# not exist — and an unguarded open() there raises, kills the process before |
|
# the tally, and the self-check can only report "the mutant did not RUN". That |
|
# is a true statement about a suite that ran fine and simply crashed, i.e. the |
|
# harness lying about its own subject. Missing reads as empty, and the rows |
|
# below then fail for the reason they are written to fail for. |
|
lines = open(_ALOG).read().splitlines() if os.path.exists(_ALOG) else [] |
|
|
|
|
|
def _kind(i): |
|
"""Field 2 of record i, or "" — never an IndexError that eats the tally.""" |
|
if i >= len(lines): |
|
return "" |
|
parts = lines[i].split() |
|
return parts[1] if len(parts) > 1 else "" |
|
|
|
|
|
def _rec(i): |
|
return lines[i] if i < len(lines) else "" |
|
|
|
|
|
for want, got, label in ( |
|
(3, len(lines), "one line per record, even with an embedded newline"), |
|
(1, sum(1 for i in range(len(lines)) if _kind(i) == "OVERRIDE"), "the override is recorded, not silent"), |
|
(2, sum(1 for i in range(len(lines)) if _kind(i) == "BLOCKED"), "both refusals are recorded"), |
|
): |
|
total += 1 |
|
ok = (want == got) |
|
if not ok: |
|
fails += 1 |
|
print("%-4s %-52s %s" % ("ok" if ok else "FAIL", "log: " + label, |
|
"%s" % got if ok else "got %r, want %r" % (got, want))) |
|
|
|
for want, got, label in ( |
|
(True, "line one\\nline two" in _rec(0), |
|
"an embedded newline is ESCAPED, not written raw"), |
|
(True, "[TRUNCATED," in _rec(1), |
|
"a cut record SAYS it was cut — a partial that looks whole is the bug"), |
|
(True, len(_rec(1)) > 4000, |
|
"the cap is far past the 400 that lost the last investigation"), |
|
(False, "[TRUNCATED," in _rec(0), |
|
"a short record is not marked as cut"), |
|
): |
|
total += 1 |
|
ok = (want == got) |
|
if not ok: |
|
fails += 1 |
|
print("%-4s %-52s %s" % ("ok" if ok else "FAIL", "log: " + label, "ok" if ok else "WRONG")) |
|
|
|
|
|
# ─── MUTATION: PROVE THE SCANNER AND THE INVOCATION TEST ARE LOAD-BEARING ─── |
|
# |
|
# The rows above are ALLOWs for the most part, and an ALLOW is silence: a guard |
|
# that never fires scores identically. These four mutants each remove one thing |
|
# and require the matrix to move in a NAMED direction. |
|
src = open(HOOK).read() |
|
for label, frm, to, probes, want_deny in ( |
|
# The scan is what stops the false positive: force the raw fallback and the |
|
# refusal that started all this comes straight back. |
|
("scan-off", " parts = scan_statements(text)\n", " parts = None\n", |
|
(("gc bd comment bl-x 'foo; bd update y --notes z'", "raw split re-refuses prose"), |
|
('gc bd comment bl-x "a && bd update y --notes z"', "raw split, double quotes")), |
|
True), |
|
# Harvesting substitution bodies is what stops the false negative. |
|
("subs-off", " for s in subs:\n", " for s in []:\n", |
|
(('echo "$(bd update bl-x --notes y)"', "substitution goes unjudged"), |
|
('echo "`bd update bl-x --notes y`"', "backticks go unjudged")), |
|
False), |
|
# The invocation test is what keeps documentation editable. Note which |
|
# probes can show that and which cannot: a mention inside a QUOTED argument |
|
# is protected twice over, by this test AND by tokenisation, so it stays |
|
# allowed under the mutant and proves nothing. Written that way first, and |
|
# the two rows passed while testing nothing. The probes have to be UNQUOTED |
|
# mentions, where the invocation test is the only thing standing there. |
|
("invocation-off", |
|
" args = invokes_bd(tokens, env)\n if args is None:\n continue\n", |
|
" args = invokes_bd(tokens, env)\n if args is None:\n args = tokens\n", |
|
(("echo bd update bl-x --notes y", "an unquoted mention after echo"), |
|
("printf bd update bl-x --notes y", "and after printf")), |
|
True), |
|
# And the heredoc strip, for the same reason, by a different route. |
|
("heredoc-strip-off", " for stmt in statements(cmd):\n", |
|
" for stmt in statements(cmd.replace('<<', '< <')):\n", |
|
(("cat > f <<'EOF'\nbd update x --notes y\nEOF", "heredoc body judged as commands"),), |
|
True), |
|
): |
|
if src.count(frm) != 1: |
|
print("FAIL %-52s %s" % (label + " anchor", |
|
"matched %d times, not 1 — vacuous" % src.count(frm))) |
|
fails += 1 |
|
total += 1 |
|
continue |
|
mp = os.path.join(_FX, "mutant-" + label) |
|
with open(mp, "w") as fh: |
|
fh.write(src.replace(frm, to)) |
|
os.chmod(mp, 0o755) |
|
for cmd, what in probes: |
|
total += 1 |
|
out = run(cmd, hook=mp) |
|
denied = is_deny(out) |
|
ok = (denied == want_deny) |
|
if not ok: |
|
fails += 1 |
|
print("%-4s %-52s %s" % ("ok" if ok else "FAIL", "%s: %s" % (label, what), |
|
"guard is load-bearing" if ok |
|
else ("STILL DENIES — fires without it" if denied |
|
else "NO LONGER DENIES — it was dead code"))) |
|
|
|
print("\n%d/%d passed" % (total - fails, total)) |
|
sys.exit(1 if (fails or SELF_CHECK_RC) else 0) |