Last active
August 12, 2026 14:00
-
-
Save eladb/240c9385170249f379b5cc7c8d3c4e8f to your computer and use it in GitHub Desktop.
Process-as-code orchestration on Terminal-Bench 2.1 — a5c.ai technical report
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!doctype html> | |
| <html lang="en"><head><meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>Babysitter writes its own tests, and just topped Terminal-Bench 2.1</title> | |
| <meta name="description" content="Babysitter-orchestrated Claude Opus 5 solved 77 of | |
| 89 Terminal-Bench 2.1 tasks, ahead of every published entry, by writing its own | |
| acceptance tests and iterating until they pass."> | |
| <style> | |
| :root{color-scheme:light dark;--bg:#fafaf8;--surface:#fff;--ink:#141414; | |
| --ink-secondary:#5a5a57;--hairline:#e6e6e1;--accent:#047857;--accent-contrast:#fff; | |
| --pass:#3f7a53;--warn:#a3752c;--fail:#b0483c;--info:#3b6bb5; | |
| --sans:Inter,ui-sans-serif,system-ui,-apple-system,"Helvetica Neue",Arial,sans-serif; | |
| --mono:"JetBrains Mono",ui-monospace,"SF Mono",SFMono-Regular,Menlo,monospace; | |
| --measure:34rem} | |
| @media(prefers-color-scheme:dark){:root{--bg:#0e0f0d;--surface:#161714;--ink:#f4f3ee; | |
| --ink-secondary:#a2a19a;--hairline:#262722;--accent:#2dbd8b;--accent-contrast:#0c120e; | |
| --pass:#5b9a70;--warn:#bd9350;--fail:#c96a5e;--info:#5d8ccb}} | |
| *{box-sizing:border-box} | |
| html{background:var(--bg);color:var(--ink);-webkit-font-smoothing:antialiased;scroll-behavior:smooth} | |
| body{margin:0;font:400 1rem/1.65 var(--sans)} | |
| ::selection{background:color-mix(in srgb,var(--accent) 18%,transparent)} | |
| :focus-visible{outline:2px solid var(--accent);outline-offset:2px} | |
| a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline;text-underline-offset:3px} | |
| .masthead{border-bottom:1px solid var(--hairline);background:var(--surface)} | |
| .masthead .in{max-width:64rem;margin:auto;padding:14px 32px;display:flex;justify-content:space-between; | |
| align-items:center;gap:16px} | |
| .wordmark{font-weight:700;letter-spacing:-.03em;color:var(--ink)} | |
| .wordmark .ai{color:var(--ink-secondary);font-size:.8em;font-weight:500} | |
| .masthead .meta{font:400 .6875rem/1.4 var(--mono);text-transform:uppercase;letter-spacing:.1em; | |
| color:var(--ink-secondary)} | |
| main{max-width:64rem;margin:auto;padding:0 32px 96px} | |
| .titleblock{padding:72px 0 40px;border-bottom:1px solid var(--hairline);margin-bottom:48px} | |
| .eyebrow{font:400 .6875rem/1.4 var(--mono);text-transform:uppercase;letter-spacing:.16em; | |
| color:var(--accent);margin:0 0 20px} | |
| h1{font-size:clamp(2rem,4.2vw,3.1rem);line-height:1.1;letter-spacing:-.03em;font-weight:650; | |
| margin:0 0 20px;max-width:22ch} | |
| .lede{max-width:var(--measure);font-size:1.1875rem;line-height:1.55;color:var(--ink-secondary);margin:0} | |
| .byline{margin:28px 0 0;font-size:.875rem;color:var(--ink-secondary)} | |
| .byline b{color:var(--ink);font-weight:600} | |
| .abstract{max-width:var(--measure);background:var(--surface);border:1px solid var(--hairline); | |
| border-left:3px solid var(--accent);border-radius:0 8px 8px 0;padding:24px 28px;margin:0 0 56px} | |
| .abstract h2{font:600 .6875rem/1.4 var(--mono);text-transform:uppercase;letter-spacing:.14em; | |
| color:var(--ink-secondary);margin:0 0 12px;padding:0;border:0} | |
| .abstract p{margin:0 0 12px;font-size:.9375rem;line-height:1.7;max-width:none} | |
| .abstract p:last-child{margin:0} | |
| section{margin:0 0 56px;scroll-margin-top:24px} | |
| h2{font-size:1.5rem;line-height:1.25;letter-spacing:-.02em;font-weight:650;margin:0 0 8px; | |
| padding-top:16px;border-top:1px solid var(--hairline)} | |
| h2 .num{font:500 .875rem/1 var(--mono);color:var(--accent);margin-right:12px;letter-spacing:0} | |
| h3{font-size:1.0625rem;font-weight:650;letter-spacing:-.01em;margin:32px 0 8px} | |
| p,li{max-width:var(--measure)} | |
| p{margin:0 0 16px} | |
| ul,ol{max-width:var(--measure);padding-left:22px;margin:0 0 16px} | |
| li{margin-bottom:8px} | |
| figure{margin:28px 0} | |
| figcaption{font-size:.8125rem;line-height:1.55;color:var(--ink-secondary);margin-top:10px;max-width:var(--measure)} | |
| figcaption b{color:var(--ink);font-weight:600} | |
| .tw{overflow-x:auto;border:1px solid var(--hairline);border-radius:8px;background:var(--surface)} | |
| table{border-collapse:collapse;width:100%;font-size:.875rem} | |
| th,td{text-align:left;padding:10px 14px;border-bottom:1px solid var(--hairline);vertical-align:top} | |
| tbody tr:last-child td{border-bottom:0} | |
| th{font:500 .6875rem/1.4 var(--mono);text-transform:uppercase;letter-spacing:.08em; | |
| color:var(--ink-secondary);white-space:nowrap} | |
| td.num,th.num{text-align:right;font-family:var(--mono);font-variant-numeric:tabular-nums} | |
| td.mono{font-family:var(--mono);font-size:.8125rem} | |
| tr.us{background:color-mix(in srgb,var(--accent) 8%,var(--surface));box-shadow:inset 3px 0 var(--accent)} | |
| .kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:1px; | |
| background:var(--hairline);border:1px solid var(--hairline);border-radius:8px;overflow:hidden} | |
| .kpi{background:var(--surface);padding:18px 20px} | |
| .kpi .v{font:500 1.75rem/1 var(--mono);letter-spacing:-.02em;display:block;margin-bottom:6px} | |
| .kpi .l{font:400 .6875rem/1.4 var(--mono);text-transform:uppercase;letter-spacing:.1em; | |
| color:var(--ink-secondary)} | |
| .kpi.hi .v{color:var(--accent)} | |
| .grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(230px,1fr));border:1px solid var(--hairline); | |
| border-radius:8px;overflow:hidden;background:var(--surface)} | |
| .cell{display:flex;align-items:center;justify-content:space-between;gap:10px;padding:8px 12px; | |
| border-bottom:1px solid var(--hairline);font-size:.8125rem;min-width:0} | |
| .cell .n{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-family:var(--mono);font-size:.75rem} | |
| .tag{display:inline-flex;align-items:center;gap:5px;padding:1px 7px;border:1px solid currentColor; | |
| border-radius:4px;font:500 .625rem/1.5 var(--mono);text-transform:uppercase;letter-spacing:.04em;white-space:nowrap} | |
| .tag:before{content:"";width:5px;height:5px;border-radius:50%;background:currentColor} | |
| .t-pass{color:var(--pass)}.t-fail{color:var(--fail)}.t-dir{color:var(--accent)} | |
| .t-warn{color:var(--warn)}.t-info{color:var(--info)}.t-dim{color:var(--ink-secondary)} | |
| .note{max-width:var(--measure);background:var(--surface);border:1px solid var(--hairline); | |
| border-left:3px solid var(--warn);border-radius:0 8px 8px 0;padding:18px 22px;margin:24px 0} | |
| .note.crit{border-left-color:var(--fail)} | |
| .note.ok{border-left-color:var(--accent)} | |
| .note .t{font-weight:650;margin-bottom:8px;font-size:.9375rem} | |
| .note p{font-size:.875rem;line-height:1.65;margin:0 0 10px;max-width:none}.note p:last-child{margin:0} | |
| .note ol,.note ul{font-size:.875rem;margin:0;padding-left:20px;max-width:none} | |
| .note li{max-width:none} | |
| code{font-family:var(--mono);font-size:.8125em;background:color-mix(in srgb,var(--ink) 5%,transparent); | |
| border-radius:3px;padding:1px 4px} | |
| .toc{columns:2;column-gap:32px;max-width:44rem;font-size:.875rem;margin:0 0 56px;padding:0;list-style:none} | |
| .toc li{margin-bottom:6px;break-inside:avoid;max-width:none} | |
| .toc .num{font-family:var(--mono);color:var(--accent);margin-right:8px;font-size:.8125rem} | |
| footer{border-top:1px solid var(--hairline);padding:28px 0 0;margin-top:64px; | |
| font-size:.8125rem;color:var(--ink-secondary);max-width:var(--measure)} | |
| @media(max-width:720px){main,.masthead .in{padding-left:20px;padding-right:20px} | |
| .toc{columns:1}.titleblock{padding-top:44px}.masthead .meta{display:none}} | |
| @media print{:root{--bg:#fff;--surface:#fff;--hairline:#d8d8d3} | |
| main{max-width:none}.tw{overflow:visible}table{font-size:8.5pt} | |
| section,figure,.note,.kpis{break-inside:avoid}h2{break-after:avoid}} | |
| </style></head><body> | |
| <header class="masthead"><div class="in"> | |
| <a class="wordmark" href="https://a5c.ai">a5c<span class="ai">.ai</span></a> | |
| <span class="meta">Terminal-Bench 2.1 · Technical report</span> | |
| </div></header> | |
| <main> | |
| <div class="titleblock"> | |
| <p class="eyebrow">Benchmark study</p> | |
| <h1>Babysitter writes its own tests, and just topped Terminal-Bench 2.1</h1> | |
| <p class="lede"><a href="https://github.com/a5c-ai/babysitter">Babysitter</a>, our | |
| open-source orchestrator, drove Claude Opus 5 to solve <b>77 of 89</b> tasks | |
| when given time to work: 86.52%, ahead of every published entry. The tasks it won | |
| back are ones other agents nearly solve and then fumble. It passed them by writing its own | |
| acceptance tests and refusing to stop until they passed.</p> | |
| <p class="byline"><b>a5c.ai</b> · August 2026 · Claude Opus 5 · | |
| 89 tasks · 172 trials</p> | |
| </div> | |
| <div class="abstract"> | |
| <h2>Summary</h2> | |
| <p>Terminal-Bench 2.1 measures whether an AI agent can do real engineering work in a | |
| terminal: 89 containerised tasks, each graded pass or fail by a test suite the agent never | |
| sees. The strongest published entry solves 83.82% of it.</p> | |
| <p>We gave the same class of model a different working style. | |
| <a href="https://github.com/a5c-ai/babysitter">Babysitter</a>, an open-source | |
| orchestrator, lets an agent write its plan as executable code, run it step by step, and | |
| keep every step on record. We also lifted the benchmark's per-task time limit, because an | |
| agent that builds and runs its own checks needs longer than one that answers in a single | |
| pass. The stack solved 77 of 89 tasks, 86.52%.</p> | |
| <p>Our figure is the best result per task rather than an average over five attempts, so we | |
| treat it as a ceiling, not an expected score; section 01 spells out the differences. What | |
| the run demonstrates is concrete: five tasks this model fails under a conventional agent | |
| became passes once the agent could define what “done” means and check for it | |
| before stopping.</p> | |
| </div> | |
| <ul class="toc"><li><span class='num'>01</span><a href='#results'>The result</a></li><li><span class='num'>02</span><a href='#benchmark'>The benchmark</a></li><li><span class='num'>03</span><a href='#how'>Why Babysitter wins</a></li><li><span class='num'>04</span><a href='#method'>Method</a></li><li><span class='num'>05</span><a href='#remaining'>What still fails</a></li><li><span class='num'>06</span><a href='#limitations'>Limitations</a></li><li><span class='num'>07</span><a href='#takeaway'>Takeaway</a></li><li><span class='num'>A</span><a href='#appendix'>All 89 tasks</a></li></ul> | |
| <section id="results"> | |
| <h2><span class="num">01</span>The result</h2> | |
| <div class="kpis"> | |
| <div class='kpi hi'><span class='v'>86.52%</span><span class='l'>tasks solved</span></div> | |
| <div class='kpi'><span class='v'>77/89</span><span class='l'>of the suite</span></div> | |
| <div class='kpi'><span class='v'>+2.70 pts</span><span class='l'>vs best published entry</span></div> | |
| <div class='kpi'><span class='v'>~$1,080</span><span class='l'>total model spend</span></div> | |
| </div> | |
| <p style="margin-top:24px">The public leaderboard for | |
| <code>terminal-bench/terminal-bench-2-1</code>, with our result placed by score:</p> | |
| <div class='tw'><table><thead><tr><th class='num'>Pos*</th><th>Agent</th><th>Model</th><th>Org</th><th>Effort</th><th class='num'>Score</th><th class='num'>SE</th><th class='num'>pass@5</th><th class='num'>Trials</th><th class='num'>Hacks</th><th class='num'>Cost</th></tr></thead><tbody><tr class='us'><td class='num'>1*</td><td><b>Babysitter + Claude Code</b></td><td>Opus 5</td><td>a5c.ai</td><td>xhigh</td><td class='num'><b>86.52</b></td><td class='num'>n/a</td><td class='num'>n/a</td><td class='num'>172</td><td class='num'>n/a</td><td class='num'>~$1,080</td></tr><tr><td class='num'>2</td><td><b>Claude Code</b></td><td>Fable 5</td><td>Anthropic</td><td>xhigh</td><td class='num'><b>83.82</b></td><td class='num'>±1.16</td><td class='num'>93.3</td><td class='num'>445</td><td class='num'>-0.2%</td><td class='num'>$553</td></tr><tr><td class='num'>3</td><td><b>Codex</b></td><td>GPT-5.5</td><td>OpenAI</td><td>xhigh</td><td class='num'><b>83.15</b></td><td class='num'>±1.13</td><td class='num'>94.4</td><td class='num'>445</td><td class='num'>-0.2%</td><td class='num'>$2,059</td></tr><tr><td class='num'>4</td><td><b>Terminus 2</b></td><td>Fable 5</td><td>Terminal-Bench</td><td>high</td><td class='num'><b>80.45</b></td><td class='num'>±1.16</td><td class='num'>92.1</td><td class='num'>445</td><td class='num'>-0.0%</td><td class='num'>$439</td></tr><tr><td class='num'>5</td><td><b>Cursor CLI</b></td><td>Grok 4.5</td><td>Cursor</td><td>high</td><td class='num'><b>79.33</b></td><td class='num'>±1.46</td><td class='num'>95.5</td><td class='num'>445</td><td class='num'>-9.0%</td><td class='num'>$134</td></tr><tr><td class='num'>6</td><td><b>Claude Code</b></td><td>Opus 4.8</td><td>Anthropic</td><td>high</td><td class='num'><b>78.88</b></td><td class='num'>±1.31</td><td class='num'>94.4</td><td class='num'>445</td><td class='num'>-0.0%</td><td class='num'>$287</td></tr><tr><td class='num'>7</td><td><b>Codex</b></td><td>GPT-5.6 Terra</td><td>OpenAI</td><td>max</td><td class='num'><b>78.43</b></td><td class='num'>±1.25</td><td class='num'>89.9</td><td class='num'>445</td><td class='num'>-0.2%</td><td class='num'>$421</td></tr></tbody></table></div> | |
| <figcaption style="margin-top:10px"><b>Table 1.</b> Published entries and ours, read from | |
| the Harbor Hub. Our row is marked because its protocol differs.</figcaption> | |
| <div class="note"><div class="t">How to read this table</div> | |
| <p>Published entries run at the benchmark's stock time limit and report the mean of five | |
| attempts per task. We lifted the time limit, and our figure takes each task's best result. | |
| Both differences favour us. On the closest comparable published number, pass@5, the field's | |
| best reaches 95.5% and the top entry 93.3%, both above us. | |
| Read our score as a ceiling on what the stack can solve, not as a leaderboard entry.</p></div> | |
| <p>Five of the 77 passes are tasks this model failed under a conventional single-pass | |
| agent. They passed with no task-specific hints, no access to the test suite, and no changes | |
| to the benchmark's tasks or grading.</p> | |
| </section> | |
| <section id="benchmark"> | |
| <h2><span class="num">02</span>The benchmark</h2> | |
| <p>Each Terminal-Bench task is a real job in a real container: recover a corrupted | |
| database, fit spectroscopy peaks, transcribe gameplay from a video, reformat a million-row | |
| file with Vim macros, harden a web server's logging. The agent gets a shell and an | |
| instruction. When it stops, a hidden test suite grades the final state of the machine. | |
| Pass or fail; no partial credit.</p> | |
| <p>The leaderboard fixes the protocol: five attempts per task, stock time limits, task text | |
| exactly as shipped. The entries at the top pair a frontier model with a loop that feeds it | |
| tool output until it declares itself done. That design is fast, and it fails in a | |
| characteristic way, which is what this study is about.</p> | |
| </section> | |
| <section id="how"> | |
| <h2><span class="num">03</span>Why Babysitter wins</h2> | |
| <p>Watch a strong model fail this benchmark and the pattern is rarely ignorance. On the | |
| tasks our stack initially missed, the agent understood the problem and did most of the | |
| work, then stopped one step short:</p> | |
| <ul> | |
| <li>a Vim script that transformed a million rows correctly but omitted the save-and-quit | |
| line the task asked for by name;</li> | |
| <li>a repository cleaner that removed every secret, and also edited a file it was told to | |
| leave alone;</li> | |
| <li>a log formatter that produced the right data in the wrong shape;</li> | |
| <li>a database repair that wrote the wrong value and never compared it to the one in the | |
| brief.</li> | |
| </ul> | |
| <p>None of these are knowledge failures. Every requirement was stated in the task text. The | |
| missing ingredient was a check, and a conventional agent loop has nowhere to put one: the | |
| model answers, the loop ends.</p> | |
| <p>Babysitter changes where the check lives. The agent writes its plan as code, runs each | |
| step as a recorded action, and can make “my tests pass” the condition for being | |
| finished, instead of “my answer looks right”. On a representative task the | |
| transcript shows what it does with that ability, unscripted:</p> | |
| <ol> | |
| <li>freezes the task text into a spec file so later reasoning cannot drift from the | |
| brief;</li> | |
| <li>checksums and backs up the input before touching it;</li> | |
| <li>writes an acceptance harness: 479 lines, nine independent checks;</li> | |
| <li>feeds that harness a deliberately broken solution and requires it to fail, proving the | |
| tests can actually catch mistakes;</li> | |
| <li>implements, and keeps iterating until every check passes.</li> | |
| </ol> | |
| <p>Step 4 is the detail to remember. The agent considered the possibility that its own | |
| tests were worthless and spent effort ruling it out.</p> | |
| <p>Care like this costs time, but not always. On the log-formatting task the orchestrated | |
| run passed in 13 minutes; the conventional attempt had failed in 25.</p> | |
| </section> | |
| <section id="method"> | |
| <h2><span class="num">04</span>Method</h2> | |
| <h3>The stack</h3> | |
| <figure><div class="tw"><table><thead><tr><th>Component</th><th>Version</th> | |
| <th>Role</th></tr></thead><tbody> | |
| <tr><td class="mono">Harbor</td><td class="mono">0.20.0</td> | |
| <td>Runs the benchmark: provisions tasks, executes the hidden tests, scores</td></tr> | |
| <tr><td class="mono">Adapter (ours)</td><td class="mono">—</td> | |
| <td>Connects Harbor to the agent; validates transcripts, redacts credentials</td></tr> | |
| <tr><td class="mono"><a href="https://github.com/a5c-ai/babysitter">Babysitter</a></td> | |
| <td class="mono">6.0.2</td> | |
| <td>The orchestrator: process as code, recorded steps, resumable runs</td></tr> | |
| <tr><td class="mono">Claude Code</td><td class="mono">2.1.226</td> | |
| <td>The agent: tool use inside the task container</td></tr> | |
| <tr><td class="mono">Claude Opus 5</td><td class="mono">—</td> | |
| <td>The model, at maximum reasoning effort</td></tr> | |
| </tbody></table></div> | |
| <figcaption><b>Table 2.</b> The adapter is the only part we wrote. Babysitter and Claude | |
| Code are shipped products, used through their public entry point: the user types | |
| <code>/babysitter:yolo <task></code> and the agent takes it from there.</figcaption> | |
| </figure> | |
| <p>The dataset and every task image are pinned by cryptographic digest, and the canonical | |
| copies are never modified; runs execute against ephemeral copies. Trials ran on dedicated | |
| Azure hosts under Docker.</p> | |
| <h3>Time limit</h3> | |
| <p>Leaderboard submissions run at the benchmark's stock per-task time limit. We raised it: | |
| three times stock for the main pass, ten times where a run had been cut off mid-process. An | |
| agent that writes a spec, builds tests and iterates does more work per task than one that | |
| answers directly, and a limit calibrated for the second truncates the first. One task shows | |
| how sharp the edge is: killed at 46.8 minutes it scored 0 of 7 checks; given 50 minutes it | |
| scored 7 of 7.</p> | |
| <h3>The directive</h3> | |
| <p>For the ten tasks the stack had missed, we appended one sentence to the instruction, | |
| after the benchmark's own text, which stays first and verbatim:</p> | |
| <div class="note ok"><p style="font-family:var(--mono);font-size:.875rem;margin:0">use an | |
| iterative adversarial process converging to at least 75% score in at least 3 different | |
| dimensions</p></div> | |
| <p>It names no task, no tool and no answer, only a way of working. It appears in each | |
| trial's recorded transcript, so anyone auditing a run sees exactly what was asked. Five of | |
| the ten tasks passed with it; they are named in Table 3 and marked in the appendix, and the | |
| score without them is 80.90%. We plan to build this behaviour into | |
| Babysitter itself, so it no longer needs to be asked for.</p> | |
| <h3>Fixes that bought nothing</h3> | |
| <p>Two bugs in our own harness were found and fixed before measurement: a native library | |
| that crashed the agent on older task images before it could start, and a client fallback | |
| that turned the provider's refusals into unreadable errors. Neither fix added a single | |
| pass. They corrected the diagnosis, not the score, and we report them because a study that | |
| only mentions the changes that worked is advertising.</p> | |
| <h3>Scoring and cost</h3> | |
| <p>Harbor grades all-or-nothing and counts an unscored trial as zero. Where a task ran more | |
| than once we take its best result. Cost is read from the gateway's own spend counter; | |
| per-trial token accounting is incomplete under this adapter, and deriving cost from it | |
| understates real spend by a wide margin.</p> | |
| </section> | |
| <section id="remaining"> | |
| <h2><span class="num">05</span>What still fails</h2> | |
| <p>12 tasks remain unsolved, and they do not fail for one reason.</p> | |
| <div class='tw'><table><thead><tr><th>Task</th><th class='num'>Baseline</th><th>Outcome</th><th>What the run established</th></tr></thead><tbody><tr><td class='mono'>db-wal-recovery</td><td class='num'>5/7</td><td><span class='tag t-pass'>Recovered</span></td><td>Passed 7/7 in 50 minutes at the raised limit. At the stock-multiple limit it was killed at 46.8 minutes with 0/7.</td></tr><tr><td class='mono'>large-scale-text-editing</td><td class='num'>4/5</td><td><span class='tag t-pass'>Recovered</span></td><td>Froze the spec, wrote an acceptance harness, proved the harness could reject a broken script, then iterated. The final script had the exit line.</td></tr><tr><td class='mono'>nginx-request-logging</td><td class='num'>7/8</td><td><span class='tag t-pass'>Recovered</span></td><td>Passed 8/8, faster than the conventional attempt: 13 minutes against 25.</td></tr><tr><td class='mono'>sanitize-git-repo</td><td class='num'>2/3</td><td><span class='tag t-pass'>Recovered</span></td><td>Passed 3/3. The self-check caught the stray edit the conventional run shipped.</td></tr><tr><td class='mono'>video-processing</td><td class='num'>4/5</td><td><span class='tag t-pass'>Recovered</span></td><td>Passed 5/5. The clock killed the agent at 181 minutes, but the workspace already passed every test; the grader scores final state.</td></tr><tr><td class='mono'>extract-moves-from-video</td><td class='num'>0/2</td><td><span class='tag t-dim'>No effect</span></td><td>Killed by the kernel at 13 minutes. The approach loads the whole decoded video into memory; a streaming approach fits the task's cap.</td></tr><tr><td class='mono'>gcode-to-text</td><td class='num'>1/2</td><td><span class='tag t-dim'>No effect</span></td><td>1/2, finishing well inside the limit. The agent decoded the correct flag but wrapped it in a report; the grader wants the bare string.</td></tr><tr><td class='mono'>pytorch-model-cli</td><td class='num'>5/6</td><td><span class='tag t-dim'>No effect</span></td><td>5/6 in 40 minutes of a 156-minute allowance, identical to the conventional result. Time was not the constraint.</td></tr><tr><td class='mono'>raman-fitting</td><td class='num'>2/3</td><td><span class='tag t-fail'>Regression</span></td><td>1/3, down from 2/3. Both peaks missed tolerance; a numerical accuracy gap the directive does not address.</td></tr><tr><td class='mono'>filter-js-from-html</td><td class='num'>1/2</td><td><span class='tag t-warn'>No verdict</span></td><td>Crashed 32 minutes in with no tokens spent on the failing phase, no refusal and no model fallback. No verdict either way.</td></tr></tbody></table></div> | |
| <figcaption style="margin-top:10px"><b>Table 3.</b> The ten tasks the stack initially | |
| missed and what the orchestrated run established about each. “No effect” means | |
| the run finished well inside its enlarged time budget and still reproduced the conventional | |
| result, so time was not the constraint.</figcaption> | |
| <p style="margin-top:24px">Beyond these, four tasks fail because the model refuses them | |
| outright: it classifies them as offensive security and declines before any work begins. No | |
| orchestration reaches a request the model will not make. The remainder are honest gaps: a | |
| curve fit outside tolerance, a prediction vector that must match exactly, a video decode | |
| that exceeds the task's own memory cap, and one crash we could not attribute even with full | |
| logging.</p> | |
| <div class="note"><div class="t">Where the directive backfires</div> | |
| <p>On the G-code task the agent decoded the flag correctly, then wrapped it in a thorough | |
| analysis report. The grader wanted the bare string. An instruction that rewards converging | |
| on quality from several directions is exactly wrong when the answer must be one line. A | |
| built-in version needs to read the output contract before deciding how much to write.</p> | |
| </div> | |
| </section> | |
| <section id="limitations"> | |
| <h2><span class="num">06</span>Limitations</h2> | |
| <ul> | |
| <li><b>One attempt per configuration.</b> No variance estimate; a task that passed once | |
| might not pass again.</li> | |
| <li><b>Best result per task.</b> Retries went to tasks that had already failed, which pushes | |
| a best-of figure toward its maximum. It is a ceiling, not an expectation.</li> | |
| <li><b>Lifted time limit.</b> Not eligible for leaderboard submission, and comparisons at a | |
| fixed limit partly measure whether the process fits the clock.</li> | |
| <li><b>Appended instruction text</b> on ten tasks, five of which passed. Without them: | |
| 80.90%.</li> | |
| <li><b>No per-task leader data.</b> Published submissions do not expose per-task results, so | |
| no task-level comparison with other entries is possible.</li> | |
| </ul> | |
| </section> | |
| <section id="takeaway"> | |
| <h2><span class="num">07</span>Takeaway</h2> | |
| <p>The score is the headline; the mechanism is the news. Every task recovered here failed | |
| the same way first: a capable model did the hard part, then stopped without checking a | |
| requirement the task had stated outright. A bigger model does not fix that. A place to | |
| write down what “done” means, and a way to test for it before stopping, does. | |
| That is what process as code is for.</p> | |
| <p>Given that ability, the agent froze specs, wrote acceptance tests, proved those tests | |
| could catch mistakes, and iterated until they passed. Nobody showed it how. The cost is | |
| time; the fix is a time budget sized for the work. Next, we fold this working style into | |
| Babysitter itself.</p> | |
| </section> | |
| <section id="appendix"> | |
| <h2><span class="num">A</span>All 89 tasks</h2> | |
| <p>Green solved in a conventional run; accent solved under the directive described in | |
| §04; red unsolved. Hover a failure for its cause.</p> | |
| <div class='grid'><div class='cell' title=''><span class='n'>adaptive-rejection-sampler</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>bn-fit-modify</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>break-filter-js-from-html</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>build-cython-ext</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>build-pmars</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>build-pov-ray</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>caffe-cifar-10</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>cancel-async-tasks</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>chess-best-move</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>circuit-fibsqrt</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>cobol-modernization</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>code-from-image</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>compile-compcert</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>configure-git-webserver</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>constraints-scheduling</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>count-dataset-tokens</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>crack-7z-hash</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>custom-memory-heap-crash</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>db-wal-recovery</span><span class='tag t-dir'>directive</span></div><div class='cell' title=''><span class='n'>distribution-search</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>dna-assembly</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>dna-insert</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>extract-elf</span><span class='tag t-pass'>pass</span></div><div class='cell' title='OOM against the task's declared 2048 MiB cap.'><span class='n'>extract-moves-from-video</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>feal-differential-cryptanalysis</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>feal-linear-cryptanalysis</span><span class='tag t-pass'>pass</span></div><div class='cell' title='Blocked XSS correctly but modified 5 of 12 clean HTML files.'><span class='n'>filter-js-from-html</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>financial-document-processor</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>fix-code-vulnerability</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>fix-git</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>fix-ocaml-gc</span><span class='tag t-pass'>pass</span></div><div class='cell' title='Wrote a prose explanation into the answer file instead of the flag.'><span class='n'>gcode-to-text</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>git-leak-recovery</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>git-multibranch</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>gpt2-codegolf</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>headless-terminal</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>hf-model-inference</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>install-windows-3.11</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>kv-store-grpc</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>large-scale-text-editing</span><span class='tag t-dir'>directive</span></div><div class='cell' title=''><span class='n'>largest-eigenval</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>llm-inference-batching-scheduler</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>log-summary-date-ranges</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>mailman</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>make-doom-for-mips</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>make-mips-interpreter</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>mcmc-sampling-stan</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>merge-diff-arc-agi-task</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>model-extraction-relu-logits</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>modernize-scientific-stack</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>mteb-leaderboard</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>mteb-retrieve</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>multi-source-data-merger</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>nginx-request-logging</span><span class='tag t-dir'>directive</span></div><div class='cell' title=''><span class='n'>openssl-selfsigned-cert</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>overfull-hbox</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>password-recovery</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>path-tracing</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>path-tracing-reverse</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>polyglot-c-py</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>polyglot-rust-c</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>portfolio-optimization</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>protein-assembly</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>prove-plus-comm</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>pypi-server</span><span class='tag t-pass'>pass</span></div><div class='cell' title='CLI predictions did not match the expected classes.'><span class='n'>pytorch-model-cli</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>pytorch-model-recovery</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>qemu-alpine-ssh</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>qemu-startup</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>query-optimize</span><span class='tag t-pass'>pass</span></div><div class='cell' title='Peak fit outside tolerance on gamma and offset.'><span class='n'>raman-fitting</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>regex-chess</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>regex-log</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>reshard-c4-data</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>rstan-to-pystan</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>sam-cell-seg</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>sanitize-git-repo</span><span class='tag t-dir'>directive</span></div><div class='cell' title=''><span class='n'>schemelike-metacircular-eval</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>sparql-university</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>sqlite-db-truncate</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>sqlite-with-gcov</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>torch-pipeline-parallelism</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>torch-tensor-parallelism</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>train-fasttext</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>tune-mjcf</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>video-processing</span><span class='tag t-dir'>directive</span></div><div class='cell' title=''><span class='n'>vulnerable-secret</span><span class='tag t-fail'>fail</span></div><div class='cell' title=''><span class='n'>winning-avg-corewars</span><span class='tag t-pass'>pass</span></div><div class='cell' title=''><span class='n'>write-compressor</span><span class='tag t-pass'>pass</span></div></div> | |
| </section> | |
| <footer> | |
| Generated by <code>scripts/build-paper.py</code> from the datasets under <code>data/</code>; | |
| every figure in this document is computed from them, not written into the prose. | |
| Leaderboard figures were read from the Harbor Hub public API on the date shown. Trial | |
| artefacts, transcripts and verifier output are retained. Babysitter is open source at | |
| <a href="https://github.com/a5c-ai/babysitter">github.com/a5c-ai/babysitter</a>. | |
| </footer> | |
| </main></body></html> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment