cache-break-notifier

extension.mjs

// cache-drop-notifier
//
// In one line: every turn this tells you how many tokens were reused from the prompt
// cache, and warns you when that number suddenly collapses.
//
// Why it matters: every time you send a message, Copilot re-sends your entire
// conversation plus a large block of setup text (its instructions, every tool
// definition, your enabled skills, and so on). That is tens of thousands of tokens
// before you have typed a word. The model provider caches the front of that payload
// so it does not reprocess it each turn. You keep that discount only while the front
// of the payload stays byte-for-byte identical. Toggle a skill or MCP, switch models,
// or change reasoning, and the cached copy no longer matches, so it gets thrown out
// and rebuilt. That rebuild is what this extension calls a "cache drop".
//
// How it decides (kept deliberately simple):
//   1. Take ONE reading per turn: the first real answer from the main model. A single
//      turn can make several model calls behind the scenes (the model calls a tool,
//      then gets called again with the result); we only want the first. Background
//      helpers and tool-driven model calls are skipped, because they run in their own
//      separate conversation with their own cache.
//   2. Compare this turn's reused-token count to last turn's, per model.
//   3. If reuse fell by a large ABSOLUTE amount (DROP_TOKENS or more), warn. Otherwise
//      just show the current reuse so a drop is obvious in context.
//
// It reports the SIZE of the fall, never the cause. From the token counts alone, a
// real cache break and an intentional context shrink look identical, so guessing
// would only mislead.
//
// About the math: we compare two reused-token counts by plain subtraction, not a
// ratio. An earlier version divided reused tokens by total input, but that dips
// whenever you paste a big file or get a long tool result, even though nothing broke.
// Subtracting last turn's reuse from this turn's ignores how big your message was, so
// pasting a file no longer trips a false alarm.

import { joinSession } from "@github/copilot-sdk/extension";

const DROP_TOKENS = 4096; // the only knob: how big a fall in reused tokens is worth a warning

/** @type {Map<string, number>} each model's reused-token count from the previous turn */
const perModel = new Map();

// True only while we are still waiting to read this turn's first real answer.
let acceptingTurnEntry = false;

const session = await joinSession({});
await session.log("cache-drop-notifier active", { ephemeral: true });

// A new turn began: start watching for its first answer.
session.on("assistant.turn_start", () => {
    acceptingTurnEntry = true;
});
// The turn ended: stop watching until the next message.
session.on("assistant.turn_end", () => {
    acceptingTurnEntry = false;
});

session.on("assistant.usage", (event) => {
    const d = event.data ?? {};

    // Is this the turn's first real answer from the MAIN model? Skip background helpers
    // (sub-agents), tool-triggered model calls (mcp-sampling), and any call nested under
    // a tool. Those have their own separate conversation and cache, so their numbers
    // would mislead us. We do not use up the turn's one reading on them.
    const initiator = d.initiator;
    const isMain =
        initiator !== "sub-agent" &&
        initiator !== "mcp-sampling" &&
        d.parentToolCallId == null;
    if (!acceptingTurnEntry || !isMain) return;

    // This is the reading we want. Lock it in so the rest of the turn's calls (tool
    // follow-ups and the like) cannot masquerade as this turn's entry, even if the
    // numbers below turn out to be unusable.
    acceptingTurnEntry = false;

    // If the reused-token count is missing, skip this turn rather than treating
    // "missing" as zero, which would fire a fake alarm. Only this count matters for
    // detection, so a missing input size must never suppress a real drop.
    const cached = d.cacheReadTokens;
    if (!Number.isFinite(cached) || cached < 0) return;

    const model = d.model ?? "unknown";
    const prev = perModel.get(model); // how many tokens were reused last turn on this model

    // A "drop" means reuse fell by a large amount versus last turn on this same model.
    // We measure the fall exactly and claim nothing about why. (We say "drop", not
    // "miss": the counts prove reuse fell, not that the same prefix was re-sent and
    // reprocessed.)
    const isDrop = prev !== undefined && prev - cached >= DROP_TOKENS;

    if (isDrop) {
        // Big fall: warn, and keep it on screen so you do not miss it.
        session.log(
            `Cache drop on ${model}: ${prev.toLocaleString()} \u2192 ${cached.toLocaleString()} ` +
            `tokens reused (down ${(prev - cached).toLocaleString()}).`,
            { level: "warning" }
        ).catch(() => {});
    } else {
        // Normal turn: show how much was reused, then let the line fade.
        session.log(
            `Cache: ${cached.toLocaleString()} tokens reused on ${model}.`,
            { level: "info", ephemeral: true }
        ).catch(() => {});
    }

    // Remember this turn's number so the next turn compares against it. This is also
    // why a drop is reported once and then the baseline heals itself.
    perModel.set(model, cached);
});