scrape-my-messages

scrape-my-messages

#!/usr/bin/env bun
/**
 * Walks Claude Code, opencode, codex, and copilot session storage and
 * extracts every message the user actually typed (not tool output, not
 * system-injected instructions/reminders).
 *
 * Incremental: a state file tracks how far each source has been read
 * (byte offset + mtime for jsonl logs, last-seen timestamp for sqlite
 * sources), so a rerun only fetches what's new since last time. Results
 * accumulate in a cache file; the public output is always fully
 * regenerated (sorted by timestamp) from that cache, so it's identical
 * across reruns with no new data - idempotent either way.
 *
 * Usage: scrape-my-messages [output-path] [--full]
 * Default output: ~/logs/agent-messages.jsonl
 * --full: ignore saved state/cache and rescan everything from scratch.
 */
import { Database } from "bun:sqlite";
import {
  readdirSync,
  mkdirSync,
  writeFileSync,
  renameSync,
  existsSync,
  readFileSync,
  openSync,
  readSync,
  closeSync,
  fstatSync,
} from "node:fs";
import { join, basename } from "node:path";
import { homedir } from "node:os";

const HOME = homedir();
const args = process.argv.slice(2).filter((a) => a !== "--full");
const full = process.argv.includes("--full");
const outPath = args[0] ?? join(HOME, "logs", "agent-messages.jsonl");

const stateDir = join(HOME, ".local", "state", "scrape-my-messages");
const statePath = join(stateDir, "state.json");
const cachePath = join(stateDir, "cache.jsonl");
mkdirSync(stateDir, { recursive: true });

type Msg = {
  id: string;
  source: "claude-code" | "opencode" | "codex" | "copilot";
  timestamp: string; // ISO 8601
  project: string | null;
  session_id: string;
  text: string;
};

type State = {
  files: Record<string, { offset: number; mtime: number }>;
  opencode: Record<string, number>; // db filename -> last time_created (ms) seen
  copilot: string | null; // last timestamp (ISO) seen
};

function loadState(): State {
  if (full || !existsSync(statePath)) return { files: {}, opencode: {}, copilot: null };
  try {
    return JSON.parse(readFileSync(statePath, "utf8"));
  } catch {
    return { files: {}, opencode: {}, copilot: null };
  }
}

function loadCache(): Map<string, Msg> {
  const cache = new Map<string, Msg>();
  if (full || !existsSync(cachePath)) return cache;
  for (const line of readFileSync(cachePath, "utf8").split("\n")) {
    if (!line.trim()) continue;
    try {
      const m = JSON.parse(line) as Msg;
      cache.set(m.id, m);
    } catch {
      // skip corrupt line
    }
  }
  return cache;
}

const state = loadState();
const cache = loadCache();
const startCount = cache.size;

// Dispatch prompts foreman (~/build/foreman/prompts/*.md) sends to worker
// agents. These are machine-rendered from a template, not typed by hand, so
// they don't belong in a log of things the user wrote.
const FOREMAN_DISPATCH_PREFIXES = [
  "You are intervening in a stuck coding task after its previous agent was stopped.",
  "You are salvaging unfinished work, unattended.",
  "You are reviewing another agent's work, unattended.",
  "You are working alone on one task, unattended.",
];

function isForemanDispatch(text: string): boolean {
  return FOREMAN_DISPATCH_PREFIXES.some((p) => text.startsWith(p));
}

function addMsg(m: Msg) {
  if (isForemanDispatch(m.text)) return;
  cache.set(m.id, m);
}

function walk(dir: string, pred: (name: string) => boolean, out: string[] = []): string[] {
  let entries;
  try {
    entries = readdirSync(dir, { withFileTypes: true });
  } catch {
    return out;
  }
  for (const e of entries) {
    const p = join(dir, e.name);
    if (e.isDirectory()) walk(p, pred, out);
    else if (pred(e.name)) out.push(p);
  }
  return out;
}

/** Read only the new complete lines appended to `path` since `prevOffset` bytes. */
function tailNewLines(path: string, prevOffset: number): { lines: string[]; newOffset: number; mtimeMs: number } {
  const fd = openSync(path, "r");
  try {
    const stat = fstatSync(fd);
    let offset = prevOffset;
    if (stat.size < offset) offset = 0; // file was truncated/rotated - restart
    const len = stat.size - offset;
    if (len <= 0) return { lines: [], newOffset: offset, mtimeMs: stat.mtimeMs };
    const buf = Buffer.alloc(len);
    readSync(fd, buf, 0, len, offset);
    const str = buf.toString("utf8");
    const lastNewline = str.lastIndexOf("\n");
    if (lastNewline === -1) return { lines: [], newOffset: offset, mtimeMs: stat.mtimeMs };
    const complete = str.slice(0, lastNewline);
    const consumedBytes = Buffer.byteLength(complete, "utf8") + 1;
    const lines = complete.split("\n").filter((l) => l.trim() !== "");
    return { lines, newOffset: offset + consumedBytes, mtimeMs: stat.mtimeMs };
  } finally {
    closeSync(fd);
  }
}

/** Only reads bytes appended since the last run (skips files whose mtime hasn't changed). */
function forEachNewLine(path: string, fn: (line: string) => void) {
  const prev = state.files[path];
  const stat = fstatSync(openSync(path, "r"));
  if (prev && prev.mtime === stat.mtimeMs) return; // untouched since last run
  const { lines, newOffset, mtimeMs } = tailNewLines(path, prev?.offset ?? 0);
  for (const line of lines) fn(line);
  state.files[path] = { offset: newOffset, mtime: mtimeMs };
}

// ---- Claude Code: ~/.claude/projects/**/*.jsonl ----
const CLAUDE_TYPED_SOURCES = new Set(["typed", "queued", "suggestion_accepted"]);
const LEGACY_SYNTHETIC_RE =
  /^<(local-command-caveat|command-name|command-message|command-args|local-command-stdout|bash-input|bash-stdout|system-reminder|user-prompt-submit-hook)>/;

function scrapeClaudeCode() {
  const dir = join(HOME, ".claude", "projects");
  const files = walk(dir, (n) => n.endsWith(".jsonl"));
  for (const file of files) {
    forEachNewLine(file, (line) => {
      let obj: any;
      try {
        obj = JSON.parse(line);
      } catch {
        return;
      }
      if (obj.type !== "user" || obj.isSidechain === true) return;
      const msg = obj.message;
      if (!msg || msg.role !== "user" || typeof msg.content !== "string") return;
      const text = msg.content.trim();
      if (!text) return;

      if (obj.promptSource !== undefined) {
        if (!CLAUDE_TYPED_SOURCES.has(obj.promptSource)) return;
      } else if (obj.origin?.kind !== undefined) {
        if (obj.origin.kind !== "human") return;
      } else if (LEGACY_SYNTHETIC_RE.test(text)) {
        return;
      }

      addMsg({
        id: obj.uuid ?? `${file}:${obj.timestamp}`,
        source: "claude-code",
        timestamp: obj.timestamp,
        project: obj.cwd ?? null,
        session_id: obj.sessionId ?? basename(file, ".jsonl"),
        text,
      });
    });
  }
}

// ---- opencode: sqlite dbs under ~/.local/share/opencode ----
function scrapeOpencode() {
  const dir = join(HOME, ".local", "share", "opencode");
  let entries: string[];
  try {
    entries = readdirSync(dir).filter((f) => f.endsWith(".db"));
  } catch {
    entries = [];
  }
  for (const dbFile of entries) {
    const dbPath = join(dir, dbFile);
    let db: Database;
    try {
      db = new Database(dbPath, { readonly: true });
    } catch {
      continue;
    }
    try {
      const since = state.opencode[dbFile] ?? 0;
      const sessionCache = new Map<string, string | null>();
      const getSessionDir = (sid: string): string | null => {
        if (sessionCache.has(sid)) return sessionCache.get(sid)!;
        const row = db.query("SELECT directory FROM session WHERE id = ?").get(sid) as
          | { directory: string }
          | null;
        const d = row?.directory ?? null;
        sessionCache.set(sid, d);
        return d;
      };

      const userMessages = db
        .query(
          "SELECT id, session_id, time_created, data FROM message WHERE json_extract(data, '$.role') = 'user' AND time_created > ?",
        )
        .all(since) as { id: string; session_id: string; time_created: number; data: string }[];

      const partStmt = db.query("SELECT data FROM part WHERE message_id = ? ORDER BY id");
      let maxSeen = since;

      for (const m of userMessages) {
        maxSeen = Math.max(maxSeen, m.time_created);
        const parts = partStmt.all(m.id) as { data: string }[];
        const texts: string[] = [];
        for (const p of parts) {
          let pd: any;
          try {
            pd = JSON.parse(p.data);
          } catch {
            continue;
          }
          if (pd.type === "text" && typeof pd.text === "string" && pd.text.trim()) {
            texts.push(pd.text.trim());
          }
        }
        const text = texts.join("\n\n").trim();
        if (!text) continue;

        addMsg({
          id: `${dbFile}:${m.id}`,
          source: "opencode",
          timestamp: new Date(m.time_created).toISOString(),
          project: getSessionDir(m.session_id),
          session_id: m.session_id,
          text,
        });
      }
      state.opencode[dbFile] = maxSeen;
    } finally {
      db.close();
    }
  }
}

// ---- codex: ~/.codex/sessions/**/*.jsonl ----
const CODEX_SYNTHETIC_TAG_RE = /^<[A-Za-z_]+>/;
const CODEX_AGENTS_MD_RE = /^# .*\.md instructions for/;

function scrapeCodex() {
  const dir = join(HOME, ".codex", "sessions");
  const files = walk(dir, (n) => n.endsWith(".jsonl"));
  for (const file of files) {
    // session_meta is always the first line of the file; if we've already
    // read past it before, recover id/cwd from any cached message from this file.
    let sessionId: string | null = null;
    let cwd: string | null = null;
    forEachNewLine(file, (line) => {
      let obj: any;
      try {
        obj = JSON.parse(line);
      } catch {
        return;
      }
      if (obj.type === "session_meta") {
        sessionId = obj.payload?.id ?? sessionId;
        cwd = obj.payload?.cwd ?? cwd;
        return;
      }
      if (obj.type !== "response_item") return;
      const payload = obj.payload;
      if (!payload || payload.type !== "message" || payload.role !== "user") return;
      const content = Array.isArray(payload.content) ? payload.content : [];
      const text = content
        .map((c: any) => (typeof c.text === "string" ? c.text : ""))
        .join("\n\n")
        .trim();
      if (!text) return;
      if (CODEX_SYNTHETIC_TAG_RE.test(text) || CODEX_AGENTS_MD_RE.test(text)) return;

      addMsg({
        id: `${file}:${obj.timestamp}`,
        source: "codex",
        timestamp: obj.timestamp,
        project: cwd,
        session_id: sessionId ?? basename(file, ".jsonl"),
        text,
      });
    });
  }
}

// ---- copilot: ~/.copilot/session-store.db ----
function scrapeCopilot() {
  const dbPath = join(HOME, ".copilot", "session-store.db");
  let db: Database;
  try {
    db = new Database(dbPath, { readonly: true });
  } catch {
    return;
  }
  try {
    const since = state.copilot ?? "0000-00-00";
    const rows = db
      .query(
        `SELECT turns.id as id, turns.session_id as session_id, turns.user_message as text,
                turns.timestamp as timestamp, sessions.cwd as cwd
         FROM turns JOIN sessions ON turns.session_id = sessions.id
         WHERE user_message IS NOT NULL AND trim(user_message) != '' AND timestamp > ?`,
      )
      .all(since) as { id: number; session_id: string; text: string; timestamp: string; cwd: string | null }[];

    let maxSeen = since;
    for (const r of rows) {
      if (r.timestamp > maxSeen) maxSeen = r.timestamp;
      addMsg({
        id: `copilot:${r.id}`,
        source: "copilot",
        timestamp: new Date(r.timestamp).toISOString(),
        project: r.cwd,
        session_id: r.session_id,
        text: r.text.trim(),
      });
    }
    state.copilot = maxSeen;
  } finally {
    db.close();
  }
}

scrapeClaudeCode();
scrapeOpencode();
scrapeCodex();
scrapeCopilot();

const messages = [...cache.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp));

function atomicWrite(path: string, content: string) {
  mkdirSync(join(path, ".."), { recursive: true });
  const tmp = `${path}.tmp-${process.pid}`;
  writeFileSync(tmp, content);
  renameSync(tmp, path);
}

atomicWrite(cachePath, messages.map((m) => JSON.stringify(m)).join("\n") + "\n");
atomicWrite(outPath, messages.map((m) => JSON.stringify(m)).join("\n") + "\n");
atomicWrite(statePath, JSON.stringify(state, null, 2));

const bySource = messages.reduce<Record<string, number>>((acc, m) => {
  acc[m.source] = (acc[m.source] ?? 0) + 1;
  return acc;
}, {});
console.error(`${messages.length} total messages (${messages.length - startCount} new) written to ${outPath}`);
console.error(bySource);