map-reduce-conversations

summarize.py

#!/usr/bin/env python3
"""
Map-reduce summarization of conversation chunks using Claude Sonnet.

Usage:
    python summarize.py              # run full pipeline
    python summarize.py --map-only   # run map phase only
    python summarize.py --reduce-only # run reduce phase only (expects summaries/ to exist)
"""

import anthropic
import os
import sys
import json
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed

MODEL = "claude-sonnet-4-6"
CHUNKS_DIR = Path("chunks_80k")
SUMMARIES_DIR = Path("summaries")
OUTPUT_FILE = Path("dani_summary.md")

MAP_PROMPT = """\
You are reading a batch of conversations between Dani and Claude (an AI assistant).
Your task: extract everything these conversations reveal about Dani as a person.
Go deep. Be honest. Don't flatter.

EXTRACT ALONG THESE DIMENSIONS:

**Identity** — who Dani is. Gender, pronouns, queerness, neurodivergence, nationality, \
languages, where they live, where they study, where they work. Anything they say about \
how they see themselves or want to be seen.

**Preferences & opinions** — what Dani likes, dislikes, gravitates toward, avoids. \
Tools, aesthetics, workflows, philosophies. What do they choose when they have a choice? \
What do they complain about? What makes them say "NICE"?

**Code style** — how Dani writes code and what they value in code. Naming conventions, \
architecture preferences, frameworks, languages. Do they prefer explicit or magic? \
Simple or powerful? What patterns do they reach for? What do they reject?

**Writing & communication style** — how Dani writes messages. Typos, abbreviations, \
tone, formality level, emoticons, language mixing. How do they give instructions? \
How do they react to suggestions? How terse or verbose are they?

**As a person** — temperament, energy, emotional patterns visible in the conversations. \
How do they handle frustration? What excites them? How do they relate to others? \
Do they plan or improvise? Are they careful or reckless? Patient or impatient?

**Interests & rabbit holes** — what topics pull Dani in beyond immediate work needs. \
Side projects, curiosities, things they explore for fun. What do they nerd out about?

**Problems & struggles** — where Dani gets stuck, what they find hard, what frustrates \
them. Technical struggles, workflow friction, emotional difficulty, patterns of avoidance \
or overcommitment. Be specific and honest — this is for self-understanding, not judgment.

**Projects & work** — what Dani is building, studying, employed to do. Stage, stack, \
context. What role do they play? How do they talk about their work?

HOW TO EXTRACT:
- Cite or closely paraphrase Dani's own words when possible. Quotes are gold.
- Label evidence strength: [stated] for direct statements, [demonstrated] for things \
shown through behavior, [inferred] for reasonable conclusions from patterns.
- Extract from what Dani writes and asks — not from Claude's responses.
- Even a bare technical question reveals something: what the topic is, that they needed \
help, how they asked. Note the signal, skip the tutorial content.
- A conversation where Dani asks Claude to "make it more blaming" then "a little less \
blaming" tells you something about how they think and iterate.
- Group by theme, not by conversation.
- Don't soften, don't hedge, don't add "however" to balance negatives. Just describe \
what you see.
"""

REDUCE_PROMPT = """\
You have 10 partial profiles of Dani, each extracted from a different batch of their \
conversations with an AI assistant over ~6 months (Sep 2025 - Mar 2026).

Merge them into one deep, honest profile. This is for Dani's own self-understanding. \
It should read like something Dani would look at and say "yeah, that's me" — including \
the uncomfortable parts.

Write in third person ("Dani is...", "Dani tends to...").

GUIDELINES:
- Go deep on identity, preferences, style, personality, and struggles. These matter \
more than listing technical skills.
- Lead with what's most distinctive about Dani, not what's most common.
- Preserve specific quotes and details — "I like being referred to as an object" is \
more valuable than "uses non-standard pronouns."
- Where multiple sources confirm something, state it with confidence and combine the \
best evidence.
- Where something appears only once but is [stated], include it.
- Where something is [inferred], include it only if it forms a pattern across sources.
- Don't moralize, psychoanalyze, or editorialize. Describe behavior and let it speak.
- Don't soften problems or struggles. If Dani gets distracted, overcommits, rushes, \
gets frustrated — say so plainly with evidence.
- Don't add disclaimers like "it should be noted that this is normal" or "many people \
struggle with this." Just describe Dani.
- Let structure emerge from content. Use headings that fit what's actually there, \
not a generic HR template.
- Be comprehensive. Include everything that's real. No length limit — completeness \
over brevity.
"""

client = anthropic.Anthropic()


def map_chunk(chunk_path: Path) -> str:
    """Summarize a single chunk."""
    text = chunk_path.read_text()
    print(f"  Processing {chunk_path.name} ({len(text):,} chars)...")

    response = client.messages.create(
        model=MODEL,
        max_tokens=4096,
        messages=[
            {"role": "user", "content": f"{MAP_PROMPT}\n\n---\n\n{text}"}
        ],
    )

    summary = response.content[0].text
    usage = response.usage
    print(f"  Done {chunk_path.name}: {usage.input_tokens:,} in / {usage.output_tokens:,} out")
    return summary


def map_phase():
    """Run map phase: summarize each chunk in parallel."""
    SUMMARIES_DIR.mkdir(exist_ok=True)

    chunk_files = sorted(CHUNKS_DIR.glob("chunk_*.txt"))
    print(f"Map phase: {len(chunk_files)} chunks\n")

    total_input = 0
    total_output = 0

    # Process up to 5 in parallel
    with ThreadPoolExecutor(max_workers=5) as executor:
        futures = {
            executor.submit(map_chunk, f): f for f in chunk_files
        }
        for future in as_completed(futures):
            chunk_file = futures[future]
            summary = future.result()
            out_path = SUMMARIES_DIR / f"{chunk_file.stem}_summary.md"
            out_path.write_text(summary)

    print(f"\nMap phase complete. Summaries in {SUMMARIES_DIR}/")


def reduce_phase():
    """Run reduce phase: merge all summaries into one."""
    summary_files = sorted(SUMMARIES_DIR.glob("chunk_*_summary.md"))
    print(f"Reduce phase: merging {len(summary_files)} summaries\n")

    combined = ""
    for f in summary_files:
        combined += f"--- Summary from {f.stem} ---\n\n{f.read_text()}\n\n"

    response = client.messages.create(
        model=MODEL,
        max_tokens=8192,
        messages=[
            {"role": "user", "content": f"{REDUCE_PROMPT}\n\n---\n\n{combined}"}
        ],
    )

    result = response.content[0].text
    usage = response.usage
    print(f"Reduce: {usage.input_tokens:,} in / {usage.output_tokens:,} out")

    OUTPUT_FILE.write_text(result)
    print(f"\nFinal profile written to {OUTPUT_FILE}")


if __name__ == "__main__":
    args = set(sys.argv[1:])

    if "--reduce-only" in args:
        reduce_phase()
    elif "--map-only" in args:
        map_phase()
    else:
        map_phase()
        reduce_phase()