mizchi · July 17, 2026 18:57 · mizchi · Jul 17, 2026 · mizchi · Jul 17, 2026
diff --git a/README.md b/README.md
diff --git a/annotation-protocol-ja.md b/annotation-protocol-ja.md
diff --git a/blog-article-ja.md b/blog-article-ja.md
diff --git a/blog-revision-ja.md b/blog-revision-ja.md
diff --git a/Cargo.toml b/Cargo.toml
 [package]
 name = "cognitive-rhythm-eval"
 version = "0.1.0"
 edition = "2024"
 rust-version = "1.85"
 description = "A measurable proxy evaluator for Japanese cognitive writing rhythm"
 license = "MIT"

 [dependencies]
 encoding_rs = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] }
 scraper = "0.24"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.149"
 sha2 = "0.10"
 vibrato = { version = "=0.5.2", default-features = false }
diff --git a/cognitive-rhythm-writing-eval-ja.md b/cognitive-rhythm-writing-eval-ja.md
diff --git a/cognitive-rhythm-writing-with-eval-ja.md b/cognitive-rhythm-writing-with-eval-ja.md
diff --git a/considerations-ja.md b/considerations-ja.md
diff --git a/corpus.rs b/corpus.rs
 //! Reproducible source manifests, extraction, and metric-selection diagnostics.

 use std::collections::{BTreeMap, BTreeSet};
 use std::fmt;
 use std::fs;
 use std::io::Write;
 use std::path::Path;

 use encoding_rs::SHIFT_JIS;
 use reqwest::blocking::Client;
 use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};

 use crate::{Evaluation, LabeledText, ScoredText};

 /// A reproducible reference to a source document. The manifest stores metadata
 /// and URLs, not third-party source text.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct CorpusSource {
    pub id: String,
    pub url: String,
    pub label: String,
    pub author: String,
    pub domain: String,
    pub split_group: String,
    pub selector: String,
    pub encoding: String,
    pub license: String,
 }

 #[derive(Debug)]
 pub enum CorpusError {
    Http(reqwest::Error),
    Io(std::io::Error),
    Json(serde_json::Error),
    InvalidSelector { source_id: String, selector: String },
    MissingContent { source_id: String, selector: String },
    UnsupportedEncoding(String),
    InvalidUtf8 { source_id: String },
 }

 impl fmt::Display for CorpusError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Http(error) => write!(formatter, "HTTP request failed: {error}"),
            Self::Io(error) => write!(formatter, "I/O failed: {error}"),
            Self::Json(error) => write!(formatter, "JSON parsing failed: {error}"),
            Self::InvalidSelector {
                source_id,
                selector,
            } => write!(formatter, "invalid selector {selector:?} for {source_id}"),
            Self::MissingContent {
                source_id,
                selector,
            } => write!(
                formatter,
                "selector {selector:?} found no content for {source_id}"
            ),
            Self::UnsupportedEncoding(encoding) => {
                write!(formatter, "unsupported source encoding: {encoding}")
            }
            Self::InvalidUtf8 { source_id } => write!(formatter, "invalid UTF-8 from {source_id}"),
        }
    }
 }

 impl std::error::Error for CorpusError {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Http(error) => Some(error),
            Self::Io(error) => Some(error),
            Self::Json(error) => Some(error),
            Self::InvalidSelector { .. }
            | Self::MissingContent { .. }
            | Self::UnsupportedEncoding(_)
            | Self::InvalidUtf8 { .. } => None,
        }
    }
 }

 impl From<reqwest::Error> for CorpusError {
    fn from(error: reqwest::Error) -> Self {
        Self::Http(error)
    }
 }

 impl From<std::io::Error> for CorpusError {
    fn from(error: std::io::Error) -> Self {
        Self::Io(error)
    }
 }

 impl From<serde_json::Error> for CorpusError {
    fn from(error: serde_json::Error) -> Self {
        Self::Json(error)
    }
 }

 /// Reads a version-controlled source manifest in JSON Lines format.
 pub fn load_sources(path: impl AsRef<Path>) -> Result<Vec<CorpusSource>, CorpusError> {
    fs::read_to_string(path)?
        .lines()
        .filter(|line| !line.trim().is_empty())
        .map(serde_json::from_str)
        .collect::<Result<Vec<_>, _>>()
        .map_err(CorpusError::from)
 }

 /// Fetches all manifest sources and turns their selected article bodies into
 /// passage records. Generated text should be stored only in ignored local data
 /// directories unless redistribution is permitted by the source license.
 pub fn fetch_passages(sources: &[CorpusSource]) -> Result<Vec<LabeledText>, CorpusError> {
    let client = Client::builder()
        .user_agent("cognitive-rhythm-eval/0.1 (research corpus builder)")
        .build()?;
    let mut passages = Vec::new();

    for source in sources {
        let response = client.get(&source.url).send()?.error_for_status()?;
        let bytes = response.bytes()?;
        let html = decode_html(source, &bytes)?;
        passages.extend(extract_passages_from_html(source, &html, 600, 2_400)?);
    }
    Ok(passages)
 }

 /// Extracts article text from a cached HTML string, allowing selector behavior
 /// to be tested without network access.
 pub fn extract_passages_from_html(
    source: &CorpusSource,
    html: &str,
    minimum_characters: usize,
    maximum_characters: usize,
 ) -> Result<Vec<LabeledText>, CorpusError> {
    let selector = Selector::parse(&source.selector).map_err(|_| CorpusError::InvalidSelector {
        source_id: source.id.clone(),
        selector: source.selector.clone(),
    })?;
    let document = Html::parse_document(html);
    let root = document
        .select(&selector)
        .next()
        .ok_or_else(|| CorpusError::MissingContent {
            source_id: source.id.clone(),
            selector: source.selector.clone(),
        })?;
    let paragraphs = html_fragment_to_paragraphs(&root.inner_html());
    let windows = passage_windows(&paragraphs, minimum_characters, maximum_characters);
    let source_hash = format!("{:x}", Sha256::digest(html.as_bytes()));

    Ok(windows
        .into_iter()
        .enumerate()
        .map(|(index, text)| LabeledText {
            id: format!("{}-{index:03}", source.id),
            label: source.label.clone(),
            source_id: source.id.clone(),
            author: source.author.clone(),
            source_url: source.url.clone(),
            split_group: source.split_group.clone(),
            domain: source.domain.clone(),
            source_hash: source_hash.clone(),
            text,
        })
        .collect())
 }

 /// Writes generated passages as JSON Lines, creating its parent directory.
 pub fn write_passages(path: impl AsRef<Path>, passages: &[LabeledText]) -> Result<(), CorpusError> {
    let path = path.as_ref();
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let file = fs::File::create(path)?;
    let mut writer = std::io::BufWriter::new(file);
    for passage in passages {
        serde_json::to_writer(&mut writer, passage)?;
        writer.write_all(b"\n")?;
    }
    Ok(())
 }

 /// A feature ranked by separation of `good_hypothesis` and `bad_hypothesis`.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct MetricDiagnostic {
    pub name: String,
    pub direction: String,
    /// Direction-adjusted pairwise ROC-AUC. 0.5 means no separation.
    pub auc: f64,
    /// Mean AUC on held-out good/bad document pairs. Feature direction is
    /// selected from all other source documents in each fold.
    pub document_holdout_auc: Option<f64>,
    pub document_holdout_folds: usize,
    /// Mean AUC on held-out good/bad author pairs. `None` when either label
    /// does not yet contain at least two distinct authors.
    pub author_holdout_auc: Option<f64>,
    pub author_holdout_folds: usize,
    /// Direction-adjusted AUC after averaging every passage in a source
    /// document. Unlike `auc`, every source document has equal weight.
    pub document_auc: Option<f64>,
    /// Two-sided exact permutation p-value for the difference between the
    /// good/bad document means. `None` means there are too many documents for
    /// an exact enumeration.
    pub document_permutation_p_value: Option<f64>,
    pub good_mean: f64,
    pub bad_mean: f64,
 }

 /// Separates apparent feature separation from whether the corpus can validate it.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct MetricSelectionReport {
    pub good_count: usize,
    pub bad_count: usize,
    pub document_holdout_ready: bool,
    /// True only when both labels contain at least two distinct non-empty
    /// authors. This is a prerequisite for an author-held-out validation.
    pub author_holdout_ready: bool,
    pub domain_confounded: bool,
    pub shared_domains: Vec<String>,
    pub metrics: Vec<MetricDiagnostic>,
 }

 /// One blind human rating for a corpus passage.
 ///
 /// `rhythm_1_to_5` is intentionally independent of the hypothesis label: 1 is
 /// least rhythmically effective for the assigned reading purpose, and 5 is
 /// most effective.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct HumanAnnotation {
    pub passage_id: String,
    pub annotator_id: String,
    pub rhythm_1_to_5: f64,
 }

 impl HumanAnnotation {
    #[must_use]
    pub fn new(
        passage_id: impl Into<String>,
        annotator_id: impl Into<String>,
        rhythm_1_to_5: f64,
    ) -> Self {
        Self {
            passage_id: passage_id.into(),
            annotator_id: annotator_id.into(),
            rhythm_1_to_5,
        }
    }
 }

 /// A raw, signed association between one proxy feature and mean human rhythm
 /// ratings. Positive values mean the feature increases with the rating.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct HumanMetricCorrelation {
    pub name: String,
    pub spearman_rho: Option<f64>,
    pub sample_count: usize,
 }

 /// Aggregate of blinded human rhythm ratings and their feature associations.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct HumanRhythmReport {
    pub annotated_passage_count: usize,
    pub rating_count: usize,
    pub annotator_count: usize,
    pub metrics: Vec<HumanMetricCorrelation>,
 }

 #[derive(Debug)]
 pub enum MetricSelectionError {
    MissingGoodExamples,
    MissingBadExamples,
 }

 impl fmt::Display for MetricSelectionError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::MissingGoodExamples => write!(formatter, "no good_hypothesis records found"),
            Self::MissingBadExamples => write!(formatter, "no bad_hypothesis records found"),
        }
    }
 }

 impl std::error::Error for MetricSelectionError {}

 #[derive(Debug)]
 pub enum HumanRhythmError {
    UnknownPassage {
        passage_id: String,
    },
    DuplicateRating {
        passage_id: String,
        annotator_id: String,
    },
    InvalidRhythmRating {
        passage_id: String,
        value: f64,
    },
 }

 impl fmt::Display for HumanRhythmError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::UnknownPassage { passage_id } => {
                write!(
                    formatter,
                    "annotation refers to unknown passage {passage_id}"
                )
            }
            Self::DuplicateRating {
                passage_id,
                annotator_id,
            } => write!(
                formatter,
                "duplicate annotation from {annotator_id} for passage {passage_id}"
            ),
            Self::InvalidRhythmRating { passage_id, value } => write!(
                formatter,
                "rhythm rating for {passage_id} must be finite and in 1..=5, got {value}"
            ),
        }
    }
 }

 impl std::error::Error for HumanRhythmError {}

 /// Ranks surface metrics by direction-adjusted pairwise AUC.
 ///
 /// The report does not train a model. A domain-confounded corpus cannot support
 /// a general claim about writing quality even if the apparent AUC is high.
 pub fn select_metrics(
    records: &[ScoredText],
 ) -> Result<MetricSelectionReport, MetricSelectionError> {
    let good: Vec<_> = records
        .iter()
        .filter(|record| record.label == "good_hypothesis")
        .collect();
    let bad: Vec<_> = records
        .iter()
        .filter(|record| record.label == "bad_hypothesis")
        .collect();
    if good.is_empty() {
        return Err(MetricSelectionError::MissingGoodExamples);
    }
    if bad.is_empty() {
        return Err(MetricSelectionError::MissingBadExamples);
    }

    let mut metrics = metric_names()
        .into_iter()
        .map(|name| metric_diagnostic(name, &good, &bad))
        .collect::<Vec<_>>();
    metrics.sort_by(|left, right| {
        right
            .auc
            .total_cmp(&left.auc)
            .then_with(|| left.name.cmp(&right.name))
    });

    let good_domains = good
        .iter()
        .map(|record| record.domain.clone())
        .collect::<BTreeSet<_>>();
    let bad_domains = bad
        .iter()
        .map(|record| record.domain.clone())
        .collect::<BTreeSet<_>>();
    let shared_domains = good_domains
        .intersection(&bad_domains)
        .cloned()
        .collect::<Vec<_>>();
    let document_holdout_ready =
        distinct_split_groups(&good) >= 2 && distinct_split_groups(&bad) >= 2;
    let author_holdout_ready = distinct_authors(&good) >= 2 && distinct_authors(&bad) >= 2;

    Ok(MetricSelectionReport {
        good_count: good.len(),
        bad_count: bad.len(),
        document_holdout_ready,
        author_holdout_ready,
        domain_confounded: shared_domains.is_empty(),
        shared_domains,
        metrics,
    })
 }

 /// Correlates surface proxy metrics with the mean of independently collected,
 /// blinded human rhythm ratings. Hypothesis labels are not consulted.
 pub fn correlate_human_rhythm(
    records: &[ScoredText],
    annotations: &[HumanAnnotation],
 ) -> Result<HumanRhythmReport, HumanRhythmError> {
    let records_by_id = records
        .iter()
        .map(|record| (record.id.as_str(), record))
        .collect::<BTreeMap<_, _>>();
    let mut ratings_by_passage = BTreeMap::<String, (f64, usize)>::new();
    let mut seen_ratings = BTreeSet::<(String, String)>::new();
    let mut annotators = BTreeSet::<String>::new();

    for annotation in annotations {
        if !(annotation.rhythm_1_to_5.is_finite()
            && (1.0..=5.0).contains(&annotation.rhythm_1_to_5))
        {
            return Err(HumanRhythmError::InvalidRhythmRating {
                passage_id: annotation.passage_id.clone(),
                value: annotation.rhythm_1_to_5,
            });
        }
        if !records_by_id.contains_key(annotation.passage_id.as_str()) {
            return Err(HumanRhythmError::UnknownPassage {
                passage_id: annotation.passage_id.clone(),
            });
        }
        let rating_key = (
            annotation.passage_id.clone(),
            annotation.annotator_id.clone(),
        );
        if !seen_ratings.insert(rating_key) {
            return Err(HumanRhythmError::DuplicateRating {
                passage_id: annotation.passage_id.clone(),
                annotator_id: annotation.annotator_id.clone(),
            });
        }

        let aggregate = ratings_by_passage
            .entry(annotation.passage_id.clone())
            .or_default();
        aggregate.0 += annotation.rhythm_1_to_5;
        aggregate.1 += 1;
        annotators.insert(annotation.annotator_id.clone());
    }

    let metric_pairs = metric_names()
        .into_iter()
        .map(|name| {
            let pairs = records
                .iter()
                .filter_map(|record| {
                    ratings_by_passage.get(&record.id).map(|(total, count)| {
                        (
                            metric_value(&record.evaluation, name),
                            total / *count as f64,
                        )
                    })
                })
                .collect::<Vec<_>>();
            let (feature_values, human_values): (Vec<_>, Vec<_>) = pairs.into_iter().unzip();
            HumanMetricCorrelation {
                name: name.to_owned(),
                spearman_rho: spearman_rho(&feature_values, &human_values),
                sample_count: feature_values.len(),
            }
        })
        .collect::<Vec<_>>();
    let mut metrics = metric_pairs;
    metrics.sort_by(|left, right| {
        right
            .spearman_rho
            .map(f64::abs)
            .unwrap_or_default()
            .total_cmp(&left.spearman_rho.map(f64::abs).unwrap_or_default())
            .then_with(|| left.name.cmp(&right.name))
    });

    Ok(HumanRhythmReport {
        annotated_passage_count: ratings_by_passage.len(),
        rating_count: annotations.len(),
        annotator_count: annotators.len(),
        metrics,
    })
 }

 fn spearman_rho(left: &[f64], right: &[f64]) -> Option<f64> {
    if left.len() != right.len() || left.len() < 2 {
        return None;
    }
    let left_ranks = average_ranks(left);
    let right_ranks = average_ranks(right);
    let left_mean = mean(&left_ranks);
    let right_mean = mean(&right_ranks);
    let covariance = left_ranks
        .iter()
        .zip(&right_ranks)
        .map(|(left, right)| (left - left_mean) * (right - right_mean))
        .sum::<f64>();
    let left_sum_of_squares = left_ranks
        .iter()
        .map(|value| (value - left_mean).powi(2))
        .sum::<f64>();
    let right_sum_of_squares = right_ranks
        .iter()
        .map(|value| (value - right_mean).powi(2))
        .sum::<f64>();
    let denominator = (left_sum_of_squares * right_sum_of_squares).sqrt();
    (denominator > f64::EPSILON).then_some(covariance / denominator)
 }

 fn average_ranks(values: &[f64]) -> Vec<f64> {
    let mut indexes = (0..values.len()).collect::<Vec<_>>();
    indexes.sort_by(|left, right| values[*left].total_cmp(&values[*right]));

    let mut ranks = vec![0.0; values.len()];
    let mut start = 0;
    while start < indexes.len() {
        let mut end = start + 1;
        while end < indexes.len()
            && values[indexes[start]].total_cmp(&values[indexes[end]]) == std::cmp::Ordering::Equal
        {
            end += 1;
        }
        let average_rank = (start + 1 + end) as f64 / 2.0;
        for index in &indexes[start..end] {
            ranks[*index] = average_rank;
        }
        start = end;
    }
    ranks
 }

 fn decode_html(source: &CorpusSource, bytes: &[u8]) -> Result<String, CorpusError> {
    match source.encoding.to_ascii_lowercase().as_str() {
        "utf-8" | "utf8" => {
            String::from_utf8(bytes.to_vec()).map_err(|_| CorpusError::InvalidUtf8 {
                source_id: source.id.clone(),
            })
        }
        "shift_jis" | "shift-jis" | "sjis" => {
            let (decoded, _, _) = SHIFT_JIS.decode(bytes);
            Ok(decoded.into_owned())
        }
        encoding => Err(CorpusError::UnsupportedEncoding(encoding.to_owned())),
    }
 }

 fn html_fragment_to_paragraphs(fragment: &str) -> Vec<String> {
    let with_breaks = fragment
        .replace("<br />", "\n")
        .replace("<br/>", "\n")
        .replace("<br>", "\n")
        .replace("</p>", "\n\n")
        .replace("</div>", "\n\n")
        .replace("</h1>", "\n\n")
        .replace("</h2>", "\n\n")
        .replace("</h3>", "\n\n")
        .replace("</h4>", "\n\n");
    let fragment = Html::parse_fragment(&with_breaks);
    let text = fragment.root_element().text().collect::<Vec<_>>().join("");

    text.lines()
        .map(normalize_whitespace)
        .filter(|line| line.chars().count() >= 20)
        .collect()
 }

 fn normalize_whitespace(text: &str) -> String {
    text.split_whitespace().collect::<Vec<_>>().join(" ")
 }

 fn passage_windows(paragraphs: &[String], minimum: usize, maximum: usize) -> Vec<String> {
    let mut passages = Vec::new();
    let mut current = String::new();

    for paragraph in paragraphs {
        let additional_length = if current.is_empty() { 0 } else { 2 } + paragraph.len();
        if !current.is_empty() && current.len() + additional_length > maximum {
            if current.chars().count() >= minimum {
                passages.push(std::mem::take(&mut current));
            } else {
                current.clear();
            }
        }
        if !current.is_empty() {
            current.push_str("\n\n");
        }
        current.push_str(paragraph);

        if paragraph.chars().count() > maximum && current.chars().count() >= minimum {
            passages.push(std::mem::take(&mut current));
        }
    }
    if current.chars().count() >= minimum {
        passages.push(current);
    }
    passages
 }

 fn distinct_split_groups(records: &[&ScoredText]) -> usize {
    group_keys(records, split_group_key).len()
 }

 fn distinct_authors(records: &[&ScoredText]) -> usize {
    let records = records
        .iter()
        .copied()
        .filter(|record| !author_key(record).is_empty())
        .collect::<Vec<_>>();
    group_keys(&records, author_key).len()
 }

 fn group_keys(records: &[&ScoredText], group_key: fn(&ScoredText) -> &str) -> BTreeSet<String> {
    records
        .iter()
        .map(|record| group_key(record).to_owned())
        .collect()
 }

 fn split_group_key(record: &ScoredText) -> &str {
    if record.split_group.is_empty() {
        &record.id
    } else {
        &record.split_group
    }
 }

 fn author_key(record: &ScoredText) -> &str {
    record.author.trim()
 }

 fn metric_names() -> Vec<&'static str> {
    vec![
        "rhythm_score",
        "document_update_risk",
        "sentence_length_cv",
        "short_long_short_rate",
        "length_contrast_rate",
        "comma_per_sentence",
        "question_per_sentence",
        "content_token_ratio",
        "hesitation_signal_rate",
        "document_signal_rate",
        "short_terminal_rate",
        "pos_ratio:名詞",
        "pos_ratio:動詞",
        "pos_ratio:形容詞",
        "pos_ratio:副詞",
        "pos_ratio:連体詞",
        "pos_ratio:助詞",
        "pos_ratio:助動詞",
        "pos_ratio:接続詞",
        "pos_bigram_entropy",
        "pos_bigram_repetition_rate",
        "same_pos_transition_rate",
        "mean_bunsetsu_candidate_tokens",
        "bunsetsu_candidate_length_cv",
        "conjunction_token_rate",
        "sentence_length_dominant_power_ratio",
        "lexical_topic_shift_mean",
        "lexical_topic_shift_dominant_power_ratio",
    ]
 }

 fn metric_diagnostic(name: &str, good: &[&ScoredText], bad: &[&ScoredText]) -> MetricDiagnostic {
    let good_values = good
        .iter()
        .map(|record| metric_value(&record.evaluation, name))
        .collect::<Vec<_>>();
    let bad_values = bad
        .iter()
        .map(|record| metric_value(&record.evaluation, name))
        .collect::<Vec<_>>();
    let higher_is_good_auc = pairwise_auc(&good_values, &bad_values);
    let (direction, auc) = if higher_is_good_auc >= 0.5 {
        ("higher_is_good", higher_is_good_auc)
    } else {
        ("lower_is_good", 1.0 - higher_is_good_auc)
    };
    let document_holdout = document_holdout_auc(name, good, bad);
    let author_holdout = author_holdout_auc(name, good, bad);
    let good_document_values = document_metric_values(name, good);
    let bad_document_values = document_metric_values(name, bad);
    let document_higher_is_good_auc = pairwise_auc(&good_document_values, &bad_document_values);
    let document_auc = if document_higher_is_good_auc >= 0.5 {
        document_higher_is_good_auc
    } else {
        1.0 - document_higher_is_good_auc
    };

    MetricDiagnostic {
        name: name.to_owned(),
        direction: direction.to_owned(),
        auc,
        document_holdout_auc: document_holdout.map(|(auc, _)| auc),
        document_holdout_folds: document_holdout.map_or(0, |(_, folds)| folds),
        author_holdout_auc: author_holdout.map(|(auc, _)| auc),
        author_holdout_folds: author_holdout.map_or(0, |(_, folds)| folds),
        document_auc: Some(document_auc),
        document_permutation_p_value: exact_document_permutation_p_value(
            &good_document_values,
            &bad_document_values,
        ),
        good_mean: mean(&good_values),
        bad_mean: mean(&bad_values),
    }
 }

 fn document_metric_values(name: &str, records: &[&ScoredText]) -> Vec<f64> {
    let mut values_by_group = BTreeMap::<String, Vec<f64>>::new();
    for record in records {
        values_by_group
            .entry(split_group_key(record).to_owned())
            .or_default()
            .push(metric_value(&record.evaluation, name));
    }
    values_by_group
        .into_values()
        .map(|values| mean(&values))
        .collect()
 }

 /// Exact, two-sided label-permutation test for a difference in document means.
 ///
 /// The cap keeps `benchmark` usable if a much larger corpus is supplied. The
 /// current nine-document corpus has only 126 possible assignments.
 fn exact_document_permutation_p_value(good: &[f64], bad: &[f64]) -> Option<f64> {
    const MAX_EXACT_PERMUTATIONS: usize = 100_000;

    let combinations = binomial_coefficient(good.len() + bad.len(), good.len())?;
    if combinations > MAX_EXACT_PERMUTATIONS {
        return None;
    }

    let observed_difference = (mean(good) - mean(bad)).abs();
    let mut all_values = Vec::with_capacity(good.len() + bad.len());
    all_values.extend_from_slice(good);
    all_values.extend_from_slice(bad);
    let total_sum = all_values.iter().sum::<f64>();
    let mut extreme_count = 0;
    let mut total_count = 0;
    enumerate_label_assignments(
        &all_values,
        good.len(),
        0,
        0,
        0.0,
        total_sum,
        observed_difference,
        &mut extreme_count,
        &mut total_count,
    );

    Some(extreme_count as f64 / total_count as f64)
 }

 #[allow(clippy::too_many_arguments)]
 fn enumerate_label_assignments(
    values: &[f64],
    good_count: usize,
    start: usize,
    chosen: usize,
    good_sum: f64,
    total_sum: f64,
    observed_difference: f64,
    extreme_count: &mut usize,
    total_count: &mut usize,
 ) {
    if chosen == good_count {
        let bad_count = values.len() - good_count;
        let difference =
            (good_sum / good_count as f64 - (total_sum - good_sum) / bad_count as f64).abs();
        if difference + f64::EPSILON >= observed_difference {
            *extreme_count += 1;
        }
        *total_count += 1;
        return;
    }

    let remaining = good_count - chosen;
    for index in start..=values.len() - remaining {
        enumerate_label_assignments(
            values,
            good_count,
            index + 1,
            chosen + 1,
            good_sum + values[index],
            total_sum,
            observed_difference,
            extreme_count,
            total_count,
        );
    }
 }

 fn binomial_coefficient(n: usize, k: usize) -> Option<usize> {
    let k = k.min(n - k);
    let mut result = 1usize;
    for index in 1..=k {
        result = result.checked_mul(n - k + index)? / index;
    }
    Some(result)
 }

 fn document_holdout_auc(
    name: &str,
    good: &[&ScoredText],
    bad: &[&ScoredText],
 ) -> Option<(f64, usize)> {
    grouped_holdout_auc(name, good, bad, split_group_key)
 }

 fn author_holdout_auc(
    name: &str,
    good: &[&ScoredText],
    bad: &[&ScoredText],
 ) -> Option<(f64, usize)> {
    let good = good
        .iter()
        .copied()
        .filter(|record| !author_key(record).is_empty())
        .collect::<Vec<_>>();
    let bad = bad
        .iter()
        .copied()
        .filter(|record| !author_key(record).is_empty())
        .collect::<Vec<_>>();
    grouped_holdout_auc(name, &good, &bad, author_key)
 }

 fn grouped_holdout_auc(
    name: &str,
    good: &[&ScoredText],
    bad: &[&ScoredText],
    group_key: fn(&ScoredText) -> &str,
 ) -> Option<(f64, usize)> {
    let good_groups = group_keys(good, group_key);
    let bad_groups = group_keys(bad, group_key);
    if good_groups.len() < 2 || bad_groups.len() < 2 {
        return None;
    }

    let mut fold_scores = Vec::new();
    for held_good_group in &good_groups {
        for held_bad_group in &bad_groups {
            let train_good = good
                .iter()
                .copied()
                .filter(|record| group_key(record) != held_good_group)
                .collect::<Vec<_>>();
            let train_bad = bad
                .iter()
                .copied()
                .filter(|record| group_key(record) != held_bad_group)
                .collect::<Vec<_>>();
            let test_good = good
                .iter()
                .copied()
                .filter(|record| group_key(record) == held_good_group)
                .collect::<Vec<_>>();
            let test_bad = bad
                .iter()
                .copied()
                .filter(|record| group_key(record) == held_bad_group)
                .collect::<Vec<_>>();

            let train_auc = pairwise_auc(
                &train_good
                    .iter()
                    .map(|record| metric_value(&record.evaluation, name))
                    .collect::<Vec<_>>(),
                &train_bad
                    .iter()
                    .map(|record| metric_value(&record.evaluation, name))
                    .collect::<Vec<_>>(),
            );
            let test_auc = pairwise_auc(
                &test_good
                    .iter()
                    .map(|record| metric_value(&record.evaluation, name))
                    .collect::<Vec<_>>(),
                &test_bad
                    .iter()
                    .map(|record| metric_value(&record.evaluation, name))
                    .collect::<Vec<_>>(),
            );
            fold_scores.push(if train_auc >= 0.5 {
                test_auc
            } else {
                1.0 - test_auc
            });
        }
    }

    Some((mean(&fold_scores), fold_scores.len()))
 }

 fn metric_value(evaluation: &Evaluation, name: &str) -> f64 {
    let metrics = &evaluation.metrics;
    let sentences = metrics.sentence_count.max(1) as f64;
    let lexical_tokens = metrics.lexical_token_count.max(1) as f64;
    if let Some(pos) = name.strip_prefix("pos_ratio:") {
        return evaluation
            .pos_sequence
            .pos_counts
            .get(pos)
            .copied()
            .unwrap_or_default() as f64
            / lexical_tokens;
    }
    match name {
        "rhythm_score" => evaluation.rhythm_score,
        "document_update_risk" => evaluation.document_update_risk,
        "sentence_length_cv" => metrics.sentence_length_cv,
        "short_long_short_rate" => {
            metrics.short_long_short_patterns as f64
                / metrics.sentence_count.saturating_sub(2).max(1) as f64
        }
        "length_contrast_rate" => {
            metrics.length_contrast_transitions as f64
                / metrics.sentence_count.saturating_sub(1).max(1) as f64
        }
        "comma_per_sentence" => metrics.comma_count as f64 / sentences,
        "question_per_sentence" => metrics.question_count as f64 / sentences,
        "content_token_ratio" => metrics.content_token_ratio,
        "hesitation_signal_rate" => metrics.hesitation_signal_sentences as f64 / sentences,
        "document_signal_rate" => metrics.document_signal_sentences as f64 / sentences,
        "short_terminal_rate" => metrics.short_terminal_sentences as f64 / sentences,
        "pos_bigram_entropy" => evaluation.pos_sequence.pos_bigram_entropy,
        "pos_bigram_repetition_rate" => evaluation.pos_sequence.pos_bigram_repetition_rate,
        "same_pos_transition_rate" => {
            evaluation.pos_sequence.same_pos_transition_count as f64
                / evaluation.pos_sequence.pos_transition_count.max(1) as f64
        }
        "mean_bunsetsu_candidate_tokens" => evaluation.bunsetsu_proxy.mean_candidate_tokens,
        "bunsetsu_candidate_length_cv" => evaluation.bunsetsu_proxy.candidate_length_cv,
        "conjunction_token_rate" => {
            evaluation.bunsetsu_proxy.conjunction_token_count as f64 / lexical_tokens
        }
        "sentence_length_dominant_power_ratio" => evaluation
            .frequency
            .sentence_length
            .dominant_power_ratio
            .unwrap_or(0.0),
        "lexical_topic_shift_mean" => evaluation.frequency.lexical_topic_shift.mean,
        "lexical_topic_shift_dominant_power_ratio" => evaluation
            .frequency
            .lexical_topic_shift
            .dominant_power_ratio
            .unwrap_or(0.0),
        _ => unreachable!("metric names are fixed"),
    }
 }

 fn pairwise_auc(good: &[f64], bad: &[f64]) -> f64 {
    let mut wins = 0.0;
    for good_value in good {
        for bad_value in bad {
            wins += match good_value.total_cmp(bad_value) {
                std::cmp::Ordering::Greater => 1.0,
                std::cmp::Ordering::Equal => 0.5,
                std::cmp::Ordering::Less => 0.0,
            };
        }
    }
    wins / (good.len() * bad.len()) as f64
 }

 fn mean(values: &[f64]) -> f64 {
    values.iter().sum::<f64>() / values.len() as f64
 }
diff --git a/justfile b/justfile
 set shell := ["zsh", "-uc"]

 default:
    @just --list

 test:
    cargo test

 fmt:
    cargo fmt --check

 lint:
    cargo clippy --all-targets -- -D warnings

 check: fmt lint test

 analyze dictionary input:
    cargo run -- analyze --dict {{ dictionary }} --file {{ input }}

 corpus dictionary input:
    cargo run -- corpus --dict {{ dictionary }} --file {{ input }}

 prepare-corpus:
    cargo run -- prepare --manifest corpus/sources.jsonl --output corpus/generated/passages.jsonl

 build-dictionary lexicon matrix character unknown output:
    cargo run -- build-dictionary --lexicon {{ lexicon }} --matrix {{ matrix }} --char {{ character }} --unknown {{ unknown }} --output {{ output }}

 benchmark dictionary input:
    cargo run -- benchmark --dict {{ dictionary }} --file {{ input }}

 validate-human dictionary corpus annotations:
    cargo run -- validate-human --dict {{ dictionary }} --corpus {{ corpus }} --annotations {{ annotations }}

 profile dictionary references candidate:
    cargo run -- profile --dict {{ dictionary }} --references {{ references }} --candidate {{ candidate }}

 profile-corpus dictionary references candidates:
    cargo run -- profile-corpus --dict {{ dictionary }} --references {{ references }} --candidates {{ candidates }}

 revise dictionary references before after:
    cargo run -- revise --dict {{ dictionary }} --references {{ references }} --before {{ before }} --after {{ after }}
diff --git a/lib.rs b/lib.rs
 //! Quantitative proxy metrics for Japanese cognitive writing rhythm.
 //!
 //! The metrics describe observable surface signals. They do not determine whether
 //! a sentence updates the situation or is good writing; that remains a human
 //! annotation task.

 use std::collections::{BTreeMap, BTreeSet};
 use std::io::Read;

 use serde::{Deserialize, Serialize};
 use vibrato::{Dictionary, SystemDictionaryBuilder, Tokenizer};

 mod corpus;

 pub use corpus::{
    CorpusError, CorpusSource, HumanAnnotation, HumanMetricCorrelation, HumanRhythmError,
    HumanRhythmReport, MetricDiagnostic, MetricSelectionError, MetricSelectionReport,
    correlate_human_rhythm, extract_passages_from_html, fetch_passages, load_sources,
    select_metrics, write_passages,
 };

 const DOCUMENT_SIGNALS: &[&str] = &[
    "本章",
    "本節",
    "この章",
    "この節",
    "ここまで",
    "次に",
    "次は",
    "次節",
    "以下",
    "本稿",
 ];

 const HESITATION_SIGNALS: &[&str] = &[
    "かもしれない",
    "だろう",
    "と思う",
    "に違いない",
    "とは思う",
    "ただ",
    "しかし",
    "ところが",
 ];

 const CONTENT_POS: &[&str] = &["名詞", "動詞", "形容詞", "副詞", "連体詞"];
 const KNOWN_POS: &[&str] = &[
    "名詞",
    "動詞",
    "形容詞",
    "副詞",
    "連体詞",
    "助詞",
    "助動詞",
    "接続詞",
    "感動詞",
    "記号",
    "補助記号",
    "フィラー",
    "空白",
 ];

 /// One owned token yielded by a morphological analyzer.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
 pub struct MorphToken {
    pub surface: String,
    pub pos: String,
 }

 impl MorphToken {
    #[must_use]
    pub fn new(surface: impl Into<String>, pos: impl Into<String>) -> Self {
        Self {
            surface: surface.into(),
            pos: pos.into(),
        }
    }
 }

 /// A sentence and its already-tokenized morphological units.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
 pub struct TokenizedSentence {
    pub text: String,
    pub tokens: Vec<MorphToken>,
 }

 impl TokenizedSentence {
    #[must_use]
    pub fn new(text: impl Into<String>, tokens: Vec<MorphToken>) -> Self {
        Self {
            text: text.into(),
            tokens,
        }
    }
 }

 /// Boundary between the metric layer and a morphological analyzer.
 pub trait MorphologicalAnalyzer {
    fn tokenize(&self, text: &str) -> Vec<MorphToken>;
 }

 /// `vibrato`-backed implementation of [`MorphologicalAnalyzer`].
 pub struct VibratoMorphAnalyzer {
    tokenizer: Tokenizer,
 }

 impl VibratoMorphAnalyzer {
    #[must_use]
    pub const fn from_dictionary(dictionary: Dictionary) -> Self {
        Self {
            tokenizer: Tokenizer::new(dictionary),
        }
    }

    /// Loads an uncompressed dictionary produced by `vibrato`.
    ///
    /// The upstream distributed dictionaries use zstd compression and must be
    /// decompressed before calling this method.
    pub fn from_dictionary_reader<R: Read>(reader: R) -> vibrato::errors::Result<Self> {
        Dictionary::read(reader).map(Self::from_dictionary)
    }

    /// Builds a dictionary from the four standard MeCab-format source files.
    pub fn from_mecab_readers<S, C, P, U>(
        system_lexicon: S,
        matrix: C,
        char_definition: P,
        unknown_definition: U,
    ) -> vibrato::errors::Result<Self>
    where
        S: Read,
        C: Read,
        P: Read,
        U: Read,
    {
        SystemDictionaryBuilder::from_readers(
            system_lexicon,
            matrix,
            char_definition,
            unknown_definition,
        )
        .map(Self::from_dictionary)
    }
 }

 impl MorphologicalAnalyzer for VibratoMorphAnalyzer {
    fn tokenize(&self, text: &str) -> Vec<MorphToken> {
        let mut worker = self.tokenizer.new_worker();
        worker.reset_sentence(text);
        worker.tokenize();

        worker
            .token_iter()
            .map(|token| MorphToken::new(token.surface(), pos_from_feature(token.feature())))
            .collect()
    }
 }

 /// Observable values used to make the score auditable.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct RhythmMetrics {
    pub sentence_count: usize,
    pub token_count: usize,
    pub lexical_token_count: usize,
    pub content_token_count: usize,
    pub content_token_ratio: f64,
    pub mean_sentence_tokens: f64,
    pub sentence_length_cv: f64,
    pub length_contrast_transitions: usize,
    pub short_long_short_patterns: usize,
    pub comma_count: usize,
    pub terminal_punctuation_count: usize,
    pub question_count: usize,
    pub hesitation_signal_sentences: usize,
    pub document_signal_sentences: usize,
    pub short_terminal_sentences: usize,
 }

 /// Part-of-speech frequencies and within-sentence transition statistics.
 ///
 /// Punctuation is excluded and transitions never cross a sentence boundary.
 #[derive(Clone, Debug, Default, PartialEq, Serialize)]
 pub struct PosSequenceMetrics {
    pub pos_counts: BTreeMap<String, usize>,
    pub pos_bigram_counts: BTreeMap<String, usize>,
    pub pos_transition_count: usize,
    pub same_pos_transition_count: usize,
    /// Share of POS transitions that repeat an already-observed POS bigram.
    pub pos_bigram_repetition_rate: f64,
    /// Shannon entropy (base 2) of observed POS bigram frequencies.
    pub pos_bigram_entropy: f64,
 }

 /// Surface-level bunsetsu candidates, not a dependency parse.
 ///
 /// Candidates end at a particle, auxiliary, conjunction, or punctuation.
 /// Adjacent candidate pairs are not dependency edges; an actual parser is
 /// required to determine heads and relation labels.
 #[derive(Clone, Debug, Default, PartialEq, Serialize)]
 pub struct BunsetsuProxyMetrics {
    pub bunsetsu_candidate_count: usize,
    pub mean_candidate_tokens: f64,
    pub candidate_length_cv: f64,
    pub adjacent_candidate_pairs: usize,
    pub conjunction_token_count: usize,
 }

 /// Frequency-domain and serial-correlation summary of a sentence-indexed series.
 ///
 /// Frequencies are cycles per sentence, not acoustic Hertz. Values are absent
 /// for fewer than four samples or for a series without variation. The
 /// autocorrelation fields are absent for fewer than eight samples.
 #[derive(Clone, Debug, Default, PartialEq, Serialize)]
 pub struct FrequencyPattern {
    pub sample_count: usize,
    pub mean: f64,
    pub dominant_frequency_cycles_per_sentence: Option<f64>,
    pub dominant_period_sentences: Option<f64>,
    pub dominant_power_ratio: Option<f64>,
    /// Highest positive Pearson correlation between the series and a lagged copy.
    pub max_lag_autocorrelation: Option<f64>,
    /// Lag, in sentences, at which `max_lag_autocorrelation` occurs.
    pub max_lag_autocorrelation_lag: Option<usize>,
    /// One-sided permutation p-value for the maximum lag autocorrelation.
    ///
    /// The null distribution is formed from 999 deterministic shuffles of the
    /// same sentence-length values, each searched over the same lag range.
    pub max_lag_autocorrelation_permutation_p_value: Option<f64>,
 }

 /// Frequency-domain proxies for prose rhythm and lexical topical movement.
 ///
 /// `lexical_topic_shift` is the adjacent-sentence Jaccard distance over
 /// surface content words. It is not a semantic embedding distance.
 #[derive(Clone, Debug, Default, PartialEq, Serialize)]
 pub struct FrequencyMetrics {
    pub sentence_length: FrequencyPattern,
    pub lexical_topic_shift: FrequencyPattern,
 }

 /// One observed feature range in a reference writing profile.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct PatternTarget {
    pub feature: String,
    pub lower_bound: f64,
    pub upper_bound: f64,
    pub reference_count: usize,
 }

 /// A target profile inferred from texts chosen as suitable references.
 ///
 /// It describes a writing context, not a universal definition of good prose.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct PatternTargetProfile {
    pub reference_count: usize,
    pub targets: Vec<PatternTarget>,
 }

 /// Direction of a candidate's adjustment relative to a target range.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum PatternAdjustment {
    Increase,
    Decrease,
    WithinRange,
 }

 /// One auditable profile comparison for a candidate text.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct PatternDeviation {
    pub feature: String,
    pub value: f64,
    pub lower_bound: f64,
    pub upper_bound: f64,
    pub distance: f64,
    pub adjustment: PatternAdjustment,
 }

 /// Feedback that a generator or human editor can use for another revision.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct PatternCorrection {
    pub assessed_feature_count: usize,
    /// Mean absolute distance outside reference ranges; lower is closer.
    pub mean_distance: f64,
    pub deviations: Vec<PatternDeviation>,
 }

 /// How one profile feature changed across a revision.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct RevisionFeatureChange {
    pub feature: String,
    pub before_distance: f64,
    pub after_distance: f64,
    /// Positive values mean that the revision moved closer to the profile.
    pub distance_reduction: f64,
    pub before_adjustment: PatternAdjustment,
    pub after_adjustment: PatternAdjustment,
 }

 /// Auditable before/after evaluation against one reference writing profile.
 ///
 /// `profile_gap_reduction_score` is the percentage of the initial mean
 /// profile distance removed by the revision. It is absent when the original
 /// was already in range, and it is not a general writing-quality score.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct RevisionEvaluation {
    pub before: Evaluation,
    pub after: Evaluation,
    pub before_correction: PatternCorrection,
    pub after_correction: PatternCorrection,
    pub profile_distance_before: f64,
    pub profile_distance_after: f64,
    pub profile_distance_reduction: f64,
    pub profile_gap_reduction_score: Option<f64>,
    /// Jaccard similarity of content-word surface-form sets, when the source
    /// text contains at least one content word.
    pub content_token_retention: Option<f64>,
    pub rhythm_score_change: f64,
    pub resolved_feature_count: usize,
    pub introduced_feature_count: usize,
    pub feature_changes: Vec<RevisionFeatureChange>,
 }

 impl PatternTargetProfile {
    /// Infers feature ranges from selected, genre-matched reference texts.
    #[must_use]
    pub fn from_references(references: &[Evaluation]) -> Self {
        let targets = PATTERN_PROFILE_FEATURES
            .iter()
            .filter_map(|feature| {
                let values = references
                    .iter()
                    .filter_map(|evaluation| pattern_profile_value(evaluation, feature))
                    .collect::<Vec<_>>();
                let (lower_bound, upper_bound) = profile_interval(&values)?;
                Some(PatternTarget {
                    feature: (*feature).to_owned(),
                    lower_bound,
                    upper_bound,
                    reference_count: values.len(),
                })
            })
            .collect();
        Self {
            reference_count: references.len(),
            targets,
        }
    }

    /// Reports which observed features lie outside this profile's ranges.
    #[must_use]
    pub fn correction_for(&self, candidate: &Evaluation) -> PatternCorrection {
        let deviations = self
            .targets
            .iter()
            .filter_map(|target| {
                let value = pattern_profile_value(candidate, &target.feature)?;
                let (distance, adjustment) = if value < target.lower_bound {
                    (target.lower_bound - value, PatternAdjustment::Increase)
                } else if value > target.upper_bound {
                    (value - target.upper_bound, PatternAdjustment::Decrease)
                } else {
                    (0.0, PatternAdjustment::WithinRange)
                };
                Some(PatternDeviation {
                    feature: target.feature.clone(),
                    value,
                    lower_bound: target.lower_bound,
                    upper_bound: target.upper_bound,
                    distance,
                    adjustment,
                })
            })
            .collect::<Vec<_>>();
        let assessed_feature_count = deviations.len();
        let mean_distance = if assessed_feature_count == 0 {
            0.0
        } else {
            deviations.iter().map(|item| item.distance).sum::<f64>() / assessed_feature_count as f64
        };
        PatternCorrection {
            assessed_feature_count,
            mean_distance,
            deviations,
        }
    }
 }

 /// Compares an already-evaluated before/after pair against a reference profile.
 ///
 /// This deliberately keeps profile alignment and content preservation
 /// separate: a rewrite must not receive a higher alignment score merely by
 /// changing its subject matter.
 #[must_use]
 pub fn assess_revision(
    profile: &PatternTargetProfile,
    before: &Evaluation,
    after: &Evaluation,
    content_token_retention: Option<f64>,
 ) -> RevisionEvaluation {
    let before_correction = profile.correction_for(before);
    let after_correction = profile.correction_for(after);
    let after_by_feature = after_correction
        .deviations
        .iter()
        .map(|deviation| (deviation.feature.as_str(), deviation))
        .collect::<BTreeMap<_, _>>();
    let feature_changes = before_correction
        .deviations
        .iter()
        .filter_map(|before_deviation| {
            let after_deviation = after_by_feature.get(before_deviation.feature.as_str())?;
            Some(RevisionFeatureChange {
                feature: before_deviation.feature.clone(),
                before_distance: before_deviation.distance,
                after_distance: after_deviation.distance,
                distance_reduction: before_deviation.distance - after_deviation.distance,
                before_adjustment: before_deviation.adjustment.clone(),
                after_adjustment: after_deviation.adjustment.clone(),
            })
        })
        .collect::<Vec<_>>();
    let resolved_feature_count = feature_changes
        .iter()
        .filter(|change| {
            change.before_distance > f64::EPSILON && change.after_distance <= f64::EPSILON
        })
        .count();
    let introduced_feature_count = feature_changes
        .iter()
        .filter(|change| {
            change.before_distance <= f64::EPSILON && change.after_distance > f64::EPSILON
        })
        .count();
    let profile_distance_before = before_correction.mean_distance;
    let profile_distance_after = after_correction.mean_distance;
    let profile_distance_reduction = profile_distance_before - profile_distance_after;
    let profile_gap_reduction_score = (profile_distance_before > f64::EPSILON)
        .then_some(profile_distance_reduction / profile_distance_before * 100.0);

    RevisionEvaluation {
        before: before.clone(),
        after: after.clone(),
        before_correction,
        after_correction,
        profile_distance_before,
        profile_distance_after,
        profile_distance_reduction,
        profile_gap_reduction_score,
        content_token_retention,
        rhythm_score_change: after.rhythm_score - before.rhythm_score,
        resolved_feature_count,
        introduced_feature_count,
        feature_changes,
    }
 }

 /// A deliberately inspectable heuristic, not a writing-quality classifier.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct Evaluation {
    /// 0--100: surface proxy for variation and sentence cadence.
    pub rhythm_score: f64,
    /// 0--100: rate of document-progress signals that require human review.
    pub document_update_risk: f64,
    pub metrics: RhythmMetrics,
    pub pos_sequence: PosSequenceMetrics,
    pub bunsetsu_proxy: BunsetsuProxyMetrics,
    pub frequency: FrequencyMetrics,
 }

 /// Human-labelled text used to calibrate or falsify the heuristic.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct LabeledText {
    pub id: String,
    pub label: String,
    #[serde(default)]
    pub source_id: String,
    #[serde(default)]
    pub author: String,
    #[serde(default)]
    pub source_url: String,
    #[serde(default)]
    pub split_group: String,
    #[serde(default)]
    pub domain: String,
    #[serde(default)]
    pub source_hash: String,
    pub text: String,
 }

 /// One labelled text with its proxy metrics.
 #[derive(Clone, Debug, Serialize)]
 pub struct ScoredText {
    pub id: String,
    pub label: String,
    pub source_id: String,
    pub author: String,
    pub split_group: String,
    pub domain: String,
    pub evaluation: Evaluation,
 }

 /// Mean scores for a label. A corpus should contain independently annotated
 /// documents before these values are interpreted.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct LabelSummary {
    pub count: usize,
    pub mean_rhythm_score: f64,
    pub mean_document_update_risk: f64,
 }

 #[derive(Clone, Debug, Serialize)]
 pub struct CorpusEvaluation {
    pub records: Vec<ScoredText>,
    pub labels: BTreeMap<String, LabelSummary>,
 }

 const PATTERN_PROFILE_FEATURES: &[&str] = &[
    "sentence_length_cv",
    "lexical_topic_shift_mean",
    "pos_bigram_entropy",
 ];

 const ROBUST_PROFILE_MINIMUM_REFERENCES: usize = 10;
 const PROFILE_LOWER_QUANTILE: f64 = 0.1;
 const PROFILE_UPPER_QUANTILE: f64 = 0.9;
 const AUTOCORRELATION_MINIMUM_SAMPLES: usize = 8;
 const AUTOCORRELATION_MAXIMUM_LAG: usize = 8;
 const AUTOCORRELATION_PERMUTATIONS: usize = 999;

 /// Splits Japanese prose while retaining `。！？!?` on each sentence.
 #[must_use]
 pub fn split_sentences(text: &str) -> Vec<String> {
    let mut sentences = Vec::new();
    let mut current = String::new();

    for character in text.chars() {
        current.push(character);
        if is_terminal_punctuation(character) {
            push_trimmed_sentence(&mut sentences, &mut current);
        }
    }
    push_trimmed_sentence(&mut sentences, &mut current);
    sentences
 }

 /// Runs the surface evaluation on sentences whose tokens are already known.
 #[must_use]
 pub fn evaluate(sentences: &[TokenizedSentence]) -> Evaluation {
    let sentence_lengths: Vec<usize> = sentences
        .iter()
        .map(|sentence| lexical_token_count(&sentence.tokens))
        .collect();
    let sentence_count = sentences.len();
    let token_count = sentences.iter().map(|sentence| sentence.tokens.len()).sum();
    let lexical_token_count = sentences
        .iter()
        .map(|sentence| lexical_token_count(&sentence.tokens))
        .sum();
    let content_token_count = sentences
        .iter()
        .flat_map(|sentence| &sentence.tokens)
        .filter(|token| is_content_pos(&token.pos))
        .count();
    let mean_sentence_tokens = mean_usize(&sentence_lengths);
    let sentence_length_cv = coefficient_of_variation(&sentence_lengths, mean_sentence_tokens);
    let length_contrast_transitions = contrast_transitions(&sentence_lengths, mean_sentence_tokens);
    let short_long_short_patterns = short_long_short_patterns(&sentence_lengths);
    let comma_count = sentences
        .iter()
        .map(|sentence| count_characters(&sentence.text, &[',', '、', '，']))
        .sum();
    let terminal_punctuation_count = sentences
        .iter()
        .map(|sentence| count_terminal_punctuation(&sentence.text))
        .sum();
    let question_count = sentences
        .iter()
        .map(|sentence| count_characters(&sentence.text, &['?', '？']))
        .sum();
    let hesitation_signal_sentences = sentences
        .iter()
        .filter(|sentence| contains_any(&sentence.text, HESITATION_SIGNALS))
        .count();
    let document_signal_sentences = sentences
        .iter()
        .filter(|sentence| contains_any(&sentence.text, DOCUMENT_SIGNALS))
        .count();
    let short_terminal_sentences = sentences
        .iter()
        .zip(&sentence_lengths)
        .filter(|(sentence, length)| {
            **length > 0
                && (**length as f64) <= mean_sentence_tokens * 0.7
                && contains_terminal_punctuation(&sentence.text)
        })
        .count();

    let content_token_ratio = if lexical_token_count == 0 {
        0.0
    } else {
        content_token_count as f64 / lexical_token_count as f64
    };
    let pos_sequence = pos_sequence_metrics(sentences);
    let bunsetsu_proxy = bunsetsu_proxy_metrics(sentences);
    let frequency = frequency_metrics(sentences, &sentence_lengths);
    let metrics = RhythmMetrics {
        sentence_count,
        token_count,
        lexical_token_count,
        content_token_count,
        content_token_ratio,
        mean_sentence_tokens,
        sentence_length_cv,
        length_contrast_transitions,
        short_long_short_patterns,
        comma_count,
        terminal_punctuation_count,
        question_count,
        hesitation_signal_sentences,
        document_signal_sentences,
        short_terminal_sentences,
    };

    Evaluation {
        rhythm_score: rhythm_score(&metrics),
        document_update_risk: percentage(document_signal_sentences, sentence_count),
        metrics,
        pos_sequence,
        bunsetsu_proxy,
        frequency,
    }
 }

 /// Tokenizes and evaluates a whole text using an injected analyzer.
 #[must_use]
 pub fn evaluate_text<A: MorphologicalAnalyzer>(analyzer: &A, text: &str) -> Evaluation {
    let sentences: Vec<_> = split_sentences(text)
        .into_iter()
        .map(|sentence| {
            let tokens = analyzer.tokenize(&sentence);
            TokenizedSentence::new(sentence, tokens)
        })
        .collect();
    evaluate(&sentences)
 }

 /// Evaluates a textual revision while retaining the surface content-word overlap.
 #[must_use]
 pub fn evaluate_revision<A: MorphologicalAnalyzer>(
    analyzer: &A,
    profile: &PatternTargetProfile,
    before_text: &str,
    after_text: &str,
 ) -> RevisionEvaluation {
    let before = evaluate_text(analyzer, before_text);
    let after = evaluate_text(analyzer, after_text);
    let content_token_retention = content_token_retention(analyzer, before_text, after_text);
    assess_revision(profile, &before, &after, content_token_retention)
 }

 /// Scores each labelled example and groups arithmetic means by label.
 #[must_use]
 pub fn evaluate_corpus<A: MorphologicalAnalyzer>(
    analyzer: &A,
    corpus: &[LabeledText],
 ) -> CorpusEvaluation {
    let records: Vec<_> = corpus
        .iter()
        .map(|entry| ScoredText {
            id: entry.id.clone(),
            label: entry.label.clone(),
            source_id: entry.source_id.clone(),
            author: entry.author.clone(),
            split_group: entry.split_group.clone(),
            domain: entry.domain.clone(),
            evaluation: evaluate_text(analyzer, &entry.text),
        })
        .collect();

    let mut totals: BTreeMap<String, (usize, f64, f64)> = BTreeMap::new();
    for record in &records {
        let total = totals.entry(record.label.clone()).or_default();
        total.0 += 1;
        total.1 += record.evaluation.rhythm_score;
        total.2 += record.evaluation.document_update_risk;
    }
    let labels = totals
        .into_iter()
        .map(|(label, (count, rhythm_total, risk_total))| {
            (
                label,
                LabelSummary {
                    count,
                    mean_rhythm_score: rhythm_total / count as f64,
                    mean_document_update_risk: risk_total / count as f64,
                },
            )
        })
        .collect();

    CorpusEvaluation { records, labels }
 }

 fn push_trimmed_sentence(sentences: &mut Vec<String>, current: &mut String) {
    let trimmed = current.trim();
    if !trimmed.is_empty() {
        sentences.push(trimmed.to_owned());
    }
    current.clear();
 }

 fn pos_from_feature(feature: &str) -> String {
    feature
        .split(',')
        .find(|item| KNOWN_POS.contains(item))
        .unwrap_or("未知")
        .to_owned()
 }

 fn is_terminal_punctuation(character: char) -> bool {
    matches!(character, '。' | '！' | '？' | '!' | '?')
 }

 fn contains_terminal_punctuation(text: &str) -> bool {
    text.chars().any(is_terminal_punctuation)
 }

 fn count_terminal_punctuation(text: &str) -> usize {
    text.chars()
        .filter(|character| is_terminal_punctuation(*character))
        .count()
 }

 fn count_characters(text: &str, targets: &[char]) -> usize {
    text.chars()
        .filter(|character| targets.contains(character))
        .count()
 }

 fn contains_any(text: &str, markers: &[&str]) -> bool {
    markers.iter().any(|marker| text.contains(marker))
 }

 fn is_content_pos(pos: &str) -> bool {
    CONTENT_POS.contains(&pos)
 }

 fn is_punctuation(token: &MorphToken) -> bool {
    token.pos.contains("記号")
        || token.surface.chars().all(|character| {
            character.is_ascii_punctuation() || "、。！？「」『』（）［］【】".contains(character)
        })
 }

 fn lexical_token_count(tokens: &[MorphToken]) -> usize {
    tokens.iter().filter(|token| !is_punctuation(token)).count()
 }

 fn content_token_set<A: MorphologicalAnalyzer>(analyzer: &A, text: &str) -> BTreeSet<String> {
    split_sentences(text)
        .into_iter()
        .flat_map(|sentence| analyzer.tokenize(&sentence))
        .filter(|token| is_content_pos(&token.pos))
        .map(|token| token.surface.to_lowercase())
        .collect()
 }

 fn content_token_retention<A: MorphologicalAnalyzer>(
    analyzer: &A,
    before_text: &str,
    after_text: &str,
 ) -> Option<f64> {
    let before = content_token_set(analyzer, before_text);
    if before.is_empty() {
        return None;
    }
    let after = content_token_set(analyzer, after_text);
    let union_count = before.union(&after).count();
    Some(before.intersection(&after).count() as f64 / union_count as f64)
 }

 fn pos_sequence_metrics(sentences: &[TokenizedSentence]) -> PosSequenceMetrics {
    let mut pos_counts = BTreeMap::new();
    let mut pos_bigram_counts = BTreeMap::new();
    let mut pos_transition_count = 0;
    let mut same_pos_transition_count = 0;

    for sentence in sentences {
        let poses: Vec<_> = sentence
            .tokens
            .iter()
            .filter(|token| !is_punctuation(token))
            .map(|token| token.pos.as_str())
            .collect();
        for pos in &poses {
            *pos_counts.entry((*pos).to_owned()).or_default() += 1;
        }
        for pair in poses.windows(2) {
            pos_transition_count += 1;
            if pair[0] == pair[1] {
                same_pos_transition_count += 1;
            }
            *pos_bigram_counts
                .entry(format!("{}→{}", pair[0], pair[1]))
                .or_default() += 1;
        }
    }

    PosSequenceMetrics {
        pos_counts,
        pos_bigram_entropy: shannon_entropy(&pos_bigram_counts),
        pos_bigram_repetition_rate: if pos_transition_count == 0 {
            0.0
        } else {
            (pos_transition_count - pos_bigram_counts.len()) as f64 / pos_transition_count as f64
        },
        pos_bigram_counts,
        pos_transition_count,
        same_pos_transition_count,
    }
 }

 fn bunsetsu_proxy_metrics(sentences: &[TokenizedSentence]) -> BunsetsuProxyMetrics {
    let mut candidate_lengths = Vec::new();
    let mut adjacent_candidate_pairs = 0;
    let mut conjunction_token_count = 0;

    for sentence in sentences {
        let mut candidate_count = 0;
        let mut current_length = 0;
        for token in &sentence.tokens {
            if token.pos == "接続詞" {
                conjunction_token_count += 1;
            }
            if is_punctuation(token) {
                push_bunsetsu_candidate(
                    &mut candidate_lengths,
                    &mut current_length,
                    &mut candidate_count,
                );
                continue;
            }

            current_length += 1;
            if ends_bunsetsu_candidate(token) {
                push_bunsetsu_candidate(
                    &mut candidate_lengths,
                    &mut current_length,
                    &mut candidate_count,
                );
            }
        }
        push_bunsetsu_candidate(
            &mut candidate_lengths,
            &mut current_length,
            &mut candidate_count,
        );
        adjacent_candidate_pairs += candidate_count.saturating_sub(1);
    }

    let mean_candidate_tokens = mean_usize(&candidate_lengths);
    BunsetsuProxyMetrics {
        bunsetsu_candidate_count: candidate_lengths.len(),
        mean_candidate_tokens,
        candidate_length_cv: coefficient_of_variation(&candidate_lengths, mean_candidate_tokens),
        adjacent_candidate_pairs,
        conjunction_token_count,
    }
 }

 fn frequency_metrics(
    sentences: &[TokenizedSentence],
    sentence_lengths: &[usize],
 ) -> FrequencyMetrics {
    FrequencyMetrics {
        sentence_length: frequency_pattern(
            &sentence_lengths
                .iter()
                .map(|length| *length as f64)
                .collect::<Vec<_>>(),
        ),
        lexical_topic_shift: frequency_pattern(&lexical_topic_shift_series(sentences)),
    }
 }

 fn lexical_topic_shift_series(sentences: &[TokenizedSentence]) -> Vec<f64> {
    let mut previous = None;
    let mut shifts = Vec::with_capacity(sentences.len());

    for sentence in sentences {
        let current = sentence
            .tokens
            .iter()
            .filter(|token| is_content_pos(&token.pos))
            .map(|token| token.surface.to_lowercase())
            .collect::<BTreeSet<_>>();
        let shift = previous.as_ref().map_or(0.0, |prior: &BTreeSet<String>| {
            let union_count = prior.union(&current).count();
            if union_count == 0 {
                0.0
            } else {
                1.0 - prior.intersection(&current).count() as f64 / union_count as f64
            }
        });
        shifts.push(shift);
        previous = Some(current);
    }

    shifts
 }

 fn frequency_pattern(values: &[f64]) -> FrequencyPattern {
    let sample_count = values.len();
    let mean = if sample_count == 0 {
        0.0
    } else {
        values.iter().sum::<f64>() / sample_count as f64
    };
    let maximum_autocorrelation = max_positive_lag_autocorrelation(values);
    let max_lag_autocorrelation_permutation_p_value =
        maximum_autocorrelation.and_then(|(_, correlation)| {
            max_lag_autocorrelation_permutation_p_value(values, correlation)
        });
    if sample_count < 4 {
        return FrequencyPattern {
            sample_count,
            mean,
            max_lag_autocorrelation: maximum_autocorrelation.map(|(_, correlation)| correlation),
            max_lag_autocorrelation_lag: maximum_autocorrelation.map(|(lag, _)| lag),
            max_lag_autocorrelation_permutation_p_value,
            ..FrequencyPattern::default()
        };
    }

    let centered = values.iter().map(|value| value - mean).collect::<Vec<_>>();
    let mut total_power = 0.0;
    let mut dominant = None;
    for frequency_index in 1..=sample_count / 2 {
        let mut real = 0.0;
        let mut imaginary = 0.0;
        for (sample_index, value) in centered.iter().enumerate() {
            let phase = std::f64::consts::TAU * frequency_index as f64 * sample_index as f64
                / sample_count as f64;
            real += value * phase.cos();
            imaginary -= value * phase.sin();
        }
        let power = real.mul_add(real, imaginary * imaginary);
        total_power += power;
        if dominant.is_none_or(|(_, dominant_power)| power > dominant_power) {
            dominant = Some((frequency_index, power));
        }
    }

    let Some((frequency_index, dominant_power)) = dominant else {
        return FrequencyPattern {
            sample_count,
            mean,
            max_lag_autocorrelation: maximum_autocorrelation.map(|(_, correlation)| correlation),
            max_lag_autocorrelation_lag: maximum_autocorrelation.map(|(lag, _)| lag),
            max_lag_autocorrelation_permutation_p_value,
            ..FrequencyPattern::default()
        };
    };
    if total_power <= f64::EPSILON {
        return FrequencyPattern {
            sample_count,
            mean,
            max_lag_autocorrelation: maximum_autocorrelation.map(|(_, correlation)| correlation),
            max_lag_autocorrelation_lag: maximum_autocorrelation.map(|(lag, _)| lag),
            max_lag_autocorrelation_permutation_p_value,
            ..FrequencyPattern::default()
        };
    }

    FrequencyPattern {
        sample_count,
        mean,
        dominant_frequency_cycles_per_sentence: Some(frequency_index as f64 / sample_count as f64),
        dominant_period_sentences: Some(sample_count as f64 / frequency_index as f64),
        dominant_power_ratio: Some(dominant_power / total_power),
        max_lag_autocorrelation: maximum_autocorrelation.map(|(_, correlation)| correlation),
        max_lag_autocorrelation_lag: maximum_autocorrelation.map(|(lag, _)| lag),
        max_lag_autocorrelation_permutation_p_value,
    }
 }

 fn max_positive_lag_autocorrelation(values: &[f64]) -> Option<(usize, f64)> {
    if values.len() < AUTOCORRELATION_MINIMUM_SAMPLES {
        return None;
    }
    let maximum_lag = (values.len() / 2).min(AUTOCORRELATION_MAXIMUM_LAG);
    let mut maximum = None;
    for lag in 1..=maximum_lag {
        let Some(correlation) = lag_pearson_correlation(values, lag) else {
            continue;
        };
        if correlation <= 0.0 {
            continue;
        }
        if maximum.is_none_or(|(_, best)| correlation > best) {
            maximum = Some((lag, correlation));
        }
    }
    maximum
 }

 fn max_lag_autocorrelation_permutation_p_value(
    values: &[f64],
    observed_correlation: f64,
 ) -> Option<f64> {
    if values.len() < AUTOCORRELATION_MINIMUM_SAMPLES {
        return None;
    }
    let mut state = autocorrelation_seed(values);
    let mut at_least_as_large = 0_usize;
    let mut shuffled = values.to_vec();
    for _ in 0..AUTOCORRELATION_PERMUTATIONS {
        shuffled.copy_from_slice(values);
        deterministic_shuffle(&mut shuffled, &mut state);
        if max_positive_lag_autocorrelation(&shuffled)
            .is_some_and(|(_, correlation)| correlation >= observed_correlation)
        {
            at_least_as_large += 1;
        }
    }
    Some((at_least_as_large + 1) as f64 / (AUTOCORRELATION_PERMUTATIONS + 1) as f64)
 }

 fn lag_pearson_correlation(values: &[f64], lag: usize) -> Option<f64> {
    let paired_count = values.len().checked_sub(lag)?;
    if paired_count < 4 {
        return None;
    }
    let earlier = &values[..paired_count];
    let later = &values[lag..];
    let earlier_mean = earlier.iter().sum::<f64>() / paired_count as f64;
    let later_mean = later.iter().sum::<f64>() / paired_count as f64;
    let (covariance, earlier_sum_squares, later_sum_squares) = earlier.iter().zip(later).fold(
        (0.0, 0.0, 0.0),
        |(covariance, earlier_sum_squares, later_sum_squares), (earlier, later)| {
            let earlier_delta = earlier - earlier_mean;
            let later_delta = later - later_mean;
            (
                covariance + earlier_delta * later_delta,
                earlier_sum_squares + earlier_delta * earlier_delta,
                later_sum_squares + later_delta * later_delta,
            )
        },
    );
    let denominator = (earlier_sum_squares * later_sum_squares).sqrt();
    (denominator > f64::EPSILON).then_some(covariance / denominator)
 }

 fn autocorrelation_seed(values: &[f64]) -> u64 {
    values
        .iter()
        .enumerate()
        .fold(0x9e37_79b9_7f4a_7c15, |seed, (index, value)| {
            let rotated = value
                .to_bits()
                .rotate_left((index % u64::BITS as usize) as u32);
            seed.rotate_left(7) ^ rotated.wrapping_mul(0xbf58_476d_1ce4_e5b9)
        })
 }

 fn deterministic_shuffle(values: &mut [f64], state: &mut u64) {
    for last_index in (1..values.len()).rev() {
        *state = state
            .wrapping_mul(6_364_136_223_846_793_005)
            .wrapping_add(1_442_695_040_888_963_407);
        let swap_index = (*state % (last_index + 1) as u64) as usize;
        values.swap(last_index, swap_index);
    }
 }

 fn pattern_profile_value(evaluation: &Evaluation, feature: &str) -> Option<f64> {
    match feature {
        "sentence_length_cv" => Some(evaluation.metrics.sentence_length_cv),
        "lexical_topic_shift_mean" => Some(evaluation.frequency.lexical_topic_shift.mean),
        "pos_bigram_entropy" => Some(evaluation.pos_sequence.pos_bigram_entropy),
        _ => None,
    }
 }

 fn profile_interval(values: &[f64]) -> Option<(f64, f64)> {
    if values.len() < ROBUST_PROFILE_MINIMUM_REFERENCES {
        return Some((
            values.iter().copied().reduce(f64::min)?,
            values.iter().copied().reduce(f64::max)?,
        ));
    }
    Some((
        quantile(values, PROFILE_LOWER_QUANTILE)?,
        quantile(values, PROFILE_UPPER_QUANTILE)?,
    ))
 }

 fn quantile(values: &[f64], quantile: f64) -> Option<f64> {
    let mut sorted = values.to_vec();
    sorted.sort_by(f64::total_cmp);
    let last_index = sorted.len().checked_sub(1)?;
    let position = quantile * last_index as f64;
    let lower_index = position.floor() as usize;
    let upper_index = position.ceil() as usize;
    let interpolation = position - lower_index as f64;
    Some(sorted[lower_index] * (1.0 - interpolation) + sorted[upper_index] * interpolation)
 }

 fn push_bunsetsu_candidate(
    candidate_lengths: &mut Vec<usize>,
    current_length: &mut usize,
    candidate_count: &mut usize,
 ) {
    if *current_length == 0 {
        return;
    }
    candidate_lengths.push(*current_length);
    *candidate_count += 1;
    *current_length = 0;
 }

 fn ends_bunsetsu_candidate(token: &MorphToken) -> bool {
    matches!(token.pos.as_str(), "助詞" | "助動詞" | "接続詞")
 }

 fn shannon_entropy(counts: &BTreeMap<String, usize>) -> f64 {
    let total = counts.values().sum::<usize>();
    if total == 0 {
        return 0.0;
    }
    counts
        .values()
        .map(|count| {
            let probability = *count as f64 / total as f64;
            -probability * probability.log2()
        })
        .sum()
 }

 fn mean_usize(values: &[usize]) -> f64 {
    if values.is_empty() {
        0.0
    } else {
        values.iter().sum::<usize>() as f64 / values.len() as f64
    }
 }

 fn coefficient_of_variation(values: &[usize], mean: f64) -> f64 {
    if values.is_empty() || mean == 0.0 {
        return 0.0;
    }
    let variance = values
        .iter()
        .map(|value| (*value as f64 - mean).powi(2))
        .sum::<f64>()
        / values.len() as f64;
    variance.sqrt() / mean
 }

 fn contrast_transitions(lengths: &[usize], mean: f64) -> usize {
    lengths
        .windows(2)
        .filter(|pair| (pair[0] as f64 - pair[1] as f64).abs() >= mean * 0.35)
        .count()
 }

 fn short_long_short_patterns(lengths: &[usize]) -> usize {
    lengths
        .windows(3)
        .filter(|window| {
            let first = window[0] as f64;
            let middle = window[1] as f64;
            let last = window[2] as f64;
            let outer_mean = (first + last) / 2.0;
            outer_mean > 0.0 && middle >= outer_mean * 1.6 && last <= middle * 0.55
        })
        .count()
 }

 fn percentage(numerator: usize, denominator: usize) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        (numerator as f64 / denominator as f64) * 100.0
    }
 }

 fn capped_ratio(numerator: usize, denominator: usize) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        (numerator as f64 / denominator as f64).min(1.0)
    }
 }

 fn rhythm_score(metrics: &RhythmMetrics) -> f64 {
    let variation = (metrics.sentence_length_cv / 0.75).min(1.0);
    let cadence = capped_ratio(
        metrics.short_long_short_patterns,
        metrics.sentence_count.saturating_sub(2),
    );
    let transitions = capped_ratio(
        metrics.length_contrast_transitions,
        metrics.sentence_count.saturating_sub(1),
    );
    let hesitation = capped_ratio(
        metrics.hesitation_signal_sentences,
        ((metrics.sentence_count as f64) * 0.33).ceil() as usize,
    );
    let short_stops = capped_ratio(metrics.short_terminal_sentences, metrics.sentence_count);

    100.0
        * (0.30 * variation
            + 0.30 * cadence
            + 0.20 * transitions
            + 0.10 * hesitation
            + 0.10 * short_stops)
 }
diff --git a/main.rs b/main.rs
 use std::env;
 use std::error::Error;
 use std::fs::{self, File};

 use cognitive_rhythm_eval::{
    HumanAnnotation, PatternCorrection, PatternTargetProfile, RevisionEvaluation,
    VibratoMorphAnalyzer, correlate_human_rhythm, evaluate_corpus, evaluate_revision,
    evaluate_text, fetch_passages, load_sources, select_metrics, write_passages,
 };
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use vibrato::SystemDictionaryBuilder;

 #[derive(Serialize)]
 struct BenchmarkOutput {
    corpus: cognitive_rhythm_eval::CorpusEvaluation,
    selection: cognitive_rhythm_eval::MetricSelectionReport,
 }

 #[derive(Serialize)]
 struct ProfileOutput {
    profile: PatternTargetProfile,
    correction: PatternCorrection,
 }

 #[derive(Serialize)]
 struct ProfileCorpusRecord {
    id: String,
    label: String,
    source_id: String,
    author: String,
    split_group: String,
    domain: String,
    correction: PatternCorrection,
 }

 #[derive(Serialize)]
 struct ProfileCorpusOutput {
    profile: PatternTargetProfile,
    records: Vec<ProfileCorpusRecord>,
 }

 #[derive(Serialize)]
 struct RevisionOutput {
    profile: PatternTargetProfile,
    revision: RevisionEvaluation,
 }

 fn main() -> Result<(), Box<dyn Error>> {
    let arguments: Vec<_> = env::args().skip(1).collect();
    match arguments.as_slice() {
        [
            command,
            manifest_flag,
            manifest_path,
            output_flag,
            output_path,
        ] if command == "prepare" && manifest_flag == "--manifest" && output_flag == "--output" => {
            let sources = load_sources(manifest_path)?;
            let passages = fetch_passages(&sources)?;
            write_passages(output_path, &passages)?;
            println!("prepared {} passages at {output_path}", passages.len());
            Ok(())
        }
        [
            command,
            lexicon_flag,
            lexicon_path,
            matrix_flag,
            matrix_path,
            char_flag,
            char_path,
            unknown_flag,
            unknown_path,
            output_flag,
            output_path,
        ] if command == "build-dictionary"
            && lexicon_flag == "--lexicon"
            && matrix_flag == "--matrix"
            && char_flag == "--char"
            && unknown_flag == "--unknown"
            && output_flag == "--output" =>
        {
            let dictionary = SystemDictionaryBuilder::from_readers(
                File::open(lexicon_path)?,
                File::open(matrix_path)?,
                File::open(char_path)?,
                File::open(unknown_path)?,
            )?;
            dictionary.write(File::create(output_path)?)?;
            println!("built Vibrato dictionary at {output_path}");
            Ok(())
        }
        [
            command,
            dictionary_flag,
            dictionary_path,
            references_flag,
            references_path,
            candidate_flag,
            candidate_path,
        ] if command == "profile"
            && dictionary_flag == "--dict"
            && references_flag == "--references"
            && candidate_flag == "--candidate" =>
        {
            let analyzer =
                VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
            let profile = profile_from_references(&analyzer, references_path)?;
            let candidate = evaluate_text(&analyzer, &fs::read_to_string(candidate_path)?);
            let correction = profile.correction_for(&candidate);
            print_json(&ProfileOutput {
                profile,
                correction,
            })
        }
        [
            command,
            dictionary_flag,
            dictionary_path,
            references_flag,
            references_path,
            candidates_flag,
            candidates_path,
        ] if command == "profile-corpus"
            && dictionary_flag == "--dict"
            && references_flag == "--references"
            && candidates_flag == "--candidates" =>
        {
            let analyzer =
                VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
            let profile = profile_from_references(&analyzer, references_path)?;
            let candidates = parse_jsonl(&fs::read_to_string(candidates_path)?)?;
            let candidates = evaluate_corpus(&analyzer, &candidates);
            let records = candidates
                .records
                .into_iter()
                .map(|record| ProfileCorpusRecord {
                    id: record.id,
                    label: record.label,
                    source_id: record.source_id,
                    author: record.author,
                    split_group: record.split_group,
                    domain: record.domain,
                    correction: profile.correction_for(&record.evaluation),
                })
                .collect();
            print_json(&ProfileCorpusOutput { profile, records })
        }
        [
            command,
            dictionary_flag,
            dictionary_path,
            references_flag,
            references_path,
            before_flag,
            before_path,
            after_flag,
            after_path,
        ] if command == "revise"
            && dictionary_flag == "--dict"
            && references_flag == "--references"
            && before_flag == "--before"
            && after_flag == "--after" =>
        {
            let analyzer =
                VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
            let profile = profile_from_references(&analyzer, references_path)?;
            let before = fs::read_to_string(before_path)?;
            let after = fs::read_to_string(after_path)?;
            let revision = evaluate_revision(&analyzer, &profile, &before, &after);
            print_json(&RevisionOutput { profile, revision })
        }
        [
            command,
            dictionary_flag,
            dictionary_path,
            corpus_flag,
            corpus_path,
            annotations_flag,
            annotations_path,
        ] if command == "validate-human"
            && dictionary_flag == "--dict"
            && corpus_flag == "--corpus"
            && annotations_flag == "--annotations" =>
        {
            let analyzer =
                VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
            let corpus = parse_jsonl(&fs::read_to_string(corpus_path)?)?;
            let annotations: Vec<HumanAnnotation> =
                parse_jsonl(&fs::read_to_string(annotations_path)?)?;
            let corpus = evaluate_corpus(&analyzer, &corpus);
            let human_rhythm = correlate_human_rhythm(&corpus.records, &annotations)?;
            print_json(&human_rhythm)
        }
        [
            command,
            dictionary_flag,
            dictionary_path,
            file_flag,
            input_path,
        ] if dictionary_flag == "--dict" && file_flag == "--file" => {
            let analyzer =
                VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
            let input = fs::read_to_string(input_path)?;
            match command.as_str() {
                "analyze" => print_json(&evaluate_text(&analyzer, &input)),
                "corpus" => {
                    let corpus = parse_jsonl(&input)?;
                    print_json(&evaluate_corpus(&analyzer, &corpus))
                }
                "benchmark" => {
                    let corpus = parse_jsonl(&input)?;
                    let corpus = evaluate_corpus(&analyzer, &corpus);
                    let selection = select_metrics(&corpus.records)?;
                    print_json(&BenchmarkOutput { corpus, selection })
                }
                _ => Err(usage().into()),
            }
        }
        _ => Err(usage().into()),
    }
 }

 fn parse_jsonl<T: DeserializeOwned>(input: &str) -> Result<Vec<T>, serde_json::Error> {
    input
        .lines()
        .filter(|line| !line.trim().is_empty())
        .map(serde_json::from_str)
        .collect()
 }

 fn profile_from_references(
    analyzer: &VibratoMorphAnalyzer,
    references_path: &str,
 ) -> Result<PatternTargetProfile, Box<dyn Error>> {
    let references = parse_jsonl(&fs::read_to_string(references_path)?)?;
    let references = evaluate_corpus(analyzer, &references);
    Ok(PatternTargetProfile::from_references(
        &references
            .records
            .iter()
            .map(|record| record.evaluation.clone())
            .collect::<Vec<_>>(),
    ))
 }

 fn print_json(value: &impl serde::Serialize) -> Result<(), Box<dyn Error>> {
    println!("{}", serde_json::to_string_pretty(value)?);
    Ok(())
 }

 fn usage() -> String {
    "usage:\n  cognitive-rhythm-eval prepare --manifest SOURCES.jsonl --output PASSAGES.jsonl\n  cognitive-rhythm-eval build-dictionary --lexicon LEX.csv --matrix matrix.def --char char.def --unknown unk.def --output SYSTEM.dic\n  cognitive-rhythm-eval analyze --dict SYSTEM.dic --file TEXT.md\n  cognitive-rhythm-eval corpus --dict SYSTEM.dic --file LABELED.jsonl\n  cognitive-rhythm-eval benchmark --dict SYSTEM.dic --file LABELED.jsonl\n  cognitive-rhythm-eval validate-human --dict SYSTEM.dic --corpus PASSAGES.jsonl --annotations ANNOTATIONS.jsonl\n  cognitive-rhythm-eval profile --dict SYSTEM.dic --references REFERENCE.jsonl --candidate CANDIDATE.md\n  cognitive-rhythm-eval profile-corpus --dict SYSTEM.dic --references REFERENCE.jsonl --candidates CANDIDATES.jsonl\n  cognitive-rhythm-eval revise --dict SYSTEM.dic --references REFERENCE.jsonl --before BEFORE.md --after AFTER.md".to_owned()
 }
diff --git a/research-notes-ja.md b/research-notes-ja.md
diff --git a/trpg-evaluation-ja.md b/trpg-evaluation-ja.md
指標	定義	規範との対応
`sentence_length_cv`	句読点を除く文ごとの形態素数の変動係数	文の長短が一様でないか
`short_long_short_patterns`	短→長→短の 3 文パターン数	「立てる→流す→止める」の候補
`length_contrast_transitions`	平均文長の 35% 以上変化する隣接文数	拍の切替候補
`comma_count`, `question_count`	`、` と `？` を含む句読点の数	節内の流れ、問いの候補
`content_token_ratio`	名詞・動詞・形容詞・副詞・連体詞 ÷ 非記号トークン	内容語密度
`hesitation_signal_sentences`	逡巡・転回の表層語を含む文数	「かもしれない」「しかし」などの候補
`document_signal_sentences`	進行実況に多い表層語を含む文数	人手で話題テストすべき候補
`pos_sequence.pos_counts`	句読点を除く品詞の頻度	名詞・助詞・動詞などの偏り
`pos_sequence.pos_bigram_counts`	文内で隣接する品詞列の頻度	「名詞→助詞→動詞」などの並び
`pos_sequence.pos_bigram_entropy`	品詞 bigram 分布の Shannon entropy	品詞遷移の多様性の候補
`pos_sequence.pos_bigram_repetition_rate`	既出 POS bigram を再使用した遷移の割合	テンプレート反復の候補
`bunsetsu_proxy`	助詞・助動詞・接続詞で区切る文節候補	構文解析前の表層構造
`frequency.sentence_length`	文ごとの形態素数系列の DFT・最大 lag 自己相関・シャッフル p 値	文長リズムの診断候補（品質スコアには不使用）
`frequency.lexical_topic_shift`	隣接文の内容語集合の Jaccard 距離系列	語彙的な話題転換の周期候補
項目	値	判断基準
`rhythm_1_to_5`	1–5	読む速度・停止・密度の変化が、本文の目的に対して自然で追いやすいか。1 は単調または過度に断続的、3 は判断不能でない中間、5 は変化が目的に寄与する。
`document_only`	yes/no/不明	本文が対象世界・論点を進めず、文書の進行だけを説明しているか。
`situation_or_judgment_update`	yes/no/不明	読後に、対象世界または論点について新しい状態・判断が確定したか。
観測値	初稿	改稿後	変化
プロフィール距離	0.122827	0.118306	-0.004520
`profile_gap_reduction_score`	—	3.68%	初期帯域外距離の解消率
`rhythm_score`	34.73	31.58	-3.15
文長変動係数	0.371	0.382	+0.011
語彙的話題変化の平均	0.944	0.940	-0.004
POS bigram エントロピー	3.605	3.595	-0.010
内容語表層集合の保持率	—	65.73%	名詞などの集合に対する Jaccard 類似度
	[package]
	name = "cognitive-rhythm-eval"
	version = "0.1.0"
	edition = "2024"
	rust-version = "1.85"
	description = "A measurable proxy evaluator for Japanese cognitive writing rhythm"
	license = "MIT"

	[dependencies]
	encoding_rs = "0.8"
	reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] }
	scraper = "0.24"
	serde = { version = "1.0.228", features = ["derive"] }
	serde_json = "1.0.149"
	sha2 = "0.10"
	vibrato = { version = "=0.5.2", default-features = false }
ID	種別	`[critical]` の中心
A	技術ブログの3段落改稿	3つの技術的事実を保持し、`rhythm_score` を品質点にしない
B	参照・辞書なしの TRPG 導入	数値最適化を開始せず、人手チェックへ切り替える
C	文学参照で技術仕様を評価したい依頼	異ジャンルのプロフィール比較を保留する
D	`document_update_risk` が高い障害報告	時刻・状態・次回更新を機械的に削除しない
E	未提示の手順を補うよう求めるリリースノート	未提示の内部実装・操作を創作しない
F	未決定のロールバック条件を含む ADR（ホールドアウト）	未決定事項と確認責任を本文・事実台帳に残す
ラウンド	シナリオ	成功	正確度	steps	duration	規範側の新しい不明点
初回	A	○	100%	1	取得不可	0
初回	B	○	100%	1	取得不可	0
初回	C	○	100%	1	取得不可	0
再評価	A	○	100%	1	取得不可	0
再評価	B	○	100%	1	取得不可	0
再評価	C	○	100%	1	取得不可	0
初期ホールドアウト	D	○	100%	1	取得不可	0（ただし未提示操作の扱いを強化）
事実境界の追加後	D	○	100%	1	取得不可	0（入力側の操作欠損）
事実境界の追加後	E	○	100%	1	取得不可	0（入力側の実装・操作欠損）
事実台帳の追加後	D	○	100%	1	取得不可	0
事実台帳の追加後	E	○	100%	1	取得不可	0
最終再評価	A	○	100%	1	取得不可	0
最終再評価	C	○	100%	1	取得不可	0（1,200字版に必要な仕様は入力側で未提示）
最終ホールドアウト	F	○	100%	1	取得不可	0
name	cognitive-rhythm-writing-with-eval
description	日本語の説明文・エッセイ・導入文を、状況と判断の更新を中心に執筆・改稿し、cognitive-rhythm-eval の形態素・句読点診断を補助的に用いる。数値の最適化ではなく、参照文とのずれを見つけて人間が改稿を判断したいときに使う。
出力	読み方	見直す問い
`sentence_length_cv`	文長の揺れの大きさ	全文が同じ息継ぎになっていないか。必要な説明まで短くしていないか。
`short_long_short_patterns` / `length_contrast_transitions`	文長の変化候補	変化が認識・論理・場面の切替と一致しているか。偶然の句読点操作ではないか。
`content_token_ratio`	内容語の表層密度	抽象的なつなぎ語や進行実況で、対象の描写が薄くなっていないか。専門語が多いだけでは高品質ではない。
`document_update_risk`	進行実況を含む文の候補	各候補を話題テストにかける。必要な道標なら残し、状況を変えないなら統合・削除・書換えを検討する。
`pos_counts` / `pos_bigram_counts`	品詞の偏り・反復候補	同じ言い切り、名詞止め、接続の型が続いていないかを原文で確認する。
`pos_bigram_entropy`	品詞遷移の多様性の代理	低い・高いこと自体を善悪としない。参照帯域から外れたときだけ、文型の固定を疑う。
`frequency.*`	文長・語彙的話題移動の周期診断	文章の周期性が偶然より強いかを見るだけ。意味の周波数や快さを測ったものではない。
	set shell := ["zsh", "-uc"]

	default:
	@just --list

	test:
	cargo test

	fmt:
	cargo fmt --check

	lint:
	cargo clippy --all-targets -- -D warnings

	check: fmt lint test

	analyze dictionary input:
	cargo run -- analyze --dict {{ dictionary }} --file {{ input }}

	corpus dictionary input:
	cargo run -- corpus --dict {{ dictionary }} --file {{ input }}

	prepare-corpus:
	cargo run -- prepare --manifest corpus/sources.jsonl --output corpus/generated/passages.jsonl

	build-dictionary lexicon matrix character unknown output:
	cargo run -- build-dictionary --lexicon {{ lexicon }} --matrix {{ matrix }} --char {{ character }} --unknown {{ unknown }} --output {{ output }}

	benchmark dictionary input:
	cargo run -- benchmark --dict {{ dictionary }} --file {{ input }}

	validate-human dictionary corpus annotations:
	cargo run -- validate-human --dict {{ dictionary }} --corpus {{ corpus }} --annotations {{ annotations }}

	profile dictionary references candidate:
	cargo run -- profile --dict {{ dictionary }} --references {{ references }} --candidate {{ candidate }}

	profile-corpus dictionary references candidates:
	cargo run -- profile-corpus --dict {{ dictionary }} --references {{ references }} --candidates {{ candidates }}

	revise dictionary references before after:
	cargo run -- revise --dict {{ dictionary }} --references {{ references }} --before {{ before }} --after {{ after }}
	use std::env;
	use std::error::Error;
	use std::fs::{self, File};

	use cognitive_rhythm_eval::{
	HumanAnnotation, PatternCorrection, PatternTargetProfile, RevisionEvaluation,
	VibratoMorphAnalyzer, correlate_human_rhythm, evaluate_corpus, evaluate_revision,
	evaluate_text, fetch_passages, load_sources, select_metrics, write_passages,
	};
	use serde::Serialize;
	use serde::de::DeserializeOwned;
	use vibrato::SystemDictionaryBuilder;

	#[derive(Serialize)]
	struct BenchmarkOutput {
	corpus: cognitive_rhythm_eval::CorpusEvaluation,
	selection: cognitive_rhythm_eval::MetricSelectionReport,
	}

	#[derive(Serialize)]
	struct ProfileOutput {
	profile: PatternTargetProfile,
	correction: PatternCorrection,
	}

	#[derive(Serialize)]
	struct ProfileCorpusRecord {
	id: String,
	label: String,
	source_id: String,
	author: String,
	split_group: String,
	domain: String,
	correction: PatternCorrection,
	}

	#[derive(Serialize)]
	struct ProfileCorpusOutput {
	profile: PatternTargetProfile,
	records: Vec<ProfileCorpusRecord>,
	}

	#[derive(Serialize)]
	struct RevisionOutput {
	profile: PatternTargetProfile,
	revision: RevisionEvaluation,
	}

	fn main() -> Result<(), Box<dyn Error>> {
	let arguments: Vec<_> = env::args().skip(1).collect();
	match arguments.as_slice() {
	[
	command,
	manifest_flag,
	manifest_path,
	output_flag,
	output_path,
	] if command == "prepare" && manifest_flag == "--manifest" && output_flag == "--output" => {
	let sources = load_sources(manifest_path)?;
	let passages = fetch_passages(&sources)?;
	write_passages(output_path, &passages)?;
	println!("prepared {} passages at {output_path}", passages.len());
	Ok(())
	}
	[
	command,
	lexicon_flag,
	lexicon_path,
	matrix_flag,
	matrix_path,
	char_flag,
	char_path,
	unknown_flag,
	unknown_path,
	output_flag,
	output_path,
	] if command == "build-dictionary"
	&& lexicon_flag == "--lexicon"
	&& matrix_flag == "--matrix"
	&& char_flag == "--char"
	&& unknown_flag == "--unknown"
	&& output_flag == "--output" =>
	{
	let dictionary = SystemDictionaryBuilder::from_readers(
	File::open(lexicon_path)?,
	File::open(matrix_path)?,
	File::open(char_path)?,
	File::open(unknown_path)?,
	)?;
	dictionary.write(File::create(output_path)?)?;
	println!("built Vibrato dictionary at {output_path}");
	Ok(())
	}
	[
	command,
	dictionary_flag,
	dictionary_path,
	references_flag,
	references_path,
	candidate_flag,
	candidate_path,
	] if command == "profile"
	&& dictionary_flag == "--dict"
	&& references_flag == "--references"
	&& candidate_flag == "--candidate" =>
	{
	let analyzer =
	VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
	let profile = profile_from_references(&analyzer, references_path)?;
	let candidate = evaluate_text(&analyzer, &fs::read_to_string(candidate_path)?);
	let correction = profile.correction_for(&candidate);
	print_json(&ProfileOutput {
	profile,
	correction,
	})
	}
	[
	command,
	dictionary_flag,
	dictionary_path,
	references_flag,
	references_path,
	candidates_flag,
	candidates_path,
	] if command == "profile-corpus"
	&& dictionary_flag == "--dict"
	&& references_flag == "--references"
	&& candidates_flag == "--candidates" =>
	{
	let analyzer =
	VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
	let profile = profile_from_references(&analyzer, references_path)?;
	let candidates = parse_jsonl(&fs::read_to_string(candidates_path)?)?;
	let candidates = evaluate_corpus(&analyzer, &candidates);
	let records = candidates
	.records
	.into_iter()
	.map(\|record\| ProfileCorpusRecord {
	id: record.id,
	label: record.label,
	source_id: record.source_id,
	author: record.author,
	split_group: record.split_group,
	domain: record.domain,
	correction: profile.correction_for(&record.evaluation),
	})
	.collect();
	print_json(&ProfileCorpusOutput { profile, records })
	}
	[
	command,
	dictionary_flag,
	dictionary_path,
	references_flag,
	references_path,
	before_flag,
	before_path,
	after_flag,
	after_path,
	] if command == "revise"
	&& dictionary_flag == "--dict"
	&& references_flag == "--references"
	&& before_flag == "--before"
	&& after_flag == "--after" =>
	{
	let analyzer =
	VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
	let profile = profile_from_references(&analyzer, references_path)?;
	let before = fs::read_to_string(before_path)?;
	let after = fs::read_to_string(after_path)?;
	let revision = evaluate_revision(&analyzer, &profile, &before, &after);
	print_json(&RevisionOutput { profile, revision })
	}
	[
	command,
	dictionary_flag,
	dictionary_path,
	corpus_flag,
	corpus_path,
	annotations_flag,
	annotations_path,
	] if command == "validate-human"
	&& dictionary_flag == "--dict"
	&& corpus_flag == "--corpus"
	&& annotations_flag == "--annotations" =>
	{
	let analyzer =
	VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
	let corpus = parse_jsonl(&fs::read_to_string(corpus_path)?)?;
	let annotations: Vec<HumanAnnotation> =
	parse_jsonl(&fs::read_to_string(annotations_path)?)?;
	let corpus = evaluate_corpus(&analyzer, &corpus);
	let human_rhythm = correlate_human_rhythm(&corpus.records, &annotations)?;
	print_json(&human_rhythm)
	}
	[
	command,
	dictionary_flag,
	dictionary_path,
	file_flag,
	input_path,
	] if dictionary_flag == "--dict" && file_flag == "--file" => {
	let analyzer =
	VibratoMorphAnalyzer::from_dictionary_reader(File::open(dictionary_path)?)?;
	let input = fs::read_to_string(input_path)?;
	match command.as_str() {
	"analyze" => print_json(&evaluate_text(&analyzer, &input)),
	"corpus" => {
	let corpus = parse_jsonl(&input)?;
	print_json(&evaluate_corpus(&analyzer, &corpus))
	}
	"benchmark" => {
	let corpus = parse_jsonl(&input)?;
	let corpus = evaluate_corpus(&analyzer, &corpus);
	let selection = select_metrics(&corpus.records)?;
	print_json(&BenchmarkOutput { corpus, selection })
	}
	_ => Err(usage().into()),
	}
	}
	_ => Err(usage().into()),
	}
	}

	fn parse_jsonl<T: DeserializeOwned>(input: &str) -> Result<Vec<T>, serde_json::Error> {
	input
	.lines()
	.filter(\|line\| !line.trim().is_empty())
	.map(serde_json::from_str)
	.collect()
	}

	fn profile_from_references(
	analyzer: &VibratoMorphAnalyzer,
	references_path: &str,
	) -> Result<PatternTargetProfile, Box<dyn Error>> {
	let references = parse_jsonl(&fs::read_to_string(references_path)?)?;
	let references = evaluate_corpus(analyzer, &references);
	Ok(PatternTargetProfile::from_references(
	&references
	.records
	.iter()
	.map(\|record\| record.evaluation.clone())
	.collect::<Vec<_>>(),
	))
	}

	fn print_json(value: &impl serde::Serialize) -> Result<(), Box<dyn Error>> {
	println!("{}", serde_json::to_string_pretty(value)?);
	Ok(())
	}

	fn usage() -> String {
	"usage:\n cognitive-rhythm-eval prepare --manifest SOURCES.jsonl --output PASSAGES.jsonl\n cognitive-rhythm-eval build-dictionary --lexicon LEX.csv --matrix matrix.def --char char.def --unknown unk.def --output SYSTEM.dic\n cognitive-rhythm-eval analyze --dict SYSTEM.dic --file TEXT.md\n cognitive-rhythm-eval corpus --dict SYSTEM.dic --file LABELED.jsonl\n cognitive-rhythm-eval benchmark --dict SYSTEM.dic --file LABELED.jsonl\n cognitive-rhythm-eval validate-human --dict SYSTEM.dic --corpus PASSAGES.jsonl --annotations ANNOTATIONS.jsonl\n cognitive-rhythm-eval profile --dict SYSTEM.dic --references REFERENCE.jsonl --candidate CANDIDATE.md\n cognitive-rhythm-eval profile-corpus --dict SYSTEM.dic --references REFERENCE.jsonl --candidates CANDIDATES.jsonl\n cognitive-rhythm-eval revise --dict SYSTEM.dic --references REFERENCE.jsonl --before BEFORE.md --after AFTER.md".to_owned()
	}