make a single interactive plotly QC plot from the PacBio Hifi Pipeline Output

qc_plots.py

#!/usr/bin/env python3
"""Create an interactive Plotly QC report from stats TSV files.

Usage:
    python3 plot_stats.py stats/*.stats.txt -o qc_stats.html

Dependencies:
    uv pip install -r requirements.txt --python .venv/bin/python
"""

from __future__ import annotations

import argparse
import html
import json
from pathlib import Path
from typing import Iterable

import pandas as pd
from plotly.offline import get_plotlyjs


SOURCE_COLUMN = "source_file"
ROW_COLUMN = "row"
CHART_COLOR = "#2f6f73"
STRIP_COLORS = (
    "#2f6f73",
    "#d9822b",
    "#6b5b95",
    "#4c78a8",
    "#b5525c",
    "#58a55c",
    "#8f6d45",
    "#6f7c80",
)
SEX_COLORS = {
    "FEMALE": "#b5525c",
    "MALE": "#4c78a8",
    "Unknown": "#6f7c80",
}


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Build an interactive Plotly histogram report for stats files."
    )
    parser.add_argument(
        "files",
        nargs="+",
        type=Path,
        help="Tab-delimited stats files with a header row.",
    )
    parser.add_argument(
        "-o",
        "--output",
        type=Path,
        default=Path("qc_stats.html"),
        help="Output HTML file. Default: qc_stats.html",
    )
    parser.add_argument(
        "--bins",
        type=int,
        default=24,
        help="Approximate number of histogram bins for numeric columns. Default: 24",
    )
    parser.add_argument(
        "--max-categories",
        type=int,
        default=30,
        help="Maximum unique values for a non-numeric column to be plotted as counts. Default: 30",
    )
    parser.add_argument(
        "--title",
        default="QC Stats Explorer",
        help="Report title. Default: QC Stats Explorer",
    )
    return parser.parse_args()


def load_stats(files: Iterable[Path]) -> pd.DataFrame:
    frames: list[pd.DataFrame] = []

    for file_path in files:
        if not file_path.exists():
            raise FileNotFoundError(file_path)

        frame = pd.read_csv(file_path, sep="\t")
        frame.insert(0, SOURCE_COLUMN, file_path.stem)
        frames.append(frame)

    if not frames:
        raise ValueError("No input files were provided.")

    combined = pd.concat(frames, ignore_index=True, sort=False)
    combined.insert(1, ROW_COLUMN, range(1, len(combined) + 1))
    return combined


def classify_columns(
    data: pd.DataFrame, max_categories: int
) -> tuple[list[str], list[str], pd.DataFrame]:
    numeric_columns: list[str] = []
    categorical_columns: list[str] = []
    converted = data.copy()

    ignored = {SOURCE_COLUMN, ROW_COLUMN, "sample_id"}
    for column in data.columns:
        if column in ignored:
            continue

        numeric_values = pd.to_numeric(data[column], errors="coerce")
        non_missing_count = data[column].notna().sum()
        if non_missing_count > 0 and numeric_values.notna().sum() == non_missing_count:
            converted[column] = numeric_values
            numeric_columns.append(column)
            continue

        unique_count = data[column].dropna().nunique()
        if 1 < unique_count <= max_categories:
            categorical_columns.append(column)

    return numeric_columns, categorical_columns, converted


def format_table_value(value: object, numeric: bool) -> str:
    if pd.isna(value):
        return ""
    if not numeric:
        return str(value)

    number = float(value)
    if number.is_integer():
        return f"{int(number):,}"
    return f"{number:,.4f}".rstrip("0").rstrip(".")


def table_payload(data: pd.DataFrame, column: str) -> dict[str, list[list[str]] | list[str]]:
    columns = [SOURCE_COLUMN, ROW_COLUMN]
    if "sample_id" in data.columns:
        columns.append("sample_id")
    columns.append(column)

    headers = [
        "File" if name == SOURCE_COLUMN else "Row" if name == ROW_COLUMN else name
        for name in columns
    ]
    rows: list[list[str]] = []

    for _, record in data[columns].iterrows():
        row: list[str] = []
        for table_column in columns:
            is_numeric = pd.api.types.is_numeric_dtype(data[table_column])
            row.append(format_table_value(record[table_column], is_numeric))
        rows.append(row)

    return {"headers": headers, "rows": rows}


def sample_payload(data: pd.DataFrame, column: str, numeric: bool) -> list[dict[str, object]]:
    samples: list[dict[str, object]] = []
    sample_ids = data["sample_id"] if "sample_id" in data.columns else data[ROW_COLUMN]

    for index, record in data.iterrows():
        value = record[column]
        if pd.isna(value):
            continue

        if numeric:
            plot_value: float | str = float(value)
        else:
            plot_value = str(value)

        sex = record.get("inferred_sex", "Unknown")
        if pd.isna(sex) or not str(sex).strip():
            sex = "Unknown"

        samples.append(
            {
                "sample": str(sample_ids.iloc[index]),
                "file": str(record[SOURCE_COLUMN]),
                "sex": str(sex),
                "value": plot_value,
                "display": format_table_value(value, numeric),
            }
        )

    return samples


def column_payloads(
    data: pd.DataFrame, numeric_columns: list[str], categorical_columns: list[str]
) -> list[dict[str, object]]:
    payloads: list[dict[str, object]] = []

    for column in numeric_columns:
        values = [
            float(value)
            for value in data[column].dropna().to_list()
        ]
        payloads.append(
            {
                "name": column,
                "kind": "numeric",
                "values": values,
                "samples": sample_payload(data, column, numeric=True),
                "table": table_payload(data, column),
            }
        )

    for column in categorical_columns:
        counts = (
            data[column]
            .fillna("Missing")
            .astype(str)
            .value_counts()
            .sort_index()
        )
        payloads.append(
            {
                "name": column,
                "kind": "categorical",
                "categories": counts.index.to_list(),
                "counts": [int(value) for value in counts.values],
                "samples": sample_payload(data, column, numeric=False),
                "table": table_payload(data, column),
            }
        )

    return payloads


def report_html(data: pd.DataFrame, columns: list[dict[str, object]], bins: int, title: str) -> str:
    if not columns:
        raise ValueError("No plottable columns were found.")

    payload = {
        "title": title,
        "bins": bins,
        "rowCount": len(data),
        "fileCount": int(data[SOURCE_COLUMN].nunique()),
        "chartColor": CHART_COLOR,
        "stripColors": STRIP_COLORS,
        "sexColors": SEX_COLORS,
        "hasSex": "inferred_sex" in data.columns,
        "columns": columns,
    }
    payload_json = json.dumps(payload, separators=(",", ":")).replace("</", "<\\/")
    escaped_title = html.escape(title)
    plotly_js = get_plotlyjs()

    return f"""<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>{escaped_title}</title>
  <style>
    :root {{
      color-scheme: light;
      --bg: #eef3f1;
      --surface: #ffffff;
      --ink: #1f2933;
      --muted: #52606d;
      --line: #c8d5d2;
      --line-soft: #e4ece9;
      --accent: {CHART_COLOR};
      --accent-dark: #1f4f52;
      --row: #f8fbfa;
      --row-alt: #eef5f3;
    }}

    * {{
      box-sizing: border-box;
    }}

    body {{
      margin: 0;
      background: var(--bg);
      color: var(--ink);
      font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
    }}

    .page {{
      width: min(1320px, calc(100vw - 40px));
      margin: 28px auto 40px;
    }}

    .topbar {{
      display: grid;
      grid-template-columns: minmax(0, 1fr) auto;
      gap: 20px;
      align-items: end;
      margin-bottom: 16px;
    }}

    h1 {{
      margin: 0;
      font-size: 28px;
      line-height: 1.15;
      font-weight: 760;
      letter-spacing: 0;
    }}

    .meta {{
      margin-top: 6px;
      color: var(--muted);
      font-size: 13px;
    }}

    .control {{
      display: grid;
      gap: 6px;
      min-width: 280px;
    }}

    .controls {{
      display: flex;
      gap: 12px;
      align-items: end;
    }}

    .color-control {{
      display: grid;
      gap: 6px;
    }}

    label {{
      color: var(--muted);
      font-size: 12px;
      font-weight: 650;
      text-transform: uppercase;
      letter-spacing: 0.04em;
    }}

    select {{
      appearance: none;
      border: 1px solid var(--line);
      border-radius: 6px;
      background:
        linear-gradient(45deg, transparent 50%, var(--accent-dark) 50%),
        linear-gradient(135deg, var(--accent-dark) 50%, transparent 50%),
        var(--surface);
      background-position:
        calc(100% - 18px) 52%,
        calc(100% - 12px) 52%,
        0 0;
      background-size: 6px 6px, 6px 6px, 100% 100%;
      background-repeat: no-repeat;
      color: var(--ink);
      font-size: 14px;
      line-height: 1.2;
      padding: 10px 36px 10px 12px;
      min-height: 40px;
    }}

    .segmented {{
      display: inline-flex;
      min-height: 40px;
      padding: 3px;
      border: 1px solid var(--line);
      border-radius: 6px;
      background: var(--surface);
    }}

    .segmented button {{
      border: 0;
      border-radius: 4px;
      background: transparent;
      color: var(--muted);
      cursor: pointer;
      font: inherit;
      font-size: 13px;
      font-weight: 680;
      padding: 7px 12px;
    }}

    .segmented button:hover {{
      color: var(--accent-dark);
    }}

    .segmented button[aria-pressed="true"] {{
      background: var(--accent);
      color: #ffffff;
    }}

    .segmented button:focus-visible {{
      outline: 2px solid var(--accent-dark);
      outline-offset: 1px;
    }}

    .chart-panel,
    .table-panel {{
      background: var(--surface);
      border: 1px solid var(--line-soft);
      border-radius: 8px;
      box-shadow: 0 10px 28px rgba(31, 41, 51, 0.08);
    }}

    .chart-panel {{
      padding: 8px 10px 0;
    }}

    #chart {{
      width: 100%;
      height: 540px;
    }}

    .table-panel {{
      margin-top: 16px;
      overflow: hidden;
    }}

    .table-head {{
      display: flex;
      justify-content: space-between;
      gap: 16px;
      align-items: center;
      padding: 13px 16px;
      border-bottom: 1px solid var(--line-soft);
    }}

    .table-title {{
      font-size: 15px;
      font-weight: 720;
    }}

    .table-meta {{
      color: var(--muted);
      font-size: 12px;
      white-space: nowrap;
    }}

    .table-scroll {{
      max-height: 460px;
      overflow: auto;
    }}

    table {{
      width: 100%;
      border-collapse: collapse;
      font-size: 12px;
    }}

    thead th {{
      position: sticky;
      top: 0;
      z-index: 1;
      background: #203a43;
      color: #fff;
      text-align: left;
      font-weight: 690;
      padding: 9px 12px;
      border-right: 1px solid rgba(255, 255, 255, 0.15);
      cursor: pointer;
      user-select: none;
      white-space: nowrap;
    }}

    thead th::after {{
      content: "↕";
      display: inline-block;
      margin-left: 8px;
      color: rgba(255, 255, 255, 0.58);
      font-size: 10px;
    }}

    thead th.sort-asc::after {{
      content: "↑";
      color: #ffffff;
    }}

    thead th.sort-desc::after {{
      content: "↓";
      color: #ffffff;
    }}

    tbody td {{
      padding: 7px 12px;
      border-top: 1px solid var(--line-soft);
      color: var(--ink);
      white-space: nowrap;
    }}

    tbody tr:nth-child(odd) td {{
      background: var(--row);
    }}

    tbody tr:nth-child(even) td {{
      background: var(--row-alt);
    }}

    @media (max-width: 780px) {{
      .page {{
        width: min(100vw - 24px, 1320px);
        margin-top: 16px;
      }}

      .topbar {{
        grid-template-columns: 1fr;
        align-items: stretch;
      }}

      .control {{
        min-width: 0;
      }}

      .controls {{
        align-items: stretch;
        flex-direction: column;
      }}

      .segmented {{
        align-self: start;
      }}

      #chart {{
        height: 500px;
      }}
    }}
  </style>
</head>
<body>
  <main class="page">
    <div class="topbar">
      <div>
        <h1>{escaped_title}</h1>
        <div class="meta"><span id="selected-column"></span> · {len(data):,} rows · {data[SOURCE_COLUMN].nunique():,} files</div>
      </div>
      <div class="controls">
        <div class="control">
          <label for="column-select">Column</label>
          <select id="column-select"></select>
        </div>
        <div class="color-control" id="color-control">
          <label id="color-mode-label">Color points by</label>
          <div class="segmented" role="group" aria-labelledby="color-mode-label">
            <button type="button" data-color-mode="family" aria-pressed="true">Family</button>
            <button type="button" data-color-mode="sex" aria-pressed="false">Sex</button>
          </div>
        </div>
      </div>
    </div>

    <section class="chart-panel">
      <div id="chart"></div>
    </section>

    <section class="table-panel">
      <div class="table-head">
        <div class="table-title">Values</div>
        <div class="table-meta" id="table-meta"></div>
      </div>
      <div class="table-scroll">
        <table id="values-table"></table>
      </div>
    </section>
  </main>

  <script>{plotly_js}</script>
  <script id="qc-data" type="application/json">{payload_json}</script>
  <script>
    const payload = JSON.parse(document.getElementById("qc-data").textContent);
    const select = document.getElementById("column-select");
    const selectedColumn = document.getElementById("selected-column");
    const tableMeta = document.getElementById("table-meta");
    const valuesTable = document.getElementById("values-table");
    const colorControl = document.getElementById("color-control");
    const colorButtons = Array.from(document.querySelectorAll("[data-color-mode]"));
    let activeSort = {{ columnIndex: null, direction: "asc" }};
    let colorMode = "family";

    colorControl.hidden = !payload.hasSex;

    for (const [index, column] of payload.columns.entries()) {{
      const option = document.createElement("option");
      option.value = String(index);
      option.textContent = column.name;
      select.appendChild(option);
    }}

    function escapeCell(value) {{
      return String(value)
        .replaceAll("&", "&amp;")
        .replaceAll("<", "&lt;")
        .replaceAll(">", "&gt;")
        .replaceAll('"', "&quot;")
        .replaceAll("'", "&#039;");
    }}

    function jitterPosition(sample) {{
      const key = sample.sample + "|" + sample.file;
      let hash = 0;
      for (let index = 0; index < key.length; index += 1) {{
        hash = ((hash * 31) + key.charCodeAt(index)) >>> 0;
      }}
      return 0.12 + ((hash % 1000) / 1000) * 0.76;
    }}

    function stripTraces(column) {{
      const groupKey = colorMode === "sex" ? "sex" : "file";
      const groups = Array.from(new Set(column.samples.map((sample) => sample[groupKey]))).sort();
      return groups.map((group, index) => {{
        const samples = column.samples.filter((sample) => sample[groupKey] === group);
        const color = colorMode === "sex"
          ? (payload.sexColors[group] ?? payload.stripColors[index % payload.stripColors.length])
          : payload.stripColors[index % payload.stripColors.length];
        return {{
          type: "scatter",
          mode: "markers",
          name: group,
          legendgroup: group,
          x: samples.map((sample) => jitterPosition(sample)),
          y: samples.map((sample) => sample.value),
          customdata: samples.map((sample) => [sample.sample, sample.file, sample.sex, sample.display]),
          xaxis: "x2",
          yaxis: "y2",
          marker: {{
            color: color,
            size: 8,
            opacity: 0.86,
            line: {{ color: "#ffffff", width: 0.7 }}
          }},
          hovertemplate: "<b>%{{customdata[0]}}</b><br>Family: %{{customdata[1]}}<br>Sex: %{{customdata[2]}}<br>" + column.name + ": %{{customdata[3]}}<extra></extra>"
        }};
      }});
    }}

    function tracesFor(column) {{
      if (column.kind === "numeric") {{
        return [
          {{
            type: "histogram",
            y: column.values,
            nbinsy: payload.bins,
            orientation: "h",
            xaxis: "x",
            yaxis: "y",
            marker: {{
              color: payload.chartColor,
              line: {{ color: "#1f4f52", width: 0.5 }}
            }},
            opacity: 0.9,
            showlegend: false,
            hovertemplate: "Count: %{{x}}<br><b>" + column.name + "</b>: %{{y}}<extra></extra>"
          }}
        ].concat(stripTraces(column));
      }}

      return [
        {{
          type: "bar",
          x: column.counts,
          y: column.categories,
          orientation: "h",
          xaxis: "x",
          yaxis: "y",
          marker: {{
            color: payload.chartColor,
            line: {{ color: "#1f4f52", width: 0.5 }}
          }},
          showlegend: false,
          hovertemplate: "Count: %{{x}}<br><b>" + column.name + "</b>: %{{y}}<extra></extra>"
        }}
      ].concat(stripTraces(column));
    }}

    function layoutFor(column) {{
      return {{
        title: {{
          text: column.kind === "numeric" ? "Histogram and sample strip" : "Counts and sample strip",
          x: 0,
          xanchor: "left",
          font: {{ size: 17, color: "#1f2933" }}
        }},
        height: 540,
        margin: {{ l: 118, r: 34, t: 72, b: 78 }},
        paper_bgcolor: "#ffffff",
        plot_bgcolor: "#ffffff",
        bargap: column.kind === "numeric" ? 0.04 : 0.18,
        showlegend: true,
        legend: {{
          title: {{ text: colorMode === "sex" ? "Sex" : "Family" }},
          orientation: "h",
          x: 1,
          xanchor: "right",
          y: 1.14,
          yanchor: "top",
          bgcolor: "rgba(255,255,255,0)",
          font: {{ size: 12 }}
        }},
        font: {{
          family: "Inter, Arial, sans-serif",
          color: "#243b53"
        }},
        xaxis: {{
          domain: [0, 0.86],
          anchor: "y",
          title: {{ text: "Sample count", standoff: 18, font: {{ size: 18 }} }},
          tickfont: {{ size: 15 }},
          ticks: "outside",
          automargin: true,
          showline: true,
          linecolor: "#9fb3b2",
          gridcolor: "#edf2f0",
          zeroline: false
        }},
        yaxis: {{
          anchor: "x",
          title: {{ text: column.name, standoff: 16, font: {{ size: 18 }} }},
          tickfont: {{ size: 15 }},
          ticks: "outside",
          automargin: true,
          showline: true,
          linecolor: "#9fb3b2",
          gridcolor: "#e4ece9",
          zeroline: false
        }},
        xaxis2: {{
          domain: [0.9, 1],
          anchor: "y2",
          title: {{ text: "Samples", standoff: 18, font: {{ size: 18 }} }},
          range: [0, 1],
          fixedrange: true,
          showticklabels: false,
          ticks: "",
          showgrid: false,
          showline: true,
          linecolor: "#9fb3b2",
          zeroline: false
        }},
        yaxis2: {{
          anchor: "x2",
          matches: "y",
          showticklabels: false,
          ticks: "",
          automargin: true,
          showline: true,
          linecolor: "#9fb3b2",
          gridcolor: "#edf2f0",
          zeroline: false
        }}
      }};
    }}

    function renderTable(column) {{
      activeSort = {{ columnIndex: null, direction: "asc" }};
      const headers = column.table.headers
        .map((header, index) => "<th data-column-index=\\"" + index + "\\" title=\\"Sort by " + escapeCell(header) + "\\">" + escapeCell(header) + "</th>")
        .join("");
      const rows = column.table.rows
        .map((row) => "<tr>" + row.map((value) => "<td>" + escapeCell(value) + "</td>").join("") + "</tr>")
        .join("");

      valuesTable.innerHTML = "<thead><tr>" + headers + "</tr></thead><tbody>" + rows + "</tbody>";
      tableMeta.textContent = column.table.rows.length.toLocaleString() + " rows";
      valuesTable.querySelectorAll("th").forEach((header) => {{
        header.addEventListener("click", () => sortTable(Number(header.dataset.columnIndex)));
      }});
    }}

    function comparableValue(value) {{
      const normalized = String(value).replaceAll(",", "").trim();
      if (normalized !== "" && !Number.isNaN(Number(normalized))) {{
        return {{ type: "number", value: Number(normalized) }};
      }}
      return {{ type: "text", value: String(value).toLocaleLowerCase() }};
    }}

    function compareCells(left, right) {{
      const a = comparableValue(left);
      const b = comparableValue(right);
      if (a.type === "number" && b.type === "number") {{
        return a.value - b.value;
      }}
      return String(a.value).localeCompare(String(b.value), undefined, {{
        numeric: true,
        sensitivity: "base"
      }});
    }}

    function sortTable(columnIndex) {{
      const tbody = valuesTable.querySelector("tbody");
      const headers = valuesTable.querySelectorAll("th");
      const direction =
        activeSort.columnIndex === columnIndex && activeSort.direction === "asc"
          ? "desc"
          : "asc";
      const multiplier = direction === "asc" ? 1 : -1;
      const rows = Array.from(tbody.querySelectorAll("tr"));

      rows.sort((leftRow, rightRow) => {{
        const left = leftRow.children[columnIndex]?.textContent ?? "";
        const right = rightRow.children[columnIndex]?.textContent ?? "";
        return compareCells(left, right) * multiplier;
      }});

      tbody.replaceChildren(...rows);
      activeSort = {{ columnIndex, direction }};
      headers.forEach((header) => header.classList.remove("sort-asc", "sort-desc"));
      headers[columnIndex].classList.add(direction === "asc" ? "sort-asc" : "sort-desc");
    }}

    function renderChart(index) {{
      const column = payload.columns[index];
      selectedColumn.textContent = column.name;
      Plotly.react("chart", tracesFor(column), layoutFor(column), {{
        responsive: true,
        displaylogo: false
      }});
    }}

    function render(index) {{
      const column = payload.columns[index];
      renderChart(index);
      renderTable(column);
    }}

    select.addEventListener("change", () => render(Number(select.value)));
    colorButtons.forEach((button) => {{
      button.addEventListener("click", () => {{
        colorMode = button.dataset.colorMode;
        colorButtons.forEach((candidate) => {{
          candidate.setAttribute("aria-pressed", String(candidate === button));
        }});
        renderChart(Number(select.value));
      }});
    }});
    render(0);
  </script>
</body>
</html>
"""


def main() -> None:
    args = parse_args()
    data = load_stats(args.files)
    numeric_columns, categorical_columns, converted = classify_columns(
        data, args.max_categories
    )
    columns = column_payloads(converted, numeric_columns, categorical_columns)

    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(
        report_html(converted, columns=columns, bins=args.bins, title=args.title),
        encoding="utf-8",
    )
    print(f"Wrote {args.output}")
    print(f"Numeric histogram columns: {len(numeric_columns)}")
    print(f"Categorical count columns: {len(categorical_columns)}")


if __name__ == "__main__":
    main()