disarm #937: Boucher's four classes scored one at a time (canonicalize recovers 14 of 4,800 reorderings, 0 of 4,820 deletions) and a cell-aware cursor model that recovers 4,820 of 4,820

bad_characters_by_class.py

#!/usr/bin/env python3
"""disarm #937: Boucher et al.'s four classes scored one at a time, and a cursor model
for the deletion class.

The meta-benchmark scores the released corpus as one population. Split by the
experiment key, the aggregate is the sum of two classes near 100%, one near 0%, and
one at 0% — and the two near zero are the two whose *rendering* is the clean string.

Corpus: https://raw.githubusercontent.com/nickboucher/imperceptible/main/results/adversarial-examples.json
(MIT). Nested experiment -> budget -> row -> {"adv_example", "input", ...}. Pass the path
as argv[1] or set DISARM_META_CACHE; the default is the meta-benchmark's cache name.

    pip install disarm==0.15.0
    python bad_characters_by_class.py /path/to/bad-characters.json

Expected output on disarm 0.15.0 at 0b8ddbf:

    class        perturbed  canonicalize  strip_bidi==input  cursor model
    invisible         4800          4800                  -             -
    homoglyph         4750          3492                  -             -
    reorder           4800            14                 14             -
    deletion          4820             0                  -          4820
    unperturbed       3200          3200                  -             -

    11506 of 22370 = 51.4% (the baseline's xmr_best_surface figure)
    deletion rows carrying BS 4820, DEL 0, CR 0; inputs changed by the model: 0
    cursor model idempotent on every deletion row: True
"""

from __future__ import annotations

import json
import os
import sys
import unicodedata
from collections import Counter, defaultdict

import disarm
from disarm import canonicalize, strip_bidi

BS, DEL, CR, LF = "\x08", "\x7f", "\r", "\n"
BIDI = set("\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069\u200e\u200f")  # escapes, not literals (#802)


def occupies_cell(ch: str) -> bool:
    """A combining mark joins the cell before it; a format character occupies none."""
    cat = unicodedata.category(ch)
    return not (cat.startswith("M") or cat == "Cf")


def resolve_deletions(text: str, *, cr: bool = False) -> str:
    """The paper's renderer as a cursor over cells, not a stack over code points.

    BS and DEL erase the previous *cell* — a base character with its marks and any
    format characters attached to it. A CR followed by LF, or at end of text, is a line
    ending and passes through. A CR followed by anything else returns the cursor to
    column 0 so later text overwrites earlier text, and only when ``cr=True``: a classic
    Mac OS line ending is byte-identical to that overwrite, so it is a separate decision.
    """
    out: list[str] = []
    line: list[str] = []
    col = 0
    n = len(text)
    for i, ch in enumerate(text):
        if ch in (BS, DEL):
            col = max(0, col - 1)
            del line[col:]
        elif ch == LF or (ch == CR and (i + 1 == n or text[i + 1] == LF or not cr)):
            out.extend(line)
            out.append(ch)
            line, col = [], 0
        elif ch == CR:
            col = 0
        elif not occupies_cell(ch) and col > 0:
            line[col - 1] += ch
        elif col < len(line):
            line[col] = ch
            col += 1
        else:
            line.append(ch)
            col += 1
    out.extend(line)
    return "".join(out)


def check_model() -> None:
    join = lambda w: "".join(ch + "X" + BS for ch in w)  # noqa: E731
    assert resolve_deletions(join("paypal")) == "paypal"
    assert resolve_deletions("line1\r\nline2\r\nline3") == "line1\r\nline2\r\nline3"
    assert resolve_deletions("line1\r\nline2", cr=True) == "line1\r\nline2"
    assert resolve_deletions("ZZZZZZ\rpaypal") == "ZZZZZZ\rpaypal", "CR off: untouched"
    assert resolve_deletions("ZZZZZZ\rpaypal", cr=True) == "paypal"
    assert resolve_deletions("abc\rxy", cr=True) == "xyc", "overwrite, not clear"
    assert resolve_deletions("line1\rline2", cr=True) == "line2", "a classic Mac file"
    assert resolve_deletions("c\x08c") == "c", "man-page overstrike bold"
    assert resolve_deletions("_\x08c") == "c", "man-page overstrike underline"
    assert resolve_deletions("X\u200b\x08") == "", "a zero-width joins the cell"
    assert resolve_deletions("é\x08") == "", "a mark joins the cell"
    assert resolve_deletions("\x08\x08a") == "a"


def main() -> int:
    path = (
        sys.argv[1]
        if len(sys.argv) > 1
        else os.path.join(os.environ.get("DISARM_META_CACHE", "/tmp/disarm_meta_cache"), "bad-characters.json")
    )
    check_model()
    blob = json.load(open(path, encoding="utf-8"))
    per: dict[str, Counter] = defaultdict(Counter)
    idem = True
    for experiment, budgets in blob.items():
        cls = next((t for t in ("deletion", "homoglyph", "invisible", "reorder") if t in experiment), "?")
        for rows in budgets.values():
            for row in rows.values():
                adv, inp = row.get("adv_example"), row.get("input")
                if not isinstance(adv, str) or not adv.strip():
                    continue
                c = "unperturbed" if adv == inp else cls
                s = per[c]
                s["perturbed"] += 1
                out = canonicalize(adv)
                if out and out == canonicalize(inp):
                    s["canonicalize"] += 1
                if c == "reorder" and strip_bidi(adv) == inp:
                    s["strip_bidi==input"] += 1
                if c == "deletion":
                    for label, ctl in (("BS", BS), ("DEL", DEL), ("CR", CR)):
                        if ctl in adv:
                            s[label] += 1
                    got = resolve_deletions(adv)
                    if got == inp:
                        s["cursor model"] += 1
                    idem &= resolve_deletions(got) == got
                    if resolve_deletions(inp) != inp:
                        s["inputs changed"] += 1
    print(f"disarm {disarm.__version__}\n")
    print(f"{'class':<12}{'perturbed':>10}{'canonicalize':>14}{'strip_bidi==input':>19}{'cursor model':>14}")
    total = recovered = 0
    for c in ("invisible", "homoglyph", "reorder", "deletion", "unperturbed"):
        s = per[c]
        total += s["perturbed"]
        recovered += s["canonicalize"]
        cell = lambda k: str(s[k]) if k in s else "-"  # noqa: E731
        print(f"{c:<12}{s['perturbed']:>10}{s['canonicalize']:>14}{cell('strip_bidi==input'):>19}{cell('cursor model'):>14}")
    d = per["deletion"]
    print(f"\n{recovered} of {total} = {recovered / total:.1%} (the baseline's xmr_best_surface figure)")
    print(f"deletion rows carrying BS {d['BS']}, DEL {d['DEL']}, CR {d['CR']}; inputs changed by the model: {d['inputs changed']}")
    print(f"cursor model idempotent on every deletion row: {idem}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())