Replace full-width (CJK) punctuation with ASCII in Markdown, skipping code blocks and inline code.

fix_fullwidth_punct.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Replace full-width (CJK) punctuation with ASCII equivalents in Markdown.

Intended for English Markdown documents that accidentally contain stray
full-width punctuation copy-pasted from a Chinese source, without
disturbing code blocks or inline code spans.

Features
--------
* Rewrites common full-width punctuation (，。：；？！（）、 "" '' etc.)
  to ASCII equivalents.
* Skips fenced code blocks (``` ... ``` and ~~~ ... ~~~), matching the
  opening fence's marker and length (per CommonMark).
* Skips inline backtick code spans on each line.
* Operates in-place with ``--write``; default is dry-run for safety.
* Preserves the file's trailing newline.
* Accepts any number of files on the command line, and also reads a
  newline-separated list from stdin when ``-`` is passed.

Usage
-----
::

    # Preview changes (no write):
    python3 fix_fullwidth_punct.py doc/en/**/*.md

    # Actually rewrite files:
    python3 fix_fullwidth_punct.py --write doc/en/**/*.md

    # Read file list from stdin:
    git ls-files 'doc/en/*.md' | python3 fix_fullwidth_punct.py --write -

Notes
-----
* Curly / full-width quotation marks map to straight ASCII quotes.
  If a document legitimately uses curly quotes, pass ``--keep-quotes``.
* Full-width ellipsis, em-dash, en-dash, and middle-dot are only
  converted with ``--aggressive`` (they're often intentional).

License: MIT. Review the diff before committing.
"""
from __future__ import annotations

import argparse
import re
import sys
from pathlib import Path
from typing import Iterable

# Conservative map: punctuation whose ASCII equivalent is unambiguous.
BASE_MAPPING: dict[str, str] = {
    '\uff0c': ',',   # ，
    '\u3002': '.',   # 。
    '\uff1a': ':',   # ：
    '\uff1b': ';',   # ；
    '\uff1f': '?',   # ？
    '\uff01': '!',   # ！
    '\uff08': '(',   # （
    '\uff09': ')',   # ）
    '\u3001': ',',   # 、 (enumeration comma -> ASCII comma)
    '\u300a': '<',   # 《
    '\u300b': '>',   # 》
    '\u3010': '[',   # 【
    '\u3011': ']',   # 】
}

# Quote-like characters. Disabled via --keep-quotes.
QUOTE_MAPPING: dict[str, str] = {
    '\u201c': '"',   # “
    '\u201d': '"',   # ”
    '\u2018': "'",   # ‘
    '\u2019': "'",   # ’
    '\u300c': '"',   # 「
    '\u300d': '"',   # 」
    '\u300e': '"',   # 『
    '\u300f': '"',   # 』
}

# Enabled only with --aggressive.
AGGRESSIVE_MAPPING: dict[str, str] = {
    '\u2026': '...',  # …  (horizontal ellipsis)
    '\u2014': '--',   # —  (em dash)
    '\u2013': '-',    # –  (en dash)
    '\u00b7': '.',    # ·  (middle dot)
}

FENCE_RE = re.compile(r'^(\s*)(`{3,}|~{3,})')
INLINE_CODE_RE = re.compile(r'`[^`\n]+`')


def build_mapping(keep_quotes: bool, aggressive: bool) -> dict[str, str]:
    mapping: dict[str, str] = dict(BASE_MAPPING)
    if not keep_quotes:
        mapping.update(QUOTE_MAPPING)
    if aggressive:
        mapping.update(AGGRESSIVE_MAPPING)
    return mapping


def _translate(segment: str, mapping: dict[str, str]) -> str:
    # Fast path when every replacement is a single character.
    if all(len(v) == 1 for v in mapping.values()):
        return segment.translate({ord(k): v for k, v in mapping.items()})
    for fw, asc in mapping.items():
        segment = segment.replace(fw, asc)
    return segment


def transform_line(line: str, mapping: dict[str, str]) -> str:
    """Transform a line, skipping inline backtick code spans."""
    parts: list[str] = []
    last = 0
    for m in INLINE_CODE_RE.finditer(line):
        parts.append(_translate(line[last:m.start()], mapping))
        parts.append(m.group(0))
        last = m.end()
    parts.append(_translate(line[last:], mapping))
    return ''.join(parts)


def transform_text(text: str, mapping: dict[str, str]) -> tuple[str, int]:
    """Return ``(new_text, number_of_changed_lines)``.

    Preserves any trailing newline in ``text``.
    """
    had_trailing_newline = text.endswith('\n')
    lines = text.split('\n')
    if had_trailing_newline:
        lines = lines[:-1]  # drop synthetic empty tail

    out: list[str] = []
    changed = 0
    in_fence = False
    fence_marker = ''  # exact run of ` or ~ that opened the fence

    for line in lines:
        m = FENCE_RE.match(line)
        if m:
            marker = m.group(2)
            if not in_fence:
                in_fence = True
                fence_marker = marker
            else:
                # Closing fence must use the same char and be at least
                # as long as the opening fence (CommonMark 4.5).
                stripped = line.strip()
                if (stripped
                        and set(stripped) <= {fence_marker[0]}
                        and len(stripped) >= len(fence_marker)):
                    in_fence = False
                    fence_marker = ''
            out.append(line)
            continue
        if in_fence:
            out.append(line)
            continue
        new = transform_line(line, mapping)
        if new != line:
            changed += 1
        out.append(new)

    new_text = '\n'.join(out)
    if had_trailing_newline:
        new_text += '\n'
    return new_text, changed


def process_file(path: Path, mapping: dict[str, str], write: bool) -> int:
    src = path.read_text(encoding='utf-8')
    new, changed = transform_text(src, mapping)
    if changed and write:
        path.write_text(new, encoding='utf-8')
    return changed


def iter_paths(args_paths: Iterable[str]) -> Iterable[Path]:
    for p in args_paths:
        if p == '-':
            for line in sys.stdin:
                line = line.rstrip('\n')
                if line:
                    yield Path(line)
        else:
            yield Path(p)


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description='Replace full-width CJK punctuation with ASCII in Markdown.',
    )
    parser.add_argument('paths', nargs='+',
                        help='Markdown files to process. Use "-" to read a '
                             'newline-separated list from stdin.')
    parser.add_argument('-w', '--write', action='store_true',
                        help='Rewrite files in place. Without this, the '
                             'script runs in dry-run mode.')
    parser.add_argument('--keep-quotes', action='store_true',
                        help='Do not convert curly/Chinese quotation marks.')
    parser.add_argument('--aggressive', action='store_true',
                        help='Also convert ellipsis / em-dash / en-dash / '
                             'middle-dot (often intentional; use with care).')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Print one line per file, including unchanged ones.')
    opts = parser.parse_args(argv)

    mapping = build_mapping(opts.keep_quotes, opts.aggressive)
    total_lines = 0
    total_files = 0

    for path in iter_paths(opts.paths):
        try:
            changed = process_file(path, mapping, write=opts.write)
        except FileNotFoundError:
            print(f'skip (not found): {path}', file=sys.stderr)
            continue
        except OSError as exc:
            print(f'skip ({exc}): {path}', file=sys.stderr)
            continue

        if changed:
            total_lines += changed
            total_files += 1
            tag = '[written]' if opts.write else '[dry-run]'
            print(f'{path}: {changed} line(s) {tag}')
        elif opts.verbose:
            print(f'{path}: 0 line(s)')

    suffix = ('written' if opts.write
              else 'would change (dry-run; pass --write to apply)')
    print(f'TOTAL: {total_lines} line(s) across {total_files} file(s) {suffix}')
    return 0


if __name__ == '__main__':
    sys.exit(main())