inspect_unicode.py

inspect_unicode.py

#!/usr/bin/env python3
import unicodedata
import sys

VERBOSE = False

'''
Try with these characters:
が U+304C HIRAGANA LETTER GA (can be decomposed to hiragana か (U+304B) + dakuten ◌゙ (U+3099))
é U+00E9 LATIN SMALL LETTER E WITH ACUTE （can be decomposed to e (U+0065) + accute accent (U+0301))
ǚ U+01DA LATIN SMALL LETTER U WITH DIAERESIS AND CARON （can be decomposed to u (U+0075) + diaeresis (U+0308) + caron (U+030C))

〆 U+3006 IDEOGRAPHIC CLOSING MARK (represents しめ like 〆切)
々 U+3005 IDEOGRAPHIC ITERATION MARK (e.g. 人々, 時々, 様々)
ヶ U+30F6 KATAKANA LETTER SMALL KE (e.g. ヶ月)
ゝ U+309D HIRAGANA ITERATION MARK (e.g. あゝ)
〼 U+303C MASU MARK
〒 U+3012 POSTAL MARK
〱 U+3031 VERTICAL KANA REPEAT MARK

Example output:
Original: é (1 chars, 2 bytes)
hex: ['U+00E9']
bytes: [2]
('LATIN SMALL LETTER E WITH ACUTE', 'Ll')

NFC: é (1 chars, 2 bytes)
hex: ['U+00E9']
bytes: [2]
('LATIN SMALL LETTER E WITH ACUTE', 'Ll')

NFD: é (2 chars, 3 bytes)
hex: ['U+0065', 'U+0301']
bytes: [1, 2]
('LATIN SMALL LETTER E', 'Ll')
('COMBINING ACUTE ACCENT', 'Mn')
'''

def hex(c: str):
    cp = ord(c)
    return f"U+{cp:04X}"


def numbytes(s: str):
    return sum(len(c.encode('utf-8')) for c in s)


def pretty(s: str, cat: str):
    # For combining marks, show with U+25CC DOTTED CIRCLE (◌)
    if cat == 'Mn':
        return '◌' + s
    else:
        return s


def print_info(s: str, prefix: str):
    print(f'{prefix}: {s} ({len(s)} chars, {numbytes(s)} bytes)')
    print('hex:', '[' + ', '.join(hex(c) for c in s) + ']')
    print('bytes:', [numbytes(c) for c in s])

    for c in s:
        cat = unicodedata.category(c)
        print('  {} {} [{}b] {} ({})'.format(
            pretty(c, cat), hex(c), numbytes(c),
            unicodedata.name(c), cat))


def inspect(s: str):
    print_info(s, 'Original')
    print()

    nfc = unicodedata.normalize('NFC', s)
    if VERBOSE or s != nfc:
        print_info(nfc, 'NFC')
    else:
        print('NFC: (same as original)')
    print()

    nfd = unicodedata.normalize('NFD', s)
    if VERBOSE or s != nfd:
        print_info(nfd, 'NFD')
    else:
        print('NFD: (same as original)')
    print()


if __name__ == '__main__':
    if len(sys.argv) >= 2:
        inspect(sys.argv[1])
    else:
        import readline  # noqa: F401
        while True:
            try:
                user_input = input("> ")

                # Safely evaluate and print the user input
                if user_input.strip():
                    result = eval(user_input)
                    if result is None:
                        continue
                    elif isinstance(result, int):
                        c = chr(result)
                        inspect(c)
                    elif isinstance(result, str):
                        inspect(result)
                    else:
                        print(result)

            except KeyboardInterrupt:
                break
            except EOFError:
                break
            except Exception as e:
                print(f"Error: {e}")