Extract a gcgid2unicode conversion table from IBM's cdctables.zip

gcgid2unicode.csv

generate_gcgid_table.py

import json
import re
from cStringIO import StringIO
import sys
import zipfile

#.RPMAP100,.RPMAP101,.RXMAP110,.rxmap100,.RXMAP
TABLE_NAME_RE = re.compile('^[0-9A-F]{8}\.R[PX]MAP(?:\d{3})?$', re.I)

#CP       UCS   GCGID    SYNONYM   ISO 10646 NAME
#A6       00B3  ND031000           SUPERSCRIPT THREE
#A7       00AF  SM150000 SD310000  MACRON 
ROW_RE = re.compile('^{hex}{{2}}{space}{{7}}({hex}{{4}})(?:{space}{{2}}{gcgid})?(?:{space}{gcgid})?'.format(hex='[A-F0-9]', space=' ', gcgid='(?:([A-Z]{2}\d{6})|.{8})'))
assert ROW_RE.match('A6       00B3  ND031000           SUPERSCRIPT THREE').groups() == ('00B3', 'ND031000', None)
assert ROW_RE.match('A7       00AF  SM150000 SD310000  MACRON ').groups() == ('00AF', 'SM150000', 'SD310000')

def package_gcgid_to_ucs_pairs(package_zip):
    for name in package_zip.namelist():
        for result in table_gcgid_to_ucs_pairs(open_zip_from_zip(name, package_zip)):
            yield result

def table_gcgid_to_ucs_pairs(table_zip):
    name = min(n for n in table_zip.namelist() if TABLE_NAME_RE.match(n))
    f = table_zip.open(name)
    for line in f:
        if line.startswith('* Name'):
            if 'Mapping table from ISO 10646 to' in line:
                break
            else:
                return
    for line in f:
        if line.startswith('*') or not line.strip() or line in ['\x1a', '\x7f']:
            continue
        match = ROW_RE.match(line)
        ucs, gcgid, synonym = match.groups()
        if gcgid:
            yield gcgid, ucs
            if synonym:
                yield synonym, ucs

def open_zip_from_zip(name, zip_file):
    # StringIO from .read() instead of .open() because zipfile doesn't play well with it's own .open()'s result
    return zipfile.ZipFile(StringIO(zip_file.read(name)))

def main(cdctables_path):
    gcgid2ucs = {}
    cdctables_zip = zipfile.ZipFile(cdctables_path)
    for name in cdctables_zip.namelist():
        if name.lower().startswith('package1'):
            for gcgid, ucs in package_gcgid_to_ucs_pairs(open_zip_from_zip(name, cdctables_zip)):
                if gcgid in gcgid2ucs:
                    gcgid2ucs[gcgid] == min(gcgid2ucs[gcgid], ucs)
                else:
                    gcgid2ucs[gcgid] = ucs
    for gcgid in sorted(gcgid2ucs):
        print '{},{}'.format(gcgid, gcgid2ucs[gcgid])

if __name__ == '__main__':
    main(*sys.argv[1:])