TWEWY font datamining and extraction

convert_font_sheet.py

#!/usr/bin/env python3

import sys
from PIL import Image

infn = sys.argv[1]
outfn = sys.argv[2]

with open(infn, 'rb') as f:
    indata = f.read()

assert len(indata) == 256 * 256 / 2

im = Image.new('RGBA', (256, 256))
# print(im)

for ytile in range(32):
    for xtile in range(32):
        for ypixel in range(8):
            for xpixelbyte in range(4):
                off = (ytile * 32 + xtile) * 0x20
                off += ypixel * 4 + xpixelbyte
                b = indata[off]

                p0 = b & 0xF
                p1 = (b >> 4) & 0xF
                assert p0 == 0 or p0 == 14 or p0 == 15
                assert p1 == 0 or p1 == 14 or p1 == 15

                def px_to_color(p):
                    if p == 15:
                        return (255, 255, 255, 255)
                    elif p == 14:
                        return (127, 127, 127, 255)
                    else:
                        return (0, 0, 0, 0)

                youtpx = ytile * 8 + ypixel
                xoutpx = xtile * 8 + xpixelbyte * 2

                im.putpixel((xoutpx + 0, youtpx), px_to_color(p0))
                im.putpixel((xoutpx + 1, youtpx), px_to_color(p1))

im.save(outfn)

slice_font.py

#!/usr/bin/env python3

import sys
from PIL import Image

fontnum = sys.argv[1]
outdir = sys.argv[2]

if fontnum == "5":
    CHARHEIGHT = 16
    CHARWIDTH = 16
    SHEETS = 3
    IMG_FN_TMPL = 'font-5-{}.png'
    WIDTHBIN_FN_TMPL = 'fontfont/1/{}.bin'
elif fontnum == "6":
    CHARHEIGHT = 10
    CHARWIDTH = 10
    SHEETS = 4
    IMG_FN_TMPL = 'font-6-{}.png'
    WIDTHBIN_FN_TMPL = 'fontfont/2/{}.bin'
elif fontnum == "7":
    CHARHEIGHT = 16
    CHARWIDTH = 16
    SHEETS = 2
    IMG_FN_TMPL = 'font-7-{}.png'
    WIDTHBIN_FN_TMPL = 'fontfont/3/{}.bin'
elif fontnum == "9":
    CHARHEIGHT = 12
    CHARWIDTH = 10
    SHEETS = 2
    IMG_FN_TMPL = 'font-9-{}.png'
    WIDTHBIN_FN_TMPL = 'fontfont/8/{}.bin'
else:
    assert False

print(f"Processing font \"{fontnum}\"")

chari = 0
for sheet in range(SHEETS):
    print(f"Sheet {sheet + 1} of {SHEETS}")
    imgfn = IMG_FN_TMPL.format(sheet + 1)
    widthfn = WIDTHBIN_FN_TMPL.format(sheet + 1)

    im = Image.open(imgfn)

    with open(widthfn, 'rb') as f:
        widths = f.read()

    if CHARHEIGHT == 16:
        chars_per_row = 16
        chars_per_col = 16
    elif CHARHEIGHT == 10:
        chars_per_row = 25
        chars_per_col = 25
    elif CHARHEIGHT == 12:
        chars_per_row = 25
        chars_per_col = 21
    else:
        assert False

    for chary in range(chars_per_col):
        for charx in range(chars_per_row):
            w = widths[chary * chars_per_row + charx]

            print(f"Char {chari} width {w}px")

            newim = im.crop((charx * CHARWIDTH,
                             chary * CHARHEIGHT,
                             charx * CHARWIDTH + w,
                             chary * CHARHEIGHT + CHARHEIGHT))

            newim.save(f"{outdir}/char-{chari}.png")

            omitted = im.crop((charx * CHARWIDTH + w,
                               chary * CHARHEIGHT,
                               charx * CHARWIDTH + CHARWIDTH,
                               chary * CHARHEIGHT + CHARHEIGHT))
            for yy in range(omitted.height):
                for xx in range(omitted.width):
                    assert omitted.getpixel((xx, yy)) == (0, 0, 0, 0)

            chari += 1

unlz.py

#!/usr/bin/env python3

import struct
import sys

def islz(indata):
    return indata[0] == 0x10

def unlz(indata):
    outdata = b''

    header = struct.unpack("<I", indata[:4])[0]
    indata = indata[4:]
    # print(f"Header 0x{header:08X}")

    assert (header & 0xFF) == 0x10
    outsz = header >> 8
    # print(f"output size 0x{outsz:08X}")

    while indata:
        flag = indata[0]
        indata = indata[1:]
        # print(f"flag 0b{flag:08b}")

        for i in range(8):
            if flag & (1 << (7 - i)):
                b0 = indata[0]
                b1 = indata[1]
                indata = indata[2:]

                len_ = (b0 >> 4) + 3
                disp = 1 + ((b0 & 0xF) << 8) + b1
                # print(f"copy len {len_} disp -{disp}")

                off = len(outdata)
                for j in range(len_):
                    b = outdata[off - disp + j:off - disp + j + 1]
                    # print(f"adding {b}")
                    outdata += b
            else:
                b = indata[0:1]
                indata = indata[1:]
                # print(f"adding {b}")
                outdata += b

    assert len(outdata) >= outsz

    return outdata[:outsz]

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print(f"Usage: {sys.argv[0]} input.bin output.bin")
        sys.exit(1)

    infn = sys.argv[1]
    outfn = sys.argv[2]

    with open(infn, 'rb') as f:
        indata = f.read()

    outdata = unlz(indata)

    with open(outfn, 'wb') as f:
        f.write(outdata)

unpack.py

#!/usr/bin/env python3

import math
import os
import string
import struct
import sys

from unlz import islz, unlz

def hexdump(data):
    for row in range(int(math.ceil(len(data) / 16))):
        off = row * 16
        print(f"{off:08X}: ", end='')

        len_ = min(16, len(data) - off)
        for i in range(len_):
            print(f"{data[off + i]:02X} ", end='')
        for _ in range(16 - len_):
            print("   ", end='')
        print("       ", end='')

        for i in range(len_):
            c = data[off + i]
            if (c >= 0x20 and c < 0x80) or (c >= 0xA0):
                print(chr(c), end='')
            else:
                print(".", end='')

        print()

def is_pack(indata):
    return indata[:4] == b'pack'

def parse_pack(indata):
    if not is_pack(indata):
        raise Exception("not a pack file")

    magic, capacity, datasize, unk = struct.unpack("<4sII20s", indata[:0x20])
    # print(magic, capacity, datasize, unk)

    assert unk == b'\x00' * 20
    assert capacity % 4 == 0

    index_and_data = indata[0x20:]
    # print(len(index_and_data))

    assert len(index_and_data) == capacity * 8 + datasize

    ret = []

    for filei in range(capacity):
        off, sz = struct.unpack("<II", index_and_data[filei * 8:filei * 8 + 8])
        # print(filei, off, sz)

        if off == 0 and sz == 0:
            ret.append(None)
            continue

        sz_round_up = int(32 * math.ceil(sz / 32))
        # print(sz_round_up)

        filedata_with_pad = index_and_data[off:off + sz_round_up]

        filedata = filedata_with_pad[:sz]
        filepad = filedata_with_pad[sz:]

        assert filepad == b'\x00' * (sz_round_up - sz)

        # print(filedata)
        ret.append(filedata)

    return ret

def main_dump_pack():
    if len(sys.argv) < 2:
        print(f"Usage: {sys.argv[0]} input.bin")
        sys.exit(1)

    infn = sys.argv[1]
    # print(infn)

    with open(infn, 'rb') as f:
        indata = f.read()
    # print(indata)

    packdata = parse_pack(indata)
    # print(packdata)

    for filei, filedata in enumerate(packdata):
        if filedata is None:
            print(f"File {filei}: -dummy-")
            continue

        print(f"File {filei}: {len(filedata)} (0x{len(filedata):X}) bytes")
        hexdump(filedata)
        print()

def main_extract_pack():
    if len(sys.argv) < 3:
        print(f"Usage: {sys.argv[0]} input.bin outdir")
        sys.exit(1)

    infn = sys.argv[1]
    outdir = sys.argv[2]

    with open(infn, 'rb') as f:
        indata = f.read()

    def extract_to_path(data, path):
        if is_pack(data):
            is_pack_ = True
        elif islz(data):
            try:
                data = unlz(data)
            except IndexError:
                pass
            is_pack_ = is_pack(data)
        else:
            is_pack_ = False

        if is_pack_:
            try:
                os.mkdir(path)
            except FileExistsError:
                pass

            print(f"Extracting pack into {path}...")

            packdata = parse_pack(data)

            for filei, filedata in enumerate(packdata):
                if filedata is None:
                    continue

                extract_to_path(filedata, f'{path}/{filei}')
        else:
            outfn = f"{path}.bin"
            print(f"Extracting {outfn}...")
            with open(outfn, 'wb') as f:
                f.write(data)

    extract_to_path(indata, outdir)

if __name__ == '__main__':
    main_extract_pack()