Convert bbchallenge seed database format into a more compact and computer friendly binary format

bbconv.py

import struct

def find(l, x):
    if x in l: return l.index(x)
    return -1

permuteback = [0, 1, 2, 4, 3, 5, 6, 7, 12, 18, 13, 19, 8, 10, 14, 20, 16, 22, 9, 11, 15, 21, 17, 23]
def rupermute(rules, n):
    m = list(range(2,len(rules)))
    pidx = [0,1] # 1RB assumed
    inv = []
    for i in range(1,len(m)+1):
        n, k = divmod(n, i)
        inv.append(k)
    while m:
        pidx.append(m.pop(inv.pop()))
    #print(pidx)
    fixup = lambda r: (
        (r[0][0], r[0][1], find(pidx, r[0][2])),
        (r[1][0], r[1][1], find(pidx, r[1][2])))
    newrules = [fixup(rules[i]) for i in pidx]
    return newrules

def ruswap(rules, a, b):
    pidx = list(range(len(rules)))
    pidx[a] = b
    pidx[b] = a
    fixup = lambda r: (
        (r[0][0], r[0][1], find(pidx, r[0][2])),
        (r[1][0], r[1][1], find(pidx, r[1][2])))
    newrules = [fixup(rules[i]) for i in pidx]
    return newrules

# Defines lexical order on rules
def rulekey(rules):
    ts = [t for r in rules for t in r]
    ts.pop(0) # first transition is fixed, don't bother comparing
    return [t[2] for t in ts] + [b for t in ts for b in [t[0], t[1]]]

def runorm(ru):
    assert ru[0][0][0]
    assert ru[0][0][1]
    assert ru[0][0][2] == 1
    for t in (t for r in ru for t in r):
        if t[2] == -1:
            assert t[0] == 1 and t[1] == True, t
    # Choose smallest permutation to achieve canonical lexical normal form (as defined by rulekey)
    N = [1,1,2,6,24,120,720][len(ru)-2]
    mi = 0
    mp = ru
    mk = rulekey(ru)
    for i in range(1, N):
        p = rupermute(ru, i)
        k = rulekey(p)
        assert rupermute(p, permuteback[i]) == ru
        if k < mk:
            mi = i
            mp = p
            mk = k
    return mp, mi

def ruleparse(s, norm=True, partial=False):
    assert not (partial and norm)
    s=s.replace(" ","")
    s=s.replace("_","")
    assert len(s)%6 == 0
    N = len(s)//6
    ru = []
    for i in range(0, len(s), 6):
        if not partial:
            w0, d0, s0 = s[i+0] != "0", s[i+1].upper() != "L", ord(s[i+2])-ord("A")
            w1, d1, s1 = s[i+3] != "0", s[i+4].upper() != "L", ord(s[i+5])-ord("A")
            if not (0 <= s0 < N): s0 = N
            if not (0 <= s1 < N): s1 = N
            ru.append(((w0, d0, [i == s0 for i in range(N+1)]), (w1, d1, [i == s1 for i in range(N+1)])))
        else:
            w0, d0, s0 = [False, True, None]["01".find(s[i+0])], [False, True, None]["LR".find(s[i+1].upper())], ord(s[i+2])-ord("A")
            w1, d1, s1 = [False, True, None]["01".find(s[i+3])], [False, True, None]["LR".find(s[i+4].upper())], ord(s[i+5])-ord("A")
            if not (0 <= s0 <= N): s0 = None if s[i+2] not in "!HXZ" else N
            if not (0 <= s1 <= N): s1 = None if s[i+5] not in "!HXZ" else N
            ru.append(((w0, d0, ([i == s0 for i in range(N+1)] if s0 is not None else None)), (w1, d1, ([i == s1 for i in range(N+1)] if s1 is not None else None))))
    if norm:
        ru, _ = runorm(ru)
    return ru

def rustr(rules, ass=None):
    c = lambda s, x: s[x] if x is not None else "?"
    l = lambda i: chr(ord("A")+i) if 0 <= i < len(rules) else "Z?"[i == None]
    return [" ".join([c("01", w) + c("LR", d) + l(s) for (w, d, s) in r]) for r in rules]

# --- #

def read_wide(i=-1, data=None):
    if data is None: data = widedata
    b = struct.unpack_from("B"*30, widedata, i*30+30)
    return runorm([
        (([0, 1][b[i  ]], [True, False][b[i+1]], b[i+2]-1) if b[i+2] else (1, True, -1),
         ([0, 1][b[i+3]], [True, False][b[i+4]], b[i+5]-1) if b[i+5] else (1, True, -1)) for i in range(0, 30, 6)
    ])

def read_short(i=-1, data=None):
    if data is None: data = shortdata
    st, pk = struct.unpack_from("<II", data, i*8+8)
    return ([
        (((pk>>(4*i  )) & 1, bool((pk>>(4*i+1)) & 1), [0,1,2,3,4,5,6,-1][(st>>(i*6  ))&7]),
         ((pk>>(4*i+2)) & 1, bool((pk>>(4*i+3)) & 1), [0,1,2,3,4,5,6,-1][(st>>(i*6+3))&7])) for i in range(5)
    ], pk >> 20)

def write_short(rules, n, data=None, pos=-1):
    st, pk = 0, 0
    for i, (r0, r1) in enumerate(rules):
        st |= (r0[2]&7) << (6*i)
        st |= (r1[2]&7) << (6*i+3)
        pk |= r0[0] << (4*i)
        pk |= int(r0[1]) << (4*i+1)
        pk |= r1[0] << (4*i+2)
        pk |= int(r1[1]) << (4*i+3)
    pk |= n << 20
    if data is None:
        a = bytearray(8)
        struct.pack_into("<II", a, 0, st, pk)
        return a
    else:
        struct.pack_into("<II", data, pos*8+8, st, pk)

if False: # Convert
    with open("all_5_states_undecided_machines_with_global_header", "rb") as f:
        widedata = f.read()

    nmach = struct.unpack_from(">I", widedata, 8)[0]

    import sys
    shortdata = bytearray(8)
    struct.pack_into("<I", shortdata, 0, nmach)
    for i in range(nmach):
        if i % 50000 == 0:
            print(" %d / %d " % (i, nmach), end="\r")
            sys.stdout.flush()
        ru, n = read_wide(i)
        shortdata += write_short(ru, n)
    print("\nDone!")

    with open("seeddb_short_norm.bin", "rb") as f: f.write(shortdata)


with open("seeddb_short_norm.bin", "rb") as f:
    shortdata = f.read()
nmach = struct.unpack_from("<I", shortdata, 0)[0]