mbdb file decoder, ported to Python 3, no mbdx file needed

mbdbpy3.py

#!/usr/bin/env python3
import sys
import hashlib

def getint(data, offset, intsize):
    """Retrieve an integer (big-endian) and new offset from the current offset"""
    value = 0
    while intsize > 0:
        value = (value << 8) | data[offset]
        offset += 1
        intsize -= 1
    return value, offset

def getstring(data, offset):
    """Retrieve a string and new offset from the current offset into the data"""
    value, new_offset = getbytes(data, offset)
    return value.decode("utf-8"), new_offset

def getbytes(data, offset):
    """Retrieve a bytestring and a new offset from the current offset into the data"""
    if data[offset] == 0xFF and data[offset+1] == 0xFF:
        return b'', offset+2 # Blank string
    length, offset = getint(data, offset, 2) # 2-byte length
    value = data[offset:offset+length]
    return value, (offset + length)

def process_mbdb_file(filename):
    mbdb = {} # Map offset of info in this file => file info
    data = open(filename, "rb").read()
    if data[0:4] != b"mbdb":
        raise Exception("This does not look like an MBDB file")
    offset = 4
    offset += 2 # value 0x05 0x00, not sure what this is (version number?)
    while offset < len(data):
        fileinfo = {}
        fileinfo['start_offset'] = offset
        fileinfo['domain'], offset = getstring(data, offset)
        fileinfo['filename'], offset = getstring(data, offset)
        fileinfo['linktarget'], offset = getstring(data, offset)
        fileinfo['datahash'], offset = getbytes(data, offset)
        fileinfo['unknown1'], offset = getbytes(data, offset)
        fileinfo['mode'], offset = getint(data, offset, 2)
        fileinfo['unknown2'], offset = getint(data, offset, 4)
        fileinfo['unknown3'], offset = getint(data, offset, 4)
        fileinfo['userid'], offset = getint(data, offset, 4)
        fileinfo['groupid'], offset = getint(data, offset, 4)
        fileinfo['mtime'], offset = getint(data, offset, 4)
        fileinfo['atime'], offset = getint(data, offset, 4)
        fileinfo['ctime'], offset = getint(data, offset, 4)
        fileinfo['filelen'], offset = getint(data, offset, 8)
        fileinfo['flag'], offset = getint(data, offset, 1)
        fileinfo['numprops'], offset = getint(data, offset, 1)
        fileinfo['properties'] = {}
        for ii in range(fileinfo['numprops']):
            propname, offset = getstring(data, offset)
            propval, offset = getbytes(data, offset)
            fileinfo['properties'][propname] = propval
        fileinfo['backup_filename'] = hashlib.sha1("{0}-{1}".format(fileinfo['domain'], fileinfo['filename']).encode("utf-8")).hexdigest()
        mbdb[fileinfo['start_offset']] = fileinfo
    return mbdb

def modestr(val):
    def mode(val):
        r = 'r' if (val & 0x4) else '-'
        w = 'w' if (val & 0x2) else '-'
        x = 'x' if (val & 0x1) else '-'
        return r+w+x
    return mode(val>>6) + mode(val>>3) + mode(val)

def fileinfo_str(f, verbose=False):
    if not verbose:
        return "({fid}){dom}::{fn}".format(fid=f['fileID'], dom=f['domain'], fn=f['filename'])

    if (f['mode'] & 0xE000) == 0xA000:
        entry_type = 'l' # symlink
    elif (f['mode'] & 0xE000) == 0x8000:
        entry_type = '-' # file
    elif (f['mode'] & 0xE000) == 0x4000:
        entry_type = 'd' # dir
    else:
        print("Unknown file type {m:04x} for {fis}".format(m=f['mode'], fis=fileinfo_str(f, False)), file=sys.stderr)
        entry_type = '?' # unknown
    info = "{et}{mod} {uid:08x} {gid:08x} {fl:07} {mt:10} {at:10} {ct:10} ({bfn}){dom}::{fn}".format(
        et=entry_type, mod=modestr(f['mode'] & 0x0FFF), uid=f['userid'],
        gid=f['groupid'], fl=f['filelen'], mt=f['mtime'], at=f['atime'],
        ct=f['ctime'], bfn=f['backup_filename'], dom=f['domain'],
        fn=f['filename']
    )
    if type == 'l':
        info += ' -> ' + f['linktarget'] # symlink destination
    for (name, value) in f['properties'].items(): # extra properties
        info += ' {0}={1}'.format(name, repr(value))
    return info

verbose = True
if __name__ == '__main__':
    mbdb = process_mbdb_file("Manifest.mbdb")
    for offset, fileinfo in mbdb.items():
        print(fileinfo_str(fileinfo, verbose))