RGSSAD non-standard magic key recovery tool

rgssadatk.py

#!/usr/bin/python3
# This tool exploits a weakness in RGSSAD file format in order to recover
# non-standard magic key (other than the default 0xdeadcafe key) used in some
# RGSSAD archives.
#
# RGSSAD uses a LCG (Linear Congruential Generator) (Xn+1 = aXn+c mod m) to
# generate xor pads in order to encrypt files and their metadata. However,
# the LCG implementation returns the seed (X0) instead of the result of a single
# iteration of the algorithm (X1) on its first iteration. The seed is then being
# used as the xor pad for the file name length (a 32-bit value) of the first
# file, which is highly predictable (i.e. most of the bits, starts from MSB,
# will be 0 because the file name is usually not too long). By exploiting this
# flaw, one can easily recover a large portion of the bits of the seed (aka the
# magic key) if one knows the approximate length of the file name, and the rest
# of the unknown bits can be easily brute-forced in a relatively short time.
#
# Same flaw was also being exploited in other softwares (e.g. CatPaw by bsucat
# (description available at http://blog.csdn.net/bsucat/article/details/4532038,
# the actual software disappeared with the old 66rpg forum)). Although the flaw
# was discovered independently from these softwares.

import io
import sys
import struct

def usage():
    print('RGSSAD MagicKey Attack Utility\n'
          'Usage:', sys.argv[0], '<infile>\n')

def transform(key):
    key *= 7
    key += 3
    key &= 0xffffffff
    return key

def get_str_bytes(estr,key):
    tmpstr = bytearray()
    length = len(estr)
    for i in range(0,length):
        tmpstr.append(estr[i] ^ (key&0xff))
        key = transform(key)
    return bytes(tmpstr), key

def fastbf(rgssad):
    cur_offset = rgssad.tell()
    rgssad.seek(0, io.SEEK_END)
    archive_size = rgssad.tell()
    rgssad.seek(cur_offset)

    efnamesize = struct.unpack('<I', rgssad.read(4))[0]
    template = efnamesize & 0xffffff00
    double_pass = []
    for i in range(0, 256):
        tests = [False, False]
        key = template + i
        iv = key
        print(hex(key), ':')

        fnamesize = efnamesize ^ key
        rgssad.seek(12)
        if fnamesize > (archive_size - 16):
            print('    Impossible because filename is too long')
            continue

        efname = rgssad.read(fnamesize)

        fname, key = get_str_bytes(efname, transform(key))

        try:
            fname_decoded = fname.decode('utf-8')
        except UnicodeDecodeError:
            print('    File name encoding test: Failed')
        else:
            tests[0] = True
            print('    File name encoding test: Passed')
            print('    File name:', repr(fname_decoded))

        fsize = struct.unpack('<I', rgssad.read(4))[0]
        fsize ^= key

        #print('    File size: {0}'.format(fsize))
        if fsize > (archive_size - rgssad.tell()):
            print('    Subfile size check: Failed')
        else:
            tests[1] = True
            print('    Subfile size check: Passed')

        if False not in tests:
            double_pass.append(iv)

    if len(double_pass) != 0:
        print('Keys that passed both tests:')
        for key in double_pass:
            print('    ', hex(key))
    else:
        print('Sorry, no key passed both tests')

if __name__ == '__main__':
    if len(sys.argv)<2:
        usage()
        sys.exit(1)

    with open(sys.argv[1], 'rb') as rgssad:
        if rgssad.read(8) != b'RGSSAD\x00\x01':
            print('Load failed, not a rgssad file')
            sys.exit(1)

        fastbf(rgssad)