Extract TrueCrypt files from RingEdge ISOs, based on scripts by Bottersnike

partition.py

import subprocess
import tempfile
import struct
import zlib
import sys
import os
import csv

from pydantic import BaseModel
from pydantic.dataclasses import dataclass
# from devtools import debug

from Crypto.Cipher import AES


BLOCK_SIZE = 0x400
NULL_BLOCK = b"\0" * BLOCK_SIZE


def try_load(path, game_id, letter="O") -> bool:
    app_data = get_appdata(game_id)
    if app_data is None:
        return False
    tmp = tempfile.NamedTemporaryFile(delete=False)
    tmp.write(app_data.keyfile)
    tmp.close()
    try:
        tc = subprocess.Popen([
            r"C:\Program Files (x86)\TrueCrypt\TrueCrypt.exe",
            "/q", "/s",
            "/k", tmp.name,
            "/l", letter,
            "/v", os.path.abspath(path),
            "/m", "ro",
        ])
        tc.communicate()
        if tc.returncode:
            return False
        subprocess.Popen([
            r"C:\Program Files (x86)\TrueCrypt\TrueCrypt.exe",
            "/q", "/s",
            "/d", letter
        ])
        return True
    finally:
        os.remove(tmp.name)


last_was_sega = False


def dump_block(stream, path, start, end):
    global last_was_sega

    num_blocks = (end - start) // BLOCK_SIZE
    print(f"{end - 1:08x} ({num_blocks})", end="... ", flush=True)
    stream.seek(start)
    first_block = stream.read(BLOCK_SIZE)
    stream.seek(start)

    ext = ""
    if first_block[:6] == b"\x01CD001":
        ext = ".01.cdv001"
    elif first_block[:6] == b"\xffCD001":
        ext = ".ff.cdv001"
    elif start == 0:
        ext = ".alpha"
    elif b"SBTR" in first_block or b"SBYG" in first_block or (len(first_block) < 64 and (b"SD" in first_block or b"SB" in first_block)):
        ext = ".sega"
        last_was_sega = True
    elif last_was_sega:
        ext = ".bootid"
        last_was_sega = False

    block_path = f"{path}_{start:08X}{ext}"

    with open(block_path, "wb") as fBlock:
        for j in range(num_blocks):
            block = stream.read(BLOCK_SIZE)
            if j == num_blocks - 1:
                block = block.rstrip(b"\0")
            fBlock.write(block)
    print()
    stream.read(BLOCK_SIZE)

    try_load(block_path)


def partition_disk(stream, path):
    block_start = 0

    stream.seek(0, 2)
    total_size = stream.tell()
    stream.seek(0)
    in_block = False
    for i in range(0, total_size, BLOCK_SIZE):
        block = stream.read(BLOCK_SIZE)
        if block == NULL_BLOCK:
            if in_block and block_start != i:
                dump_block(stream, path, block_start, stream.tell() - BLOCK_SIZE)
            in_block = False
        elif not in_block:
            block_start = i
            print(f"Block from {block_start:08x}-", end="", flush=True)
            in_block = True

    if in_block:
        dump_block(stream, path, block_start, stream.tell() - BLOCK_SIZE)

def amAuthDiskInitKey(seed):
    key = bytearray(0x8000)

    seed <<= 1
    for i in range(len(key)):
        x = seed

        for _ in range(8):
            x |= ((((seed >> 4) ^ x) >> 11) & 1)
            seed <<= 1
            x <<= 1
        key[i] = (x >> 1) & 0xff

        seed = x & 0xffffffff

    return key

class AlphaDvd:
    HEADER_KEY = amAuthDiskInitKey(5369)

    def __init__(self, stream):
        self.stream = stream

    def decrypt_header(self, number):
        offset = 508 if number != 16 else 318
        
        self.stream.seek(number * 2048 + offset)
        buffer = bytearray(self.stream.read(53))

        for i in range(len(buffer)):
            buffer[i] ^= self.HEADER_KEY[i & 0x7ff]

        header = struct.unpack("<IxxHHH6xH4xH20sHH3x", buffer)

        if header[0] != 0xf1ffff1f:
            return None
        if not header[6].startswith(b"SEGA_DVD"):
            return None

        return header[4]  # start sector

    def locate_header(self):
        if (start := self.decrypt_header(16)) is not None:
            return start
        if (start := self.decrypt_header(1)) is not None:
            return start
        if (start := self.decrypt_header(17)) is not None:
            return start

        return None


@dataclass
class SegaDvdFile:
    offset: int
    slot: int
    name: str


def read_directory(stream) -> tuple[int, int]:
    data = stream.read(34)
    _, _, extent_lba, _, data_length, *_ = struct.unpack("<BBIIIIxxxxxxxBBBHHBx", data)
    return extent_lba, data_length


def parse_sega_table(stream) -> list[SegaDvdFile]:
    start = stream.tell()
    size, crc = struct.unpack("<II", stream.read(8))
    assert size == 0x800

    data = stream.read(size - 8)
    if zlib.crc32(data) & 0xffffffff != crc:
        print("Sega table corrupt! Unable to read storage information")
        return []
    stream.seek(start + 8)

    games = []
    while True:
        offset, filename = struct.unpack("<I4s", stream.read(8))
        if offset == 0 and filename == b"\0\0\0\0":
            break
        games.append(SegaDvdFile(
            (offset & 0xffffff) * 2048,
            offset >> 24,
            filename.decode("latin-1")
        ))

    stream.seek(start + size)
    return games


class AppData:
    system_flag: int
    function_type: int
    region: int
    platform_id: str


@dataclass
class AppDataRing(AppData):
    game_id: str
    
    # Crypto
    key: bytes
    iv: bytes
    seed: bytes
    keyfile: bytes


def scramble(val, a, b, c, d):
    ret = bytearray(val)
    ret[a], ret[b] = ret[b], ret[a]
    ret[c], ret[d] = ret[d], ret[c]
    return bytes(ret)


class Stream:
    def __init__(self, stream):
        self.stream = stream

    def read(self, n):
        return self.stream.read(n)

    def seek(self, n):
        self.stream.seek(n)

    def crc32(self, nbytes):
        start = self.stream.tell()
        data = self.stream.read(nbytes)
        self.stream.seek(start)
        return zlib.crc32(data) & 0xffffffff

    def skip(self, n):
        self.read(n)

    def unpack(self, fmt):
        return struct.unpack("<" + fmt, self.stream.read(struct.calcsize(fmt)))

    def u8(self):
        return self.unpack("B")[0]

    def u16(self):
        return self.unpack("H")[0]

    def u32(self):
        return self.unpack("I")[0]

    def ipv4(self):
        return self.unpack("4B")

    def ipv6(self):
        return self.unpack("8H")

    def str(self, n):
        return self.unpack(f"{n}s")[0].decode("latin-1")


def parse_appdata_ring(row):
    game_id = row[0]
    key = bytes.fromhex(row[13])
    iv = bytes.fromhex(row[14])
    seed = bytes.fromhex(row[15])
    keyfile = bytes.fromhex(row[16])
    
    cipher = AES.new(key, AES.MODE_CBC, iv)
    keyfile2 = cipher.encrypt(seed)
    
    if keyfile != keyfile2:
        print(f"ERROR: {game_id} KEYS MISMATCH {keyfile.hex()} {keyfile2.hex()}")

    return AppDataRing(
        game_id, key, iv, seed, keyfile
    )


def all_appdata():
    with open("ringedge.csv", newline="") as kf:
        rows = list(csv.reader(kf))
        
        for row in rows[1:]:
            app_data = parse_appdata_ring(row)
            yield app_data


def get_appdata(game_id):
    for app_data in all_appdata():
        if app_data.game_id == game_id:
            return app_data

    return None


@dataclass
class Timestamp:
    year: int
    month: int
    day: int
    hour: int
    minute: int
    second: int


class BootID(BaseModel):
    model_type: int
    image_type: int
    game_id: str
    game_timestamp: Timestamp
    version: tuple[int, int]
    flags: int  # two nibbles
    segcount: int
    segsize: int
    hw: str
    instant: bool
    orgtime: Timestamp
    orgversion: tuple[int, int]
    osver: tuple[int, int, int]
    ossegcount: int

    name: str
    strings: list[str]


def decode_btid(btid, game_id):
    app_data = get_appdata(game_id)
    # print("KEY", app_data.key.hex())
    # print("IV", app_data.iv.hex())
    if not app_data:
        print(f"Unable to locate keys for {game_id}. Brute forcing.")

        for app_data in all_appdata():
            cipher = AES.new(app_data.key, AES.MODE_CBC, app_data.iv)
            bootid = cipher.decrypt(btid)
            if bootid[8:12] == b"BTID":
                print("-> Found keys:", app_data)
                break
        else:
            print("-> Brute force failed.")
            return None

    cipher = AES.new(app_data.key, AES.MODE_CBC, app_data.iv)
    bootid = cipher.decrypt(btid)

    # print(bootid[:16])
    content = struct.unpack(
        "<"
        "II4sBBxx"  # Boot ID header
        "4sH5Bx2H7xBII3sBH5Bx2H3BxI8x"  # Install info
        "16x"  # Padding
        "160s"  # Game name
        "32s32s32s32s32s32s32s32s",  # Strings
        bootid
    )

    if content[0] != zlib.crc32(bootid[4:]) & 0xffffffff:
        return None
    assert content[1] == 512
    assert content[2] == b"BTID"

    with open(f"partout/{game_id}_{content[14]:02x}.keyfile", "wb") as kyf:
        kyf.write(app_data.keyfile)

    return BootID(
        model_type=content[3],
        image_type=content[4],
        game_id=content[5].decode("latin-1"),
        game_timestamp=Timestamp(
            content[6], content[7], content[8], content[9], content[10], content[11],
        ),
        version=(content[13], content[12]),
        flags=content[14],
        segcount=content[15],
        segsize=content[16],
        hw=content[17].decode("latin-1"),
        instant=bool(content[18]),
        orgtime=Timestamp(
            content[19], content[20], content[21], content[22], content[23], content[24],
        ),
        orgversion=(content[26], content[25]),
        osver=(content[29], content[28], content[27]),
        ossegcount=content[30],
        name=content[31].split(b"\0")[0].decode("latin-1"),
        strings=[i.split(b"\0")[0].decode("latin-1") for i in content[32:]],
    )


class Bar:
    def __init__(self, msg, max_, width=30):
        self.msg = msg
        self.max = max(0, max_)
        self.width = width
        self.last_n = 0

    def __enter__(self):
        self.update(0)
        return self

    def update(self, n):
        n = max(0, min(n, self.max))
        self.last_n = n
        bar = round((n / self.max) * self.width) * "="
        print(f"{self.msg} [{bar}{' ' * (self.width - len(bar))}] {n/self.max * 100:.2f}%", end="\r", flush=True)

    def print_over(self, *args):
        self.clear()
        print(*args)
        self.update(self.last_n)

    def clear(self):
        print(" " * (len(self.msg) + 2 + self.width + 2 + len("100.00%")), end="\r", flush=True)

    def __exit__(self, *_):
        self.clear()


def parse_dvd(stream, path):
    """
    Pending review:

    - "[SBYG]DVR-0069"\DVR-0069A.iso
    """

    alpha_base = AlphaDvd(stream).locate_header()
    if alpha_base is not None:
        # print("ALPHA")
        alpha_base *= 2048
        stream.seek(alpha_base)
    else:
        alpha_base = 0
        stream.seek(0)
    
    # print(alpha_base)
    
    stream.seek(0x8000, 1)

    # Alpha DVD is done, time for ISO 6990
    while True:
        vol_type = stream.read(1)[0]
        vol_id = stream.read(5)
        assert vol_id == b"CD001"
        vol_version = stream.read(1)[0]
        assert vol_version == 1

        if vol_type == 1:
            start = stream.tell()
            stream.seek(1, 1)
            system = stream.read(32).decode("latin-1")
            volume = stream.read(32).decode("latin-1")
            stream.seek(8, 1)
            logical_blocks = struct.unpack("<II", stream.read(8))[0]
            stream.seek(40, 1)
            logical_block_size = struct.unpack("<HH", stream.read(4))[0]
            assert logical_block_size == 2048
            path_table_size = struct.unpack("<II", stream.read(8))[0]
            assert path_table_size == 0
            struct.unpack("<II", stream.read(8))[0]
            struct.unpack(">II", stream.read(8))[0]
            extent_lba, _ = read_directory(stream)

            # Primary volume
            print(f"Disc name:    {volume.strip()}")
            print(f"System name:  {system.strip()}")
            print(f"Blocks:       {logical_blocks} x {logical_block_size}")
            stream.seek(start + 2041)
        elif vol_type == 255:
            stream.seek(2041, 1)
            break
        else:
            print(f"Unknown ISO 9660 volume: {vol_type}")
            return

    stream.seek(alpha_base + extent_lba * logical_block_size)
    lba, length = read_directory(stream)
    assert lba == extent_lba

    # At this point we're finished with ISO 9660, and it's onto SEGA stuff
    stream.seek(alpha_base + lba * logical_block_size + length)
    stream.seek(0xf3800, 1)  # ?? It works, but... what?

    # Looking at segaboot, the answer is we just skip over all this entirely
    # and jump to hardcoded offsets
    stream.seek(alpha_base + 0xff800)

    dvdfiles = parse_sega_table(stream)
    print(f":: Found {len(dvdfiles)} file{'' if len(dvdfiles) == 1 else 's'}")

    stream.seek(alpha_base + 512 * 2048)
    boot_sectors = []
    for dvdfile in dvdfiles:
        # print(dvdfile)
        btid_enc = stream.read(0x200)
        # print(btid_enc[:16].hex())
        # print(hex(stream.tell()))
        btid = decode_btid(btid_enc, dvdfile.name)

        chunk_sums = struct.unpack("<65408I", stream.read(0x3fe00))
        boot_sectors.append((btid, chunk_sums))

    for n, dvdfile in enumerate(dvdfiles):
        if n != 0:
            #dvdfile.offset += 0x40000 * n
            pass

    for n, (dvdfile, (bootid, chunk_sums)) in enumerate(zip(dvdfiles, boot_sectors)):
        print(f"Reading file: {dvdfile.name} @ +{alpha_base + dvdfile.offset:08x} ({dvdfile.slot})")
        stream.seek(alpha_base + dvdfile.offset)
        use_other_btid = False

        if bootid is None:
            print(f"! Trying bootid @ {alpha_base + dvdfile.offset:08x}!")

            stream.seek(alpha_base + dvdfile.offset)
            bootid = decode_btid(stream.read(0x200), dvdfile.name)
            chunk_sums = struct.unpack("<65408I", stream.read(0x3fe00))
            use_other_btid = True
            # print(bootid)
        
        if bootid is None:
            print("! No boot ID. Guestimating values!")
            filename = f"{dvdfile.name}_{dvdfile.offset}"
            segcount = len(chunk_sums)
            for i in chunk_sums[::-1]:
                if i:
                    break
                segcount -= 1
            segsize = 0x40000

            # DVR-0069A
            # segcount = 1896
        else:
            image_type = ("original", "original", "patch", "patch")
            filename = (
                f"{bootid.game_id}"
                f"_Ver.{bootid.version[0]}.{str(bootid.version[1]).zfill(2)}"
                f"_{bootid.flags:02x}"
                f".{image_type[bootid.image_type]}"
            )

        segcount = bootid.segcount
        segsize = bootid.segsize
        
        if use_other_btid:
            segcount -= 1

        # The first chunk is the boot ID. I don't know what's being CRC'd there..
        stream.seek(alpha_base + dvdfile.offset + 0x40000)
        chunk_sums = chunk_sums[1:]

        start_chunk = 1 if n == 0 else 0

        with Bar(f"Extracting to {filename}", segcount) as bar:
            with open("partout/" + filename, "wb") as fOut:
                for chunk_n in range(start_chunk, segcount):
                    chunk = stream.read(segsize)
                    if chunk_sums[chunk_n - start_chunk] != zlib.crc32(chunk) & 0xffffffff:
                        bar.print_over(
                            f"Invaid chunk: {chunk_n - start_chunk} ({stream.tell() - segsize:08x})"
                            + f"({chunk_sums[chunk_n - start_chunk]:08x} != {zlib.crc32(chunk) & 0xffffffff:08x})"
                        )
                    fOut.write(chunk)
                    bar.update(chunk_n)
        print(f"Game extracted to {filename}")

        if False:
            print("Verifying TrueCrypt volume... ", end="", flush=True)

            loaded = try_load("partout/" + filename, dvdfile.name)
            if loaded:
                print("Verified")
            else:
                print("FAILED!")
                print("PLEASE CHECK CONTAINER!")


def main():
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} <filename>")
        return 1
    path = sys.argv[1]
    if not os.path.exists(path) or not os.path.isfile(path):
        print(f"No such file: {path}")
        return 2
        
    with open(path, "rb") as stream:
        parse_dvd(stream, path)
        # partition_disk(stream, path)


if __name__ == "__main__":
    sys.exit(main())

ringedge.csv