Code for reading Vblank Entertainment' Shakedown: Hawaii's BFP archives

ShakedownHawaiiBFP2.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Linq;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using System.IO.Compression;

static class ShakedownHawaiiBFP2
{
    private static readonly Dictionary<uint, string> NameLookup;
    private static readonly Regex ScriptRegex = new Regex("script_([0-9A-F]{2}).bin");
    private static readonly Regex UnknownRegex = new Regex("unknown_([0-9A-F]{8}).bin");

    static ShakedownHawaiiBFP2()
    {
        NameLookup = new Dictionary<uint, string>(KnownNames.Length);

        foreach (var name in KnownNames)
            NameLookup[HashName(name)] = name;
    }

    public static void Export()
    {
        Directory.CreateDirectory("Dump");

        if (!File.Exists("gamedata.bfp"))
            throw new Exception("Unable to find gamedata.bfp");

        var fs = File.OpenRead("gamedata - Copy.bfp");
        var header = fs.Read<BFP2>();

        var fileEntries = new FileEntry[header.FileEntryCount + 0x100];
        var eagerLoadSegmentSizes = new Queue<int>(header.EagerLoadSegmentCount);

        // read file entries
        for (int i = 0; i < header.FileEntryCount; i++)
        {
            fileEntries[i] = new FileEntry
            {
                NameHash = fs.Read<uint>(),
                DataOffset = fs.Read<int>(),
                DecompressedSize = fs.Read<int>(),
                CompressedSize = fs.Read<int>()
            };
        }

        // read script entries
        for (int i = header.FileEntryCount; i < fileEntries.Length; i++)
        {
            fileEntries[i] = new FileEntry
            {
                DataOffset = fs.Read<int>(),
                DecompressedSize = fs.Read<int>(),
                CompressedSize = fs.Read<int>()
            };
        }

        // read eager load sizes
        for (int i = 0; i < header.EagerLoadSegmentCount; i++)
            eagerLoadSegmentSizes.Enqueue(fs.Read<int>());

        int lazyLoadByteCount = header.LazyLoadByteCount;
        int prevLazyLoadCount = header.LazyLoadByteCount;

        for (int i = 0; i < fileEntries.Length; i++)
        {
            var entry = fileEntries[i];

            // null entry
            if (entry.DecompressedSize == 0 || entry.CompressedSize == 0)
                continue;

            // seek data position
            fs.Position = entry.DataOffset;

            // check if this file exceeds the lazy load limit
            if (entry.DataOffset + entry.CompressedSize > lazyLoadByteCount)
            {
                if (!eagerLoadSegmentSizes.TryDequeue(out var size))
                    throw new Exception("Exceeded EagerLoadSegmentCount");

                // increase cutoff
                prevLazyLoadCount = lazyLoadByteCount;
                lazyLoadByteCount += size;

                // the game uses this mechanic to check for which
                // files to lazy load and which ones to eager load
                // e.g.
                if (entry.DataOffset > header.LazyLoadByteCount &&
                    entry.DataOffset > prevLazyLoadCount &&
                    entry.DataOffset + entry.CompressedSize < lazyLoadByteCount)
                {
                    // Eager load
                    // NOTE: scripts MUST be eager loaded!!!!
                } 
            }

            // read the file's data
            byte[] buffer = fs.ReadBytes(entry.CompressedSize);

            // zlib deflate
            if (entry.CompressedSize != entry.DecompressedSize)
            {
                using (var msIn = new MemoryStream(buffer, 2, buffer.Length - 2))
                using (var msOut = new MemoryStream(entry.DecompressedSize))
                using (var ds = new DeflateStream(msIn, CompressionMode.Decompress))
                {
                    ds.CopyTo(msOut);
                    buffer = msOut.ToArray();
                }
            }

            // export
            if (NameLookup.TryGetValue(entry.NameHash, out var name))
            {
                Console.WriteLine(name);
                File.WriteAllBytes($"Dump\\{name}", buffer);
            }
            else if (entry.NameHash > 0)
            {
                Console.WriteLine(entry.NameHash.ToString("X8"));
                File.WriteAllBytes($"Dump\\unknown_{entry.NameHash:X8}.bin", buffer);
            }
            else
            {
                Console.WriteLine($"script_{i - header.FileEntryCount:X2}");
                File.WriteAllBytes($"Dump\\script_{i - header.FileEntryCount:X2}.bin", buffer);
            }
        }
    }

    public static void Import()
    {
        var fileEntries = new List<FileEntry>();
        var scriptEntries = new FileEntry[0x100];
        var unionedEntries = fileEntries.Union(scriptEntries);

        AllocateEntries(ref fileEntries, ref scriptEntries);

        var fileCompressedSize = fileEntries.Sum(e => e.AlignedCompressedSize);
        var scriptCompressedSize = scriptEntries.Sum(e => e.AlignedCompressedSize);
        var headerBlockSize = 64 + (fileEntries.Count * 16) + 0xC00 + 0x110;
        var dataBlockSize = fileCompressedSize + scriptCompressedSize;
        var totalFileSize = headerBlockSize + dataBlockSize;

        // update offsets
        var lastOffset = headerBlockSize;
        foreach (var entry in unionedEntries)
        {
            if (entry.CompressedSize != 0)
            {
                entry.DataOffset = lastOffset;
                lastOffset += entry.AlignedCompressedSize;
            }
        }

        using (var fs = File.Create("gamedata.bfp"))
        {
            Console.WriteLine("Writing gamedata.bfp");

            fs.Write(new BFP2
            {
                Magic = 0x32504642,
                FileEntryCount = fileEntries.Count,
                TotalAlignedFileDecompressedSize = fileEntries.Sum(e => e.AlignedDecompressedSize),
                TotalAlignedScriptDecompressedSize = scriptEntries.Sum(e => e.AlignedDecompressedSize),
                AllDataDecompressedBufferSize = unionedEntries.Sum(e => e.AlignedDecompressedSize),
                LazyLoadByteCount = headerBlockSize + fileCompressedSize, // all files
                FileDataSize = scriptCompressedSize,
                DecompressionBufferSize = unionedEntries.Max(e => e.AlignedDecompressedSize),
                EagerLoadSegmentCount = 1,
            });

            foreach (var entry in fileEntries)
            {
                fs.Write(entry.NameHash);
                fs.Write(entry.DataOffset);
                fs.Write(entry.DecompressedSize);
                fs.Write(entry.CompressedSize);
            }

            foreach (var entry in scriptEntries)
            {
                fs.Write(entry.DataOffset);
                fs.Write(entry.DecompressedSize);
                fs.Write(entry.CompressedSize);
            }

            // set the first script to eager load and use the
            // remainder of the file as the eager load size -
            // this forces all scripts to load
            fs.Write(totalFileSize - scriptEntries[0].DataOffset);

            fs.Position = headerBlockSize; // seek to data table

            foreach (var entry in unionedEntries)
            {
                if (entry.DataOffset != 0)
                {
                    fs.Position = entry.DataOffset;
                    fs.Write(entry.Data, 0, entry.Data.Length);
                }
            }

            // final padding
            fs.SetLength(totalFileSize);
        }
    }

    private static void AllocateEntries(ref List<FileEntry> files, ref FileEntry[] script)
    {
        foreach (var file in Directory.GetFiles("."))
        {
            var name = Path.GetFileName(file);

            if (name.ToLower() == "gamedata.bfp")
                continue;

            Console.WriteLine($"Compression {name}...");

            var decompressedData = File.ReadAllBytes(file);
            var decompressedSize = decompressedData.Length;
            var compressedData = ZlibCompress(decompressedData);
            var compressedSize = compressedData.Length;

            var entry = new FileEntry()
            {
                DecompressedSize = decompressedSize,
                CompressedSize = compressedSize,
                Data = compressedData
            };

            if (ScriptRegex.IsMatch(file))
            {
                var ordinal = int.Parse(ScriptRegex.Match(file).Groups[1].Value, NumberStyles.HexNumber);
                script[ordinal] = entry;
            }
            else if (UnknownRegex.IsMatch(file))
            {
                entry.NameHash = uint.Parse(UnknownRegex.Match(file).Groups[1].Value, NumberStyles.HexNumber);
                files.Add(entry);
            }
            else
            {
                entry.NameHash = HashName(name);
                files.Add(entry);
            }
        }

        // fill blank slots
        for (var i = 0; i < script.Length; i++)
            script[i] = script[i] ?? new FileEntry();

        files.Sort((a, b) => a.CompressedSize.CompareTo(b.CompressedSize));
    }

    public static uint HashName(string value)
    {
        value = value.ToUpper(); // uppercse

        uint h = 0u;
        for (int i = 0; i < value.Length; i++)
        {
            // xor char with hash lo byte to get table index
            uint index = value[i] ^ (h & 0xFF);
            // xor table entry with 2 x hash value
            h = HashTable[index] ^ (h * 2);
        }

        return h;
    }

    private static byte[] ZlibCompress(byte[] buffer)
    {
        using (var msIn = new MemoryStream(buffer))
        using (var msOut = new MemoryStream(buffer.Length))
        using (var ds = new DeflaterOutputStream(msOut))
        {
            msIn.CopyTo(ds);
            ds.Flush();
            ds.Finish();
            return msOut.ToArray();
        }
    }

    [StructLayout(LayoutKind.Sequential)]
    struct BFP2
    {
        public uint Magic; // "BFP2"
        public int FileEntryCount;
        public int TotalAlignedFileDecompressedSize; // sum of "file" DecompressedSizes when 64 byte aligned
        public int TotalAlignedScriptDecompressedSize; // sum of "script" DecompressedSizes when 64 byte aligned
        public int AllDataDecompressedBufferSize; //  size of a buffer which stores all decompressed data
        public int LazyLoadByteCount;
        public int FileDataSize; // length of file minus FakeSizeLimit
        public int DecompressionBufferSize; // size of the zlib decompression buffer (largest 64 byte aligned CompressedSize)
        public int EagerLoadSegmentCount;

        [MarshalAs(UnmanagedType.ByValArray, SizeConst = 28)]
        private readonly byte[] Padding;
    }

    [StructLayout(LayoutKind.Sequential)]
    class FileEntry
    {
        public uint NameHash; // if > 0, "file" else "script"
        public int DataOffset;
        public int DecompressedSize;
        public int CompressedSize; // if compressed != decompressed, zlib else uncompressed

        public int AlignedDecompressedSize => (DecompressedSize + 63) & ~0x3F;
        public int AlignedCompressedSize => (CompressedSize + 63) & ~0x3F;
        public byte[] Data;
    }

    private static readonly uint[] HashTable = new uint[]
    {
        0x3C5F0CEB, 0x67465CEA, 0x9BB5A2C, 0x377B67B7, 0x563BF7, 0x62707630, 0x5B823037, 0x79257731,
        0x86C2B35, 0x475A04FD, 0x4EF43A04, 0x63023ACB, 0x7CBC241C, 0x4FF50381, 0x467F5045, 0xBE545BF,
        0x53A570B7, 0x1693111F, 0x28143A7F, 0x4A4230BB, 0x60482C24, 0x186C46F0, 0x73A6278B, 0x7EB55662,
        0x710D6318, 0x481D5C40, 0x69AF7616, 0x6E795FCA, 0x7E1F0459, 0x30241B10, 0x41AA4D21, 0x2DF220FC,
        0x67EB2142, 0x25652492, 0x709A0482, 0x1B244577, 0x45A57E43, 0x9A805B0, 0x7F825D62, 0x68394CAD,
        0x43C3735E, 0x1E191B86, 0x71EB0ABD, 0x35FF5684, 0x12801CAB, 0x65C647E3, 0x3C6109E5, 0x3E673FD6,
        0x645A5AD5, 0x182575CC, 0x4AF64AFE, 0xF070EF2, 0x7C94239B, 0x238B6DF9, 0x17BD2983, 0x53993019,
        0x4DB52250, 0x3FB67B56, 0x515034C0, 0x307A0202, 0x2807285C, 0x10445F83, 0x114B4857, 0x2D2C3256,
        0x781A6DB8, 0x57391754, 0x22CF74BC, 0x2ED56A34, 0x153E2175, 0x377D6F52, 0x590037B8, 0x2BD4AAE,
        0x200C4A36, 0x75C6838, 0x758704E9, 0x78D4394B, 0x70DE76B2, 0x33046B77, 0x1F111E40, 0xE297C83,
        0x16523E33, 0x2F0B4FB2, 0x67CE3C81, 0x27742846, 0x63CB1119, 0x7AE62D43, 0x63F307C7, 0x5B8D5A75,
        0xFEF5958, 0x337302B4, 0x50385FFE, 0x4DF34767, 0x632A6AF6, 0x35702947, 0x485B7567, 0x19471666,
        0x762A448E, 0x5001196F, 0xD9B3118, 0x49CE0E2F, 0x621FCF, 0x72F7F54, 0x5D3D6D79, 0x67F21175,
        0x368751FD, 0x66631F53, 0x570B7EC8, 0x12C16B5F, 0x39167C6F, 0x62EF0A7C, 0x73D00B96, 0x2A6C6C05,
        0x12CB0D10, 0x4E852312, 0xBDD3548, 0xAC954F7, 0x472B0EDF, 0x59BE710E, 0x6D871096, 0x55D215B7,
        0x70FC4A6F, 0x2694469C, 0x3A66E0F, 0x4E24583A, 0x70C73667, 0x6AE8349D, 0xC187293, 0x41815D6A,
        0x2B5D3802, 0x22FE4F23, 0x5E3C7FD8, 0x34F29A7, 0x584F3390, 0x53FC41F8, 0x41786CE6, 0x77170141,
        0x60756CF4, 0x5E6F3518, 0x53B40E9B, 0x2B063500, 0x4C683824, 0x60C50132, 0x7FDC1027, 0x26F3E9D,
        0x430779AC, 0x29D4342B, 0x4621B91, 0x70472D46, 0x17F6772C, 0x3B51659B, 0x9B95230, 0x41A7621D,
        0x6A1A77D5, 0x5C5A5B4D, 0x48DB1533, 0x784E1CB9, 0x521F34EF, 0x3BED7DC3, 0x41C41E1A, 0x351C57A4,
        0x20F21A56, 0x236E1CB1, 0x1F4673B, 0x329874DC, 0x2E4756F9, 0x3926037D, 0x7AF1643E, 0x4F6C3A53,
        0x37143D5A, 0x52BE5DC5, 0x68C30AA1, 0x28E41E6E, 0x4C147410, 0x57C86BD9, 0x48772A34, 0x45726489,
        0x50467648, 0x3436073D, 0x5E9D159E, 0x4F2C0972, 0x76B6440, 0x5AE17728, 0x4DC91AD6, 0x5E4C7FE9,
        0x348B23CA, 0x58041508, 0x3D154BAB, 0x53B03D27, 0x487150CF, 0x73BE40FB, 0xE9E163D, 0x43581554,
        0x202A7DC9, 0x64932657, 0x26032D81, 0x6EEA680E, 0x538A4448, 0x11EC5024, 0x3EE941C1, 0x50311CE9,
        0x13A6256F, 0x66920D9C, 0x5379091A, 0x33996FEA, 0x195B3A74, 0x333726B2, 0x12E017FC, 0x62B50E0B,
        0x23C63523, 0x20ED6088, 0x67CE09AD, 0x5EBA01BB, 0x6CA305B, 0x33AD51F7, 0xEF878C7, 0x2B026F5A,
        0x498E508F, 0x5CD2080B, 0x3D9647B5, 0x278A21C1, 0x54FC3446, 0x1D9B7A85, 0x57E6393A, 0x7B7366B8,
        0x3243349C, 0x39AD5057, 0x37A758EB, 0xF843B7E, 0x595675BF, 0x798E742B, 0x29F33AF, 0x18A74945,
        0xF6B4773, 0x7D2A78DD, 0x11156046, 0x326831B3, 0x557C558E, 0x1E524DFC, 0x645757BF, 0x9792B62,
        0x66C9287D, 0x6339444D, 0x2D361E01, 0x16306E61, 0x475475BD, 0xF56248, 0x62853A43, 0x670770B1,
        0x62644063, 0x6E040898, 0x679D7F93, 0x7B1C72C8, 0x39034995, 0x4C4669F, 0x42DC2553, 0x2CAF5C12
    };

    private static readonly string[] KnownNames = new[]
    {
        "changes.txt", "credits.txt", "demo1.rec", "demo2.rec", "demo3.rec",
        "enemydefs.bin", "fonts.bin", "fonts_rgb_1555.bin", "gametext.bxt",
        "inklevel_1.bin", "kiki.bld", "kiki.bmd", "kiki.cls", "kiki.dyn",
        "kiki.map", "kiki.til", "kiki.wal", "kiki_anims.bin", "kiki_anims_cars.bin",
        "kiki_anims_cutscenes.bin", "kiki_anims_dynamics.bin", "kiki_anims_effects.bin",
        "kiki_anims_mainmenu.bin", "kiki_anims_minigame.bin", "kiki_anims_misc.bin",
        "kiki_anims_n3ds.bin", "kiki_anims_shared.bin", "kiki_bg_pal_rgb_1555.bin",
        "kiki_cutscenedefs.bin", "kiki_data.nav", "kiki_effects.bin",
        "kiki_interiors.bin", "kiki_regions.bin", "kiki_roads.bin", "kiki_roads.nav",
        "missiondefs.bin", "objectives.bxt", "records.bxt", "scenedefs.bin",
        "thighlevel_1.bin", "thighlevel_2.bin", "thighlevel_3.bin", "thighlevel_4.bin",
        "thighlevel_5.bin", "waterlevel_1.bin", "waterlevel_2.bin", "waterlevel_3.bin",
        "waterlevel_4.bin", "waterlevel_5.bin"
    };
}

ShakedownHawaiiExporter.zip

Extract ShakedownHawaiiExporter.exe and run with the following arguments:

• -e <Game Directory> e.g. ShakedownHawaiiExporter.exe -e "D:\Games\Shakedown" to export all files
• -i <Import Directory> e.g. ShakedownHawaiiExporter.exe -i "D:\Games\Shakedown\Dump" to import/pack all files into a new gamedata.bfp

Some notes:

• Exporting files will generate a new subfolder called Dump
• Files will be named correctly if known, [namehash].bin not known or script_xx.bin if a script
• Importing will import all files within the specified folder - be warned!
• The new gamedata.bfp generated from an import will be found in the specified folder, this is to prevent overwriting any legitimate files

Can you make another for Retro City Rampage?

@Pugemon Had a quick look and the format is the same for Retro City Rampage DX on Windows the only difference being the file names. I couldn't get a copy of the original game so can't confirm if it works for that too

All the filenames I could dump are below, the rest seem to be unnamed tsv scripts:

private static readonly string[] KnownNames = new[]
{
    "changes.txt", "credits.txt", "enemydefs.bin", "fonts.bin", "gametext.bxt",
    "objectives.bxt", "records.bxt", "tv_arcade.dds", "tv_uhf.dds", "tv_model6502.dds",
    "tv_lasertube.dds", "tv_pixel.dds", "tv_videobrick.dds", "tv_bittrip.dds", "tv_vmb.dds",
    "tv_zoomed.dds", "tv_pattern.dds", "cardefs.bin", "character.bin", "program.bin", "pedpalremap.bin",
    "regions.bin", "interiors.bin", "cutscenedefs.bin", "enemydefs.bin", "peddefs.bin", "ytoz.bin",
    "palettes.bin",  "roadslo.bin", "roadshi.bin", "anidefs.bin", "anims.bin", "vbl.map", "city_baked_bgpatches.map",
    "bgpatches.map", "arcadegames.map", "racer.map", "logos.map", "testyourgut.map", "testyourgut2.map",
    "testyourgut_harley.map", "hud.map", "frontend.map", "city_baked_bgpatches.map", "bgpatches.map", "minimap.map",
    "title.map", "achievements.map", "MG_Cards.map", "mg_lineup.map", "nes.pal", "nes2.pal", "sms.pal", "2600.pal",
    "c64.pal", "apple.pal", "ega.pal", "pccga1.pal", "pccga2.pal", "green.pal", "win16.pal", "zx.pal", "monochrome.pal",
    "videobrick.pal", "videobrick2.pal", "virtualburn.pal", "sepia.pal", "st.pal", "unsaturated.pal", "liberty.pal",
    "nextgen.pal", "saturated.pal", "mode13h.pal", "rcr.pal", "demo_4x3_1.rec", "demo_4x3_2.rec", "demo_16x9_1.rec", "demo_16x9_2.rec",
    "cars.cls", "minimap.raw", "npal_blk.raw", "npal_wht.raw", "tiles_bg_title.chr", "tiles_sp.chr", "tiles_ui.chr", "city.cls",
    "city.map", "city.set", "tiles_bg_0.chr", "tiles_bg_1.chr", "tiles_bg_anims.chr", "city_baked_bg_0.chr", "city_baked_bg_1.chr",
    "city_baked_bg_anims.chr", "city_baked.map", "city_baked.set", "racer_0.tsv", "racer_1.tsv", "racer_2.tsv", "racer_3.tsv",
    "racer_4.tsv", "racer_5.tsv", "racer_6.tsv", "racer_7.tsv", "nav_0.bin", "nav_1.bin",
};