RP6L unpacker. Usage: rp6l.lua <filename.rpack> [output_directory]

mod_dds_header.lua

assert(_VERSION == "Lua 5.3")

-- DDS header       [32]
_fourcc  = 1;    _size    = 2;    _flags   = 3;    _height  = 4
_width   = 5;    _pitch   = 6;    _depth   = 7;    _mipmaps = 8
-- reserved         [44]
--  9 10 11 12
-- 13 14 15 16
-- 17 18 19
-- DDSPixelFormat   [32]
_psize   = 20;   _pflags  = 21;   _pfourcc = 22;   _pbpp    = 23
_prmask  = 24;   _pgmask  = 25;   _pbmask  = 26;   _pamask  = 27
-- DDSCaps          [16]
_caps1   = 28;   _caps2   = 29;   _caps3   = 30;   _caps4   = 31
--
_notused = 32
-- DDSHeader10      [20]
_dxgiFmt = 33;   _resDim  = 34;   _miscFlag = 35;  _arraySize   = 36
_reserved   = 37


local function MAKEFOURCC(fourcc)
--    local a, b, c, d = string.byte(fourcc, 1, 4)
--    return a | (b << 8) | (c << 16) | (d << 24)
    return string.unpack("<I", fourcc)
end

FOURCC_DDS  = MAKEFOURCC("DDS ")
FOURCC_DXT1 = MAKEFOURCC("DXT1")
FOURCC_DXT2 = MAKEFOURCC("DXT2")
FOURCC_DXT3 = MAKEFOURCC("DXT3")
FOURCC_DXT4 = MAKEFOURCC("DXT4")
FOURCC_DXT5 = MAKEFOURCC("DXT5")
FOURCC_RXGB = MAKEFOURCC("RXGB")
FOURCC_ATI1 = MAKEFOURCC("ATI1")
FOURCC_ATI2 = MAKEFOURCC("ATI2")
FOURCC_A2XY = MAKEFOURCC("A2XY")
FOURCC_DX10 = MAKEFOURCC("DX10")


Format_RGB      = 0
Format_RGBA     = Format_RGB
-- DX9 formats.
Format_DXT1     = 1
Format_DXT1a    = 2 -- DXT1 with binary alpha.
Format_DXT3     = 3
Format_DXT5     = 4
Format_DXT5n    = 5 -- Compressed HILO: R=1, G=y, B=0, A=x
-- DX10 formats.
Format_BC1      = Format_DXT1
Format_BC1a     = Format_DXT1a
Format_BC2      = Format_DXT3
Format_BC3      = Format_DXT5
Format_BC3n     = Format_DXT5n
Format_BC4      = 6 -- ATI1
Format_BC5      = 7 -- 3DC, ATI2

DDSD_CAPS           = 0x00000001
DDSD_HEIGHT         = 0x00000002
DDSD_WIDTH          = 0x00000004
DDSD_PITCH          = 0x00000008
DDSD_PIXELFORMAT    = 0x00001000
DDSD_MIPMAPCOUNT    = 0x00020000
DDSD_LINEARSIZE     = 0x00080000
DDSD_DEPTH          = 0x00800000

DDSCAPS_COMPLEX     = 0x00000008
DDSCAPS_TEXTURE     = 0x00001000
DDSCAPS_MIPMAP      = 0x00400000
DDSCAPS2_VOLUME     = 0x00200000
DDSCAPS2_CUBEMAP    = 0x00000200
DDSCAPS2_CUBEMAP_ALL_FACES = 0x0000FC00

DDPF_ALPHAPIXELS    = 0x00000001
DDPF_ALPHA          = 0x00000002
DDPF_FOURCC         = 0x00000004
DDPF_RGB            = 0x00000040
DDPF_NORMAL         = 0x80000000    -- Custom nv flag

-- enum DXGI_FORMAT
DXGI_FORMAT_UNKNOWN = 0
-- enum D3D10_RESOURCE_DIMENSION
D3D10_RESOURCE_DIMENSION_UNKNOWN    = 0
D3D10_RESOURCE_DIMENSION_BUFFER     = 1
D3D10_RESOURCE_DIMENSION_TEXTURE1D  = 2
D3D10_RESOURCE_DIMENSION_TEXTURE2D  = 3
D3D10_RESOURCE_DIMENSION_TEXTURE3D  = 4


-- init
DDSHeader = {}

function DDSHeader:new()
    self[_fourcc]   = FOURCC_DDS
    self[_size]     = 124
    self[_flags]    = DDSD_CAPS | DDSD_PIXELFORMAT
    self[_height]   = 0
    self[_width]    = 0
    self[_pitch]    = 0
    self[_depth]    = 0
    self[_mipmaps]  = 0
    -- reserved
    for i = 9, 17 do
        self[i]     = 0
    end
    self[18]        = MAKEFOURCC("_LUA")
    self[19]        = MAKEFOURCC("_DDS")
    -- pixel format
    self[_psize]    = 32
    self[_pflags]   = 0
    self[_pfourcc]  = 0
    self[_pbpp]     = 0
    self[_prmask]   = 0
    self[_pgmask]   = 0
    self[_pbmask]   = 0
    self[_pamask]   = 0
    -- caps
    self[_caps1]    = DDSCAPS_TEXTURE
    self[_caps2]    = 0
    self[_caps3]    = 0
    self[_caps4]    = 0
    self[_notused]  = 0
    -- d3d10
    self[_dxgiFmt]  = DXGI_FORMAT_UNKNOWN
    self[_resDim]   = D3D10_RESOURCE_DIMENSION_UNKNOWN
    self[_miscFlag] = 0
    self[_arraySize]= 0
    self[_reserved] = 0
end


function DDSHeader:set_width(num)
    self[_flags] = self[_flags] | DDSD_WIDTH
    self[_width] = num
end

function DDSHeader:set_height(num)
    self[_flags] = self[_flags] | DDSD_HEIGHT
    self[_height] = num
end

function DDSHeader:set_depth(num)
    self[_flags] = self[_flags] | DDSD_DEPTH
    self[_depth] = num
end

function DDSHeader:set_mipmaps(num)
    if num == 0 or num == 1 then
        self[_flags] = self[_flags] & ~DDSD_MIPMAPCOUNT
        if self[_caps2] == 0 then
            self[_caps1] = DDSCAPS_TEXTURE
        else
            self[_caps1] = DDSCAPS_TEXTURE | DDSCAPS_COMPLEX
        end
    else
        self[_flags] = self[_flags] | DDSD_MIPMAPCOUNT
        self[_mipmaps] = num
        self[_caps1] = self[_caps1] | DDSCAPS_COMPLEX | DDSCAPS_MIPMAP
    end
end

function DDSHeader:set_texture_2d()
    self[_resDim] = D3D10_RESOURCE_DIMENSION_TEXTURE2D
end

function DDSHeader:set_texture_3d()
    self[_caps2] = DDSCAPS2_VOLUME

    self[_resDim] = D3D10_RESOURCE_DIMENSION_TEXTURE3D
end

function DDSHeader:set_texture_cube()
    self[_caps1] = self[_caps1]| DDSCAPS_COMPLEX
    self[_caps2] = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_ALL_FACES

    self[_resDim] = D3D10_RESOURCE_DIMENSION_TEXTURE2D
    self[_arraySize] = 6
end

function DDSHeader:set_linear(size)
    self[_flags] = self[_flags] & ~DDSD_PITCH
    self[_flags] = self[_flags] | DDSD_LINEARSIZE
    self[_pitch] = size
end

function DDSHeader:set_pitch(pitch)
    self[_flags] = self[_flags] & ~DDSD_LINEARSIZE
    self[_flags] = self[_flags] | DDSD_PITCH
    self[_pitch] = pitch
end

function DDSHeader:set_fourcc(fourcc)
    self[_pflags] = DDPF_FOURCC
    self[_pfourcc] = fourcc

    if self[_pfourcc] == FOURCC_ATI2 then
        self[_pbpp] = FOURCC_A2XY
    else
        self[_pbpp] = 0
    end

    self[_prmask] = 0
    self[_pgmask] = 0
    self[_pbmask] = 0
    self[_pamask] = 0
end

function DDSHeader:set_pixel_format(bpp, rmask, gmask, bmask, amask)
    rmask = rmask or 0
    gmask = gmask or 0
    bmask = bmask or 0
    amask = amask or 0

    assert((rmask & gmask) == 0)
    assert((rmask & bmask) == 0)
    assert((rmask & amask) == 0)
    assert((gmask & bmask) == 0)
    assert((gmask & amask) == 0)
    assert((bmask & amask) == 0)

    self[_pflags] = DDPF_RGB
    if amask ~= 0 then
        self[_pflags] = self[_pflags] | DDPF_ALPHAPIXELS
    end

    if bpp == 0 then
        local total = rmask | gmask | bmask | amask
        while total ~= 0 do
            bpp = bpp + 1
            total = total >> 1
        end
    end

    assert(bpp > 0 and bpp <= 32)

    -- align to 8
    if     bpp <= 8  then bpp = 8
    elseif bpp <= 16 then bpp = 16
    elseif bpp <= 24 then bpp = 24
    else                  bpp = 32
    end

    self[_pfourcc] = 0
    self[_pbpp] = bpp
    self[_prmask] = rmask
    self[_pgmask] = gmask
    self[_pbmask] = bmask
    self[_pamask] = amask
end

function DDSHeader:set_normal_flag(b)
    if b then
        self[_pflags] = self[_pflags] | DDPF_NORMAL
    else
        self[_pflags] = self[_pflags] & ~DDPF_NORMAL
    end
end

function DDSHeader:hasDX10Header()
    return self[_pfourcc] == FOURCC_DX10    -- This is according to AMD
    --return self[_pfourcc] == 0              -- This is according to MS
end


-------------------------------------------------------------------------------

local function computePitch(width, bpp)
    local pitch = (bpp + 7) // 8 * width
    return (pitch + 3) // 4 * 4
end

local BS = { 8, 8, 16, 16, 16, 8, 16 }

local function blockSize(fmt)
    return BS[fmt] or 0
end

local function computeImageSize(width, height, depth, bpp, fmt)
    if fmt == Format_RGBA then
        return computePitch(width, bpp) * depth * height
    else
        return blockSize(fmt) * ((width + 3) // 4) * ((height + 3) // 4)
    end
end

-------------------------------------------------------------------------------

function DDSHeader:generate(width, height, mips, fmt, bpp, cubemap, depth, normal)
    width   = width     or 256
    height  = height    or 256
    mips    = mips      or 1
    fmt     = fmt       or Format_DXT1
    bpp     = bpp       or 16
    cubemap = cubemap   or false
    depth   = depth     or 0
    normal  = normal    or false

    self:set_width(width)
    self:set_height(height)
    self:set_mipmaps(mips)

    if depth > 0 then
        self:set_depth(depth)
        self:set_texture_3d()
    else
        self:set_texture_2d()
    end

    if cubemap then
        self:set_texture_cube()
    end

    if fmt == Format_RGBA then
        self:set_pitch(computePitch(width, bpp))
        if bpp == 8 then
            self:set_pixel_format(bpp, 0x0f00, 0x00f0, 0x000f, 0xf000)
        elseif bpp == 16 then
            -- TODO: check this
            self:set_pixel_format(bpp, 0x03f000, 0x000fc0, 0x00003f, 0xfc0000)
        elseif bpp == 32 then
            self:set_pixel_format(bpp, 0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000)
        end
    else
        self:set_linear(computeImageSize(width, height, depth, bpp, fmt))
        if     fmt == Format_DXT1 or fmt == Format_DXT1a then
            self:set_fourcc(FOURCC_DXT1)
            if normal then self:set_normal_flag(true) end
        elseif fmt == Format_DXT3 then
            self:set_fourcc(FOURCC_DXT3)
        elseif fmt == Format_DXT5 then
            self:set_fourcc(FOURCC_DXT5)
        elseif fmt == Format_DXT5n then
            self:set_fourcc(FOURCC_DXT5)
            if normal then self:set_normal_flag(true) end
        elseif fmt == Format_BC4 then
            self:set_fourcc(FOURCC_ATI1)
        elseif fmt == Format_DXT3 then
            self:set_fourcc(FOURCC_ATI2)
            if normal then self:set_normal_flag(true) end
        end
    end

    -- TODO: swapBytes()

    local header_size = 128 // 4
    if self:hasDX10Header() then
        header_size = (128 + 20) // 4
    end

    local data = {}
    for i = 1, header_size do
        table.insert(data, string.pack("<I", self[i]))
    end
    return table.concat(data)
end

rp6l.lua

local zlib = require("zlib")

local FN = assert(arg[1])
local OUT = arg[2] or "."

local r
local uint32 = function() return string.unpack("I", r:read(4)) end
local uint16 = function() return string.unpack("H", r:read(2)) end
local uint8  = function() return string.unpack("B", r:read(1)) end

r = assert(io.open(FN, "rb"))

assert("RP6L" == r:read(4))

print("header", r:seek())
local header = {
    version = uint32(),
    c1 = uint8(),
    c2 = uint8(),
    c3 = uint8(),
    c4 = uint8(),
    parts = uint32(),
    sections = uint32(),
    files = uint32(),
    fnames_sz = uint32(),
    fnames = uint32(),
    unk = uint32(), -- ???
}

print("section", r:seek())
local section = {}
for i = 1, header.sections do
    local s = {}
    s.filetype = uint8()
    s.type2 = uint8()
    s.type3 = uint8()
    s.type4 = uint8()
    s.offset = uint32()
    s.unpackedSize = uint32()
    s.packedSize = uint32()
    s.unk1 = uint32()
    table.insert(section, s)
end

print("parts", r:seek())
local filepart = {}
for i = 1, header.parts do
    local p = {}
    p.sectionIndex = uint8() +1
    p.unk1 = uint8()
    p.fileIndex = uint16() +1
    p.offset = uint32()
    p.size = uint32()
    p.unk2 = uint32()
    table.insert(filepart, p)
end

print("filemap", r:seek())
local filemap = {}
for i = 1, header.fnames do
    local m = {}
    m.partsCount = uint8()
    m.unk1 = uint8()
    m.filetype = uint8()
    m.unk2 = uint8()
    m.fileIndex = uint32() +1
    m.firstPart = uint32() +1
    table.insert(filemap, m)
end

print("fidx", r:seek())
local fname_idx = {}
for i = 1, header.fnames do
    fname_idx[i] = uint32()
end

local fname_off = r:seek()
print("fnames", fname_off)
local filename = {}
for i = 1, #fname_idx do
    r:seek("set", fname_off + fname_idx[i])
    local chars = {}
    while true do
        local char = r:read(1)
        if char == "\x00" then break end
        table.insert(chars, char)
    end
    local name = table.concat(chars)
    filename[i] = name
end

print("data", r:seek())


local strfmt = [[
ver: %d, compress: %02d-%02d-%02d-%02d
parts: %d, sections: %d, files: %d, fnames: %d/%d, block: %d
]]
print(strfmt:format(
        header.version, header.c1, header.c2, header.c3, header.c4,
        header.parts, header.sections, header.files,
        header.fnames_sz, header.fnames, header.unk))

--[[
print("#      ft1 ft2 ft3 ft4     offset   unpacked     packed unknown")
for i = 1, #section do
    local s = section[i]
    print(("s%05d %3d %3d %3d %3d %10d %10d %10d %d"):format(
            i, s.filetype, s.type2, s.type3, s.type4,
            s.offset, s.unpackedSize, s.packedSize, s.unk1))
end
print()

print("#     sidx unk  fidx        off        unp unk2")
for i = 1, #filepart do
    local p = filepart[i]
    print(("p%05d %3d %3d %5d %10d %10d %d"):format(
            i, p.sectionIndex, p.unk1, p.fileIndex, p.offset, p.size, p.unk2))
end
print()

print("#     prts unk ftp unk    fileidx      first")
for i = 1, #filemap do
    local m = filemap[i]
    print(("m%05d %3d %3d %3d %3d %10d %10d"):format(
            i, m.partsCount, m.unk1, m.filetype, m.unk2, m.fileIndex, m.firstPart))
end
print()
]]

for i = 1, #section do
    local s = section[i]
    local pack = s.packedSize
    local unpk = s.unpackedSize
    r:seek("set", s.offset)
    print("s"..i, pack, unpk, pack > 0 and "packed" or "not packed")
    local data = r:read(unpk)
    if pack > 0 then
        s.data = zlib.inflate()(data)
    else
        s.data = data
    end
end

r:close()


require("mod_dds_header")
local dds = DDSHeader

local fmt = {
    [2] = Format_RGBA, --bpp 32
    [3] = Format_RGBA, --bpp 32
    [14] = Format_DXT3,
    [17] = Format_DXT1,
    [18] = Format_DXT3,
    [19] = Format_DXT5,
    [33] = Format_DXT5,
}

local function process_texture(data)
    local width = string.unpack("H", data:sub(1, 2))
    local height = string.unpack("H", data:sub(3, 4))
    local mips = string.unpack("H", data:sub(9, 12))
    local dxt = string.unpack("I", data:sub(13, 16))
    --print("\n"..width, height, mips, dxt)
    
    local dxtfmt = fmt[dxt]
    assert(dxtfmt, dxt)
    local bpp = (2 == dxt or 3 == dxt) and 32 or 16
    
    dds:new()
    local t = {}
    t[1] = dds:generate(width, height, mips, dxtfmt, bpp)
    t[2] = data:sub(152, -1)
    return table.concat(t)
end


local data
for i = 1, header.fnames do
    local ext
    local e = filemap[i].filetype
    if     16 == e then ext = "msh"
    elseif 32 == e then ext = "dds" --"tex"
    elseif 48 == e then ext = "shd"
    elseif 64 == e then ext = "anm"
    elseif 80 == e then ext = "fx"
    else                ext = tostring(e)
    end

    local fn = filename[i] ..".".. ext
    local m = filemap[i]
    local ptr = m.firstPart --+1
    
    io.write(i, "\\", header.fnames, "\t", fn, ": ")
    local w = assert(io.open(OUT .."/".. fn, "w+b"))

    local count = m.partsCount
    local out = {}
    while count > 0 do
        local p = filepart[ptr]
        local sidx = p.sectionIndex --+1
        local size = p.size
        local s = section[sidx]
        local sect = s.filetype
        local offs = p.offset

        io.write("+", sect)
        table.insert(out, s.data:sub(offs+1, offs+size))

        ptr = ptr + 1
        count = count - 1
    end
    print()
    
    out = table.concat(out)
    if 32 == e then
        out = process_texture(out)
    end
    w:write(out)
    w:close()
end