PlayStation 3 hook manager

HookManager.cpp

#include "HookManager.h"
#include "Syscalls.h"

HookManager g_HookManager;

HookManager::HookManager(uint32_t mainHookAddress)
    : hookAddress(mainHookAddress)
{
    // Copy HookManager::RedirectToHandler to mainHookAddress because we can't branch to it directly.
    WriteProcessMemory(sys_process_getpid(), mainHookAddress, *(uint32_t**)RedirectToHandler, 12 * 4);
}

__attribute__((noinline))
void HookManager::Handler(HookContext* ctx)
{
    // Look through all our hooks to find the corresponding callback
    for (auto hook : hookStack)
    {
        if ((ctx->lr - 4) != hook.address)
            continue;

        hook.callback(ctx);
        return;
    }

    // Just in case our hook callback isn't executed
    __builtin_trap();
}

void HookManager::Add(uint32_t address, void(*callback)(HookContext*))
{
    // If the address isn't valid don't go further
    if (address < 0x10000)
        return;

    // Setup the hook
    HookData hook
    {
        callback,
        address,
        *(uint32_t*)address
    };

    // Branch address to HookManager::RedirectToHandler()
    *(uint32_t*)address = (0x48000001 + ((hookAddress - address) & 0x3FFFFFF));
    hookStack.push_back(hook);
}

void HookManager::AddXref(uint32_t address, void(*callback)(HookContext*))
{
    for (auto addr : FindXrefs(address))
        Add(addr, callback);
}

void HookManager::ClearAll()
{
    // Restore original instructions
    for (auto hook : hookStack)
        *(uint32_t*)hook.address = hook.instruction;

    hookStack.clear();
}

uint32_t HookManager::ResolveBranch(uint32_t address)
{
    uint32_t instruction = *(uint32_t*)address;
    int offset = instruction & 0x3FFFFFC;

    // If the MSB for branch offset is set make it negative
    if (offset & (1 << 25))
        offset |= ~0x03FFFFFF;

    return address + offset;
}

std::vector<uint32_t> HookManager::FindXrefs(uint32_t address)
{
    uint32_t pos = 0x10200;
    uint32_t end = *(uint32_t*)0x1001C;
    std::vector<uint32_t> xrefs;

    while (pos < end)
    {
        uint32_t instruction = *(uint32_t*)pos;

        // Make sure it is a branch instruction
        if ((instruction & 0xFC000000) == (18 << 26))
        {
            if (ResolveBranch(pos) == address)
                xrefs.push_back(pos);
        }

        pos += 4;
    }
    return xrefs;
}

__attribute__((naked))
void HookManager::RedirectToHandler()
{
    asm(
        // Allocate enough space to save the registers
        "stdu      %r1, -0x300(%r1);"

        // Save r2, r3 and the link register now because we will need them
        "std       %r2, 0x80(%r1);"
        "std       %r3, 0x88(%r1);"
        "mflr      %r2;"
        "std       %r2, 0x170(%r1);"

        // Get the opd pointer from HookManager::HandleRegisters() and call it
        "li        %r3, 0;"
        "oris      %r3, %r3, _ZN11HookManager15HandleRegistersEv@h;"
        "ori       %r3, %r3, _ZN11HookManager15HandleRegistersEv@l;"
        "lwz       %r2, 4(%r3);"
        "lwz       %r3, 0(%r3);"
        "mtlr      %r3;"
        "blr;"
        );
}

__attribute__((naked))
void HookManager::HandleRegisters()
{
    asm(
        // Save r0
        "std       %r0, 0x70(%r1);"

        // Save r1 as r1 + 0x300 to get the hooked function stack address
        "addi      %r0, %r1, 0x300;"
        "std       %r0, 0x78(%r1);"

        // Save all gprs
        "std       %r4, 0x90(%r1);"
        "std       %r5, 0x98(%r1);"
        "std       %r6, 0xA0(%r1);"
        "std       %r7, 0xA8(%r1);"
        "std       %r8, 0xB0(%r1);"
        "std       %r9, 0xB8(%r1);"
        "std       %r10, 0xC0(%r1);"
        "std       %r11, 0xC8(%r1);"
        "std       %r12, 0xD0(%r1);"
        "std       %r13, 0xD8(%r1);"
        "std       %r14, 0xE0(%r1);"
        "std       %r15, 0xE8(%r1);"
        "std       %r16, 0xF0(%r1);"
        "std       %r17, 0xF8(%r1);"
        "std       %r18, 0x100(%r1);"
        "std       %r19, 0x108(%r1);"
        "std       %r20, 0x110(%r1);"
        "std       %r21, 0x118(%r1);"
        "std       %r22, 0x120(%r1);"
        "std       %r23, 0x128(%r1);"
        "std       %r24, 0x130(%r1);"
        "std       %r25, 0x138(%r1);"
        "std       %r26, 0x140(%r1);"
        "std       %r27, 0x148(%r1);"
        "std       %r28, 0x150(%r1);"
        "std       %r29, 0x158(%r1);"
        "std       %r30, 0x160(%r1);"
        "std       %r31, 0x168(%r1);"

        // Save all sprs
        "mfctr     %r0;"
        "std       %r0, 0x178(%r1);"
        "mfcr      %r0;"
        "std       %r0, 0x180(%r1);"
        "mfxer     %r0;"
        "std       %r0, 0x188(%r1);"

        // Save all fprs
        "stfd      %f0, 0x190(%r1);"
        "stfd      %f1, 0x198(%r1);"
        "stfd      %f2, 0x1A0(%r1);"
        "stfd      %f3, 0x1A8(%r1);"
        "stfd      %f4, 0x1B0(%r1);"
        "stfd      %f5, 0x1B8(%r1);"
        "stfd      %f6, 0x1C0(%r1);"
        "stfd      %f7, 0x1C8(%r1);"
        "stfd      %f8, 0x1D0(%r1);"
        "stfd      %f9, 0x1D8(%r1);"
        "stfd      %f10, 0x1E0(%r1);"
        "stfd      %f11, 0x1E8(%r1);"
        "stfd      %f12, 0x1F0(%r1);"
        "stfd      %f13, 0x1F8(%r1);"
        "stfd      %f14, 0x200(%r1);"
        "stfd      %f15, 0x208(%r1);"
        "stfd      %f16, 0x210(%r1);"
        "stfd      %f17, 0x218(%r1);"
        "stfd      %f18, 0x220(%r1);"
        "stfd      %f19, 0x228(%r1);"
        "stfd      %f20, 0x230(%r1);"
        "stfd      %f21, 0x238(%r1);"
        "stfd      %f22, 0x240(%r1);"
        "stfd      %f23, 0x248(%r1);"
        "stfd      %f24, 0x250(%r1);"
        "stfd      %f25, 0x258(%r1);"
        "stfd      %f26, 0x260(%r1);"
        "stfd      %f27, 0x268(%r1);"
        "stfd      %f28, 0x270(%r1);"
        "stfd      %f29, 0x278(%r1);"
        "stfd      %f30, 0x280(%r1);"
        "stfd      %f31, 0x288(%r1);"

        // Pass a pointer to the hook context as r3
        "addi      %r3, %r1, 0x70;"

        // Call our hook handler
        "bl        ._ZN11HookManager11CallHandlerEP11HookContext;"
        "nop;"

        // Restore sprs
        "ld        %r0, 0x170(%r1);"
        "mtlr      %r0;"
        "ld        %r0, 0x178(%r1);"
        "mtctr     %r0;"
        "ld        %r0, 0x180(%r1);"
        "mtcr      %r0;"
        "ld        %r0, 0x188(%r1);"
        "mtxer     %r0;"

        // Restore gprs
        "ld        %r0, 0x70(%r1);"
        "nop;"
        "ld        %r2, 0x80(%r1);"
        "ld        %r3, 0x88(%r1);"
        "ld        %r4, 0x90(%r1);"
        "ld        %r5, 0x98(%r1);"
        "ld        %r6, 0xA0(%r1);"
        "ld        %r7, 0xA8(%r1);"
        "ld        %r8, 0xB0(%r1);"
        "ld        %r9, 0xB8(%r1);"
        "ld        %r10, 0xC0(%r1);"
        "ld        %r11, 0xC8(%r1);"
        "ld        %r12, 0xD0(%r1);"
        "ld        %r13, 0xD8(%r1);"
        "ld        %r14, 0xE0(%r1);"
        "ld        %r15, 0xE8(%r1);"
        "ld        %r16, 0xF0(%r1);"
        "ld        %r17, 0xF8(%r1);"
        "ld        %r18, 0x100(%r1);"
        "ld        %r19, 0x108(%r1);"
        "ld        %r20, 0x110(%r1);"
        "ld        %r21, 0x118(%r1);"
        "ld        %r22, 0x120(%r1);"
        "ld        %r23, 0x128(%r1);"
        "ld        %r24, 0x130(%r1);"
        "ld        %r25, 0x138(%r1);"
        "ld        %r26, 0x140(%r1);"
        "ld        %r27, 0x148(%r1);"
        "ld        %r28, 0x150(%r1);"
        "ld        %r29, 0x158(%r1);"
        "ld        %r30, 0x160(%r1);"
        "ld        %r31, 0x168(%r1);"

        // Restore fprs
        "lfd       %f0, 0x190(%r1);"
        "lfd       %f1, 0x198(%r1);"
        "lfd       %f2, 0x1A0(%r1);"
        "lfd       %f3, 0x1A8(%r1);"
        "lfd       %f4, 0x1B0(%r1);"
        "lfd       %f5, 0x1B8(%r1);"
        "lfd       %f6, 0x1C0(%r1);"
        "lfd       %f7, 0x1C8(%r1);"
        "lfd       %f8, 0x1D0(%r1);"
        "lfd       %f9, 0x1D8(%r1);"
        "lfd       %f10, 0x1E0(%r1);"
        "lfd       %f11, 0x1E8(%r1);"
        "lfd       %f12, 0x1F0(%r1);"
        "lfd       %f13, 0x1F8(%r1);"
        "lfd       %f14, 0x200(%r1);"
        "lfd       %f15, 0x208(%r1);"
        "lfd       %f16, 0x210(%r1);"
        "lfd       %f17, 0x218(%r1);"
        "lfd       %f18, 0x220(%r1);"
        "lfd       %f19, 0x228(%r1);"
        "lfd       %f20, 0x230(%r1);"
        "lfd       %f21, 0x238(%r1);"
        "lfd       %f22, 0x240(%r1);"
        "lfd       %f23, 0x248(%r1);"
        "lfd       %f24, 0x250(%r1);"
        "lfd       %f25, 0x258(%r1);"
        "lfd       %f26, 0x260(%r1);"
        "lfd       %f27, 0x268(%r1);"
        "lfd       %f28, 0x270(%r1);"
        "lfd       %f29, 0x278(%r1);"
        "lfd       %f30, 0x280(%r1);"
        "lfd       %f31, 0x288(%r1);"

        // Restore r1 and return
        "ld        %r1, 0x78(%r1);"
        "blr;"
        );
}

void HookManager::CallHandler(HookContext* ctx)
{
    g_HookManager.Handler(ctx);
}

HookManager.h

#pragma once

#undef vector
#include <vector>

struct HookContext
{
    // General purpose registers (gprs)
    uint64_t r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31;

    // Special purpose registers (sprs)
    uint64_t lr, ctr, cr, xer;

    // Floating point registers (fprs)
    double f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17, f18, f19, f20, f21, f22, f23, f24, f25, f26, f27, f28, f29, f30, f31;

    template<typename T>
    inline T& GetGpr(int index)
    {
        union reg_t
        {
            uint64_t packed;
            struct
            {
                char ___padding[sizeof(uint64_t) - sizeof(T)];
                T value;
            };
        };

        reg_t& reg = *(reg_t*)(&r0 + index);
        return reg.value;
    }

    inline double& GetFpr(int index)
    {
        return (&f0)[index];
    }
};

class HookManager
{
public:
    struct HookData
    {
        void(*callback)(HookContext*);
        uint32_t address;
        uint32_t instruction;
    };

public:
    // 0x10050 for vsh or 0x10080 for game are used by enstone but conflict with the ps3 toolbox rpc.
    // 0x10200 is always .init_proc which has just enough space to place our main hook
    HookManager(uint32_t mainHookAddress = 0x10200);

    void Handler(HookContext* ctx);
    void Add(uint32_t address, void(*callback)(HookContext*));
    void AddXref(uint32_t address, void(*callback)(HookContext*));
    void ClearAll();

private:
    uint32_t ResolveBranch(uint32_t address);
    std::vector<uint32_t> FindXrefs(uint32_t address);
    
    static void RedirectToHandler();
    static void HandleRegisters();
    static void CallHandler(HookContext* ctx);

private:
    std::vector<HookData> hookStack{};
    uint32_t hookAddress{};
};

extern HookManager g_HookManager;