Max Payne 2 .ras file encryption/decryption algorithm

payne.c

#include <stdint.h>
#include <stdio.h>

static uint8_t ror8(uint8_t input, size_t rot_bits)
{
    uint8_t rot = (uint8_t)(rot_bits % (sizeof(input) * 8));
    return (input >> rot) | (input << ((sizeof(input) * 8) - rot));
}

static uint8_t rol8(uint8_t input, size_t rot_bits)
{
    uint8_t rot = (uint8_t)(rot_bits % (sizeof(input) * 8));
    return (input << rot) | (input >> ((sizeof(input) * 8) - rot));
}

void encrypt(uint32_t *key_ptr, uint8_t *data, size_t data_len)
{
    // KEY = 0x26AC4D58
    // esi = [key_ptr 0x19fc84]
    if (*key_ptr == 0)
    {
        *key_ptr = 1;
    }

    // eax = $data_len [0x19fc80]
    // ecx = 0
    // [esp+0x10 bytes_written_ptr 0x19fc70] = ecx
    if (data_len == 0)
    {
        return;
    }

    // BEGIN LOOP?
    // ebp = $source [0x19fc7c]
    // edx = 0
    // eax = ecx // 0 in first loop
    for (size_t i = 0; i < data_len; ++i)
    {
        // edi = 5
        // eax, edx = edx:eax / edi, edx:eax % edi
        uint32_t mod5 = (uint32_t)(i % 5);

        // eax = 0xB92143FB
        uint32_t manipulation = 0xB92143FB;
        // ebx = (ebx & 0xFFFFFF00) | 0x06
        //=> constant
        // [scratch_space2 0x19fc74] = edx
        //= mod5

        // edx:eax = eax *signed* esi
        uint64_t product = (uint64_t)(((int64_t)(int32_t)manipulation) * ((int64_t)(int32_t)*key_ptr));
        uint32_t top_product = (uint32_t)(product >> 32);

        // edx = edx + esi
        uint32_t summy = top_product + *key_ptr;

        // esi = esi * 0xAB
        uint32_t easy = (uint32_t)((int32_t)(*key_ptr) * 0xAB);

        // edx = edx >>arith>> 7
        uint32_t shifty = (uint32_t)(((int32_t)summy) >> 7);

        // eax = edx
        // eax = eax >>zero>> 0x1F
        uint32_t downshift = ((uint32_t)shifty) >> 0x1F;

        // eax = eax + edx
        uint32_t summy2 = (uint32_t)(((int32_t)downshift) + shifty);

        // edx = (edx & 0xFFFFFF00) | *(uint8_t)(ebp 0x1a301b18 + ecx 0x00000000)
        uint8_t b = data[i];

        // eax = eax * 0x763D
        uint32_t proddy = (uint32_t)(((int32_t)summy2) * 0x763D);

        // esi = esi - eax
        // eax = esi
        uint32_t diffy = easy - proddy;

        // edx = (edx & 0xFFFFFF00) | ((edx & 0xFF) - (eax & 0xFF))
        uint8_t diffy2 = b - (uint8_t)(diffy & 0xFF);

        // [key_ptr 0x19fc84] = eax
        *key_ptr = diffy;

        // eax = (eax & 0xFFFFFF00) | (ecx & 0xFF)
        uint8_t i8 = (uint8_t)(i & 0xFF);

        // eax = (eax & 0xFFFFFF00) | ((eax & 0xFF) + 3)
        i8 += 3;

        // eax = (eax & 0xFFFFFF00) | ((eax & 0xFF) * (ebx & 0xFF)) // ebx & 0xFF == 0x06
        i8 *= 6;

        // edx = (edx & 0xFFFFFF00) | ((edx & 0xFF) ^ (eax & 0xFF))
        uint8_t xored = diffy2 ^ i8;

        // *(uint8_t)(ebp 0x1a301b18 + ecx 0x00000000) = (edx & 0xFF)
        data[i] = xored;

        // REMEMBER REGISTERS
        // ecx = [scratch_space2 0x19fc74 &0x00000000]
        //= mod5
        // ebx = [source 0x19fc7c]
        // ebx = ebx + [bytes_written_ptr 0x19fc70 &0x00000000]
        // al = (uint8_t)(*ebx 0x1a301b18)

        // al = al `ROR` (ecx & 0xFF)
        // (*ebx 0x1a301b18) = al
        data[i] = ror8(data[i], mod5);

        // RESTORE REGISTERS

        // eax = [bytes_written_ptr 0x19fc70 &0x00000000]
        // ecx = [data_len 0x19fc80 &0x00000024]
        // eax = eax + 1
        // [bytes_written_ptr 0x19fc70] = eax // &0x00000001
        // if (eax >= ecx) return;
        // ecx = [bytes_written_ptr 0x19fc70 &0x00000001]
        // esi = [key_ptr 0x19fc84 &0xff907a04]
        // END LOOP
    }
}

void decrypt(uint32_t *key_ptr, uint8_t *data, size_t data_len)
{
    // eax = [esp+0x0c key_ptr 0x19fd14]
    // test eax, eax
    // (stack moving here)
    if (*key_ptr == 0)
    {
        // [esp+0x24 0x19fd14] = 0x00000001
        *key_ptr = 1;
    }
    // eax = $data_len [esp+0x20 0x19fd10]
    // [esp+0x10 bytes_written_ptr 0x19fd00] = 0

    // test eax, eax
    if (data_len == 0)
    {
        return;
    }

    // ebp = $data [esp+0x1c 0x19fd0c]
    // BEGIN LOOP
    for (size_t i = 0; i < data_len; ++i)
    {
        // eax = [esp+0x10 bytes_written_ptr 0x19fd00]
        // edx = 0
        // ecx = 5
        // eax, edx = eax /u/ ecx, eax %u% ecx
        // [esp+0x14 scratch0 0x19fd04] = edx
        uint32_t mod5 = ((uint32_t)i) % 5;

        // STORE REGISTERS
        // ecx = [esp+0x24 scratch0 0x19fd04]
        // ebx = $data [esp+0x2c 0x19fd0c]
        // ebx = ebx + [esp+0x20 bytes_written_ptr 0x19fd00]
        // al = [ebx]
        // al = al `ROL` (ecx & 0xFF)
        // [ebx] = al
        data[i] = rol8(data[i], mod5);
        // RESTORE REGISTERS

        // ecx = [esp+0x24 key_ptr 0x19fd14]
        // eax = 0xB92143FB
        uint32_t manipulation = 0xB92143FB;

        // edx:eax = eax *s* ecx
        uint64_t product = (uint64_t)(((int64_t)(int32_t)manipulation) * ((int64_t)(int32_t)*key_ptr));
        uint32_t top_product = (uint32_t)(product >> 32);

        // edx = edx + ecx
        uint32_t sum = top_product + *key_ptr;

        // ecx = ecx *s* 0xAB
        uint32_t proddy = (uint32_t)(((int32_t)*key_ptr) * 0xAB);

        // edx = edx >>a>> 7
        uint32_t shifty = (uint32_t)(((int32_t)sum) >> 7);

        // eax = edx
        // eax = eax >>l>> 0x1F
        uint32_t shifty2 = (shifty >> 0x1F);

        // eax = eax + edx
        uint32_t summy = shifty2 + shifty;

        // eax = eax *s* 0x763D
        uint32_t proddy2 = (uint32_t)(((int32_t)summy) * 0x763D);

        // ecx = ecx - eax
        uint32_t diffy = proddy - proddy2;

        // edx = ecx
        // ecx = [esp+0x10 bytes_written_ptr 0x19fd00]
        // al = cl
        uint8_t i8 = (uint8_t)(i & 0xFF);

        // al = al + 3
        i8 += 3;

        // bl = 6
        // ah:al = al *s* bl
        i8 *= 6;

        // al = al ^ [ebp+ecx]
        i8 ^= data[i];

        // al = al + dl
        i8 += (uint8_t)(diffy & 0xFF);

        // [ebp+ecx] = al
        data[i] = i8;

        // eax = [esp+0x20 data_len 0x19fd10]
        // ecx = ecx + 1
        // [esp+0x24 key_ptr 0x19fd14] = edx
        *key_ptr = diffy;

        // [esp+0x10 bytes_written_ptr 0x19fd00] = ecx
        // if (ecx < eax) loop;
        // END LOOP
    }
}

static void hexdump(FILE *f, uint8_t *data, size_t data_len)
{
    for (size_t i = 0; i < data_len; i += 8)
    {
        size_t j;

        fprintf(f, "0x%08zX  ", i);

        for (j = 0; j < 8 && (i + j) < data_len; ++j)
        {
            fprintf(f, "0x%02X ", data[i + j]);
        }

        for (; j < 8; ++j)
        {
            fputs("     ", f);
        }

        fputc('|', f);

        for (j = 0; j < 8 && (i + j) < data_len; ++j)
        {
            if (data[i + j] < 0x20 || data[i + j] >= 0x7F)
            {
                fputc('.', f);
            }
            else
            {
                fputc(data[i + j], f);
            }
        }

        fputc('|', f);

        fputs("\n", f);
    }
}

int main(void)
{
    uint8_t plaintext[] = {
        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,  0x2A, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00,
        0x9A, 0x99, 0x99, 0x3F, 0xDE, 0x14, 0x50, 0x5A,  0xDB, 0xD4, 0x40, 0x93, 0x04, 0xA4, 0xF7, 0x03,
        0x04, 0x00, 0x00, 0x00
    };
    uint8_t ciphertext[] = {
        0xBF, 0xE9, 0xFE, 0x23, 0x49, 0xF9, 0x9A, 0xBD,
        0x92, 0xA9, 0xA3, 0x34, 0x70, 0x0B, 0x19, 0x58, 0xB6, 0xB1, 0xC2, 0xD0, 0x91, 0x50, 0xCB, 0xE1,
        0x2B, 0xDB, 0x5F, 0x26, 0xD0, 0x6D, 0xCA, 0x40, 0xE1, 0x5B, 0x7D, 0xE7,

        0x1C, 0xE9, 0x3D, 0x23, 0x69, 0xF9, 0x1F, 0xBD,
        0xDD, 0xA9, 0xA3, 0x34, 0xCB, 0x0B, 0xE9, 0x58, 0xFB, 0x57, 0x36, 0xEC, 0x9E, 0x79, 0x45, 0xC6,
        0x79, 0x37, 0x0E, 0x05, 0xB0, 0x2B, 0xDF, 0xC2, 0xE3, 0x5B, 0x0C, 0xE7,

        0x7A, 0x59, 0x87, 0x5E, 0x7A, 0xE9, 0x1A, 0xE9,
        0xFE, 0x23, 0x69, 0xF9, 0x9A, 0xBD, 0xDD, 0xA9, 0xA3, 0x34, 0xCB, 0x0B, 0x19, 0x58, 0xFB, 0x57,
        0xAA, 0xF2, 0xFA, 0x79, 0xD5, 0x85, 0x49, 0x37, 0xBF, 0x4B, 0x70, 0x2B,

        0xD3, 0xC2, 0xE0, 0x5B, 0x7D, 0xE7, 0xF9, 0x59
    };
    size_t plain_len = sizeof(plaintext)/sizeof(plaintext[0]);
    size_t cipher_len = sizeof(ciphertext)/sizeof(ciphertext[0]);

    hexdump(stdout, plaintext, plain_len);
    hexdump(stdout, ciphertext, cipher_len);

    uint32_t plain_key = 0x106a20d4;
    encrypt(&plain_key, plaintext, plain_len);

    uint32_t crypt_key;
    crypt_key = 0xE92BD254;
    decrypt(&crypt_key, ciphertext, 0x24);
    crypt_key = 0xE92BD254;
    decrypt(&crypt_key, ciphertext + 0x24, 0x2a);
    crypt_key = 0xE92BD254;
    decrypt(&crypt_key, ciphertext + 0x24 + 0x2a, 0x2a);

    hexdump(stdout, plaintext, plain_len);
    hexdump(stdout, ciphertext, cipher_len);

    return 0;
}