Compression using RDP API

rdp_pack.cpp

/**
    Compression using undocumented API in rdpbase.dll

    RDPCompressEx supports four algorithms : MPPC-8K, MPPC-64K, NCRUSH and XCRUSH.

    This code supports all except NCRUSH.
    The MPPC compression ratio is very similar to LZSS, so this could be quite useful for shellcode trying to evade detection.
    
    NCRUSH compression appears to work but fails for decompression.
    
    cl /EHsc rdp_pack.cpp
    compress: rdp_pack e0 infile outfile.8k
    decompress: rdp_pack d0 infile.8k outfile
*/
        
#include <windows.h>

#include <cstdio>
#include <cstdint>
#include <cstdlib>
#include <cstring>

#include <string>
#include <vector>
#include <iostream>
#include <fstream>
#include <sstream>
#include <iomanip>

// type of compression
#define PACKET_COMPR_TYPE_8K    0 // MPPC
#define PACKET_COMPR_TYPE_64K   1 // MPPC
#define PACKET_COMPR_TYPE_RDP6  2 // NCRUSH
#define PACKET_COMPR_TYPE_RDP61 3 // XCRUSH
#define PACKET_COMPR_TYPE_RDP8  4 // MSDN references this, but I doubt it's actually supported by OS

#define PACKET_ENCRYPTED  0x10

// Indicates that RDP 5.0 bulk compression (see [MS-RDPBCGR] section 3.1.8.4.2) was used.
//#define PACKET_COMPR_TYPE_64K 0x01 

// The data in the MatchCount, MatchDetails, and Literals fields has been compressed with the level-2 compressor.
#define PACKET_COMPRESSED 0x20 

// The decompressed data MUST be placed at the beginning of the level-2 history buffer.
#define PACKET_AT_FRONT 0x40

// The level-2 history buffer MUST be reinitialized (by filling it with zeros).
#define PACKET_FLUSHED 0x80

// The level-1 history buffer MUST be reinitialized (by filling it with zeros).
#define L1_PACKET_AT_FRONT 0x04

// No compression was performed. In this case, the MatchCount and MatchDetails fields MUST NOT be present. 
// The Literals field MUST be present.
#define L1_NO_COMPRESSION 0x02

// Compression with the level-1 compressor was performed and the MatchCount 
// and MatchDetails fields MUST be present and contain at least one match. The Literals field MUST also be present.
#define L1_COMPRESSED 0x01

// Indicates that additional level-2 compression has been performed on the level-1 compressor output 
// and that the Level2ComprFlags /field contains valid data and MUST be processed.
#define L1_INNER_COMPRESSION 0x10

// ******************************************************
// Needed to allocate memory for compression context.
typedef
DWORD (WINAPI *_RDPCompress_GetContextSize)(DWORD ComprType);

// Initialise compression context.
typedef
void (WINAPI *_RDPCompress_InitSendContext)(void *ctx, SIZE_T ctx_len, DWORD ComprType);

// Compress inbuf and store in outbuf
typedef
BYTE (WINAPI *_RDPCompress)(DWORD ComprType, void *inbuf, void *outbuf, PDWORD outlen, void *ctx);

typedef
BYTE (WINAPI *_RDPCompressEx)(DWORD opt, void *inbuf, DWORD inlen, void *outbuf, PDWORD outlen, void *ctx);

// ******************************************************
typedef
DWORD (WINAPI *_RDPDeCompress_GetContextSize)(DWORD ComprType);

typedef
void (WINAPI *_RDPCompress_InitRecvContext)(void *ctx, SIZE_T ctx_len, DWORD ComprType, void *workspace);

typedef
BOOL (WINAPI *_RDPDecompress)(
    PVOID  inbuf, 
    DWORD  inlen, 
    DWORD  start, 
    PVOID* outbuf, 
    PDWORD outlen, 
    PVOID  RecvContext, 
    DWORD  ComprType
    );

typedef struct _RDP_pack_ctx {
    DWORD                         type;  // compression algorithm
    PCHAR                         name;
    DWORD                         blk;
    
    _RDPCompress_GetContextSize   RDPCompress_GetContextSize;
    _RDPCompress_InitSendContext  RDPCompress_InitSendContext;
    _RDPCompress                  RDPCompress;   // doesn't work with XCRUSH
    _RDPCompressEx                RDPCompressEx;
    
    _RDPDeCompress_GetContextSize RDPDeCompress_GetContextSize;
    _RDPCompress_InitRecvContext  RDPCompress_InitRecvContext;
    _RDPDecompress                RDPDecompress;
    
    std::vector<BYTE> inbuf, outbuf;
} RDP_pack_ctx;

#define RDP_COMPR_BLK_LEN (8192 * 4)

#pragma pack(push, 1)
typedef struct _RDP_pack_blk {
    WORD flags;
    WORD len;
    BYTE data[RDP_COMPR_BLK_LEN + 12];
} RDP_pack_blk;
#pragma pack(pop)

bool
init_pack_ctx(RDP_pack_ctx *c) {
    printf("%zd\n", sizeof(RDP_pack_blk) - (RDP_COMPR_BLK_LEN + 12));
    
    HMODULE rdpbase = LoadLibrary("rdpbase.dll");
    
    if (!rdpbase) return false;
    
    c->RDPCompress_GetContextSize = (_RDPCompress_GetContextSize)GetProcAddress(rdpbase, "RDPCompress_GetContextSize"); 
    c->RDPCompress_InitSendContext = (_RDPCompress_InitSendContext)GetProcAddress(rdpbase, "RDPCompress_InitSendContext");
    c->RDPCompress = (_RDPCompress)GetProcAddress(rdpbase, "RDPCompress");
    c->RDPCompressEx = (_RDPCompressEx)GetProcAddress(rdpbase, "RDPCompressEx");

    c->RDPDeCompress_GetContextSize = (_RDPDeCompress_GetContextSize)GetProcAddress(rdpbase, "RDPDeCompress_GetContextSize");
    c->RDPCompress_InitRecvContext = (_RDPCompress_InitRecvContext)GetProcAddress(rdpbase, "RDPCompress_InitRecvContext");
    c->RDPDecompress = (_RDPDecompress)GetProcAddress(rdpbase, "RDPDecompress");
    
    return c->RDPCompress_GetContextSize != NULL &&
           c->RDPCompress_InitSendContext != NULL &&
           c->RDPCompress != NULL &&
           c->RDPCompressEx != NULL &&
           c->RDPDeCompress_GetContextSize != NULL &&
           c->RDPCompress_InitRecvContext != NULL &&
           c->RDPDecompress != NULL;
}

std::vector<BYTE>
ReadFileData(std::string path) {
    std::ifstream instream(path, std::ios::in | std::ios::binary);
    std::vector<BYTE> data((std::istreambuf_iterator<char>(instream)), std::istreambuf_iterator<char>());
    return data;
}

bool
WriteFileData(std::string path, std::vector<BYTE> data) {
    std::ofstream outstream(path, std::ios::out | std::ios::binary);
    if (!outstream) return false;
    std::copy(data.begin(), data.end(), std::ostreambuf_iterator<char>(outstream));
    return outstream.good();
}

//
// Use RDP algorithms to compress file. 
//
BOOL
rdp_encode(RDP_pack_ctx *c) {
    void *SendCtx = NULL;
    BOOL result = TRUE;
    
    do {
        DWORD ctx_len = c->RDPCompress_GetContextSize(c->type);
        if (!ctx_len) { result = FALSE; break; }
        
        SendCtx = malloc(ctx_len);
        if (!SendCtx) { result = FALSE; break; }
    
        c->RDPCompress_InitSendContext(SendCtx, ctx_len, c->type);
        
        PBYTE inbuf = (PBYTE)c->inbuf.data();
        DWORD inlen = c->inbuf.size();
        DWORD cnt = 0;
        
        while (inlen) {
            RDP_pack_blk in={0}, out={0};
            
            // 12 bytes are needed for header/flags
            in.len = inlen > c->blk - 12 ? c->blk - 12: inlen;
            
            DWORD outlen = c->blk;
            memcpy(in.data, inbuf, in.len);
            
            out.flags = c->RDPCompressEx(c->type, in.data, in.len, out.data, &outlen, SendCtx);
            
            if (!out.flags) {
                printf("RDPCompressEx failed for block %ld, %ld\n", cnt, in.len);
                result = FALSE;
                break;
            }
            
            out.len = outlen & 0xFFFF;
            
            //printf("flags=%02X Block : %ld (%ld -> %ld) bytes...\n", out.flags, cnt, in.len, out.len);
            c->outbuf.insert(c->outbuf.end(), (PBYTE)&out, (PBYTE)&out + out.len + sizeof(DWORD));
            
            inlen -= in.len;
            inbuf += in.len;
            cnt++;
        }
    } while (0);
    
    if (SendCtx) free(SendCtx);
    return result;
}

//
//
//
BOOL
rdp_decode(RDP_pack_ctx *c) {
    void *RecvContext = NULL;
    BOOL result = FALSE;
    
    do {
        DWORD ctx_len = c->RDPDeCompress_GetContextSize(c->type);
        RecvContext = malloc(ctx_len);
        if (!RecvContext) break;
        
        c->RDPCompress_InitRecvContext(RecvContext, ctx_len, c->type, NULL);
        
        PBYTE inbuf = (PBYTE)c->inbuf.data();
        DWORD inlen = c->inbuf.size();
        DWORD cnt = 0;
        
        while (inlen) {
            RDP_pack_blk *in = (RDP_pack_blk*)inbuf;
            
            PBYTE outbuf = NULL;
            DWORD outlen = 0;
            
            result = c->RDPDecompress(
                                in->data, 
                                in->len, 
                                in->flags & PACKET_AT_FRONT, 
                                (PVOID*)&outbuf, 
                                &outlen, 
                                RecvContext, 
                                c->type);
                
            if (!result) {
                printf("RDPDecompress() failed for block %ld, %ld -> %ld\n", cnt, in->len, outlen);
                break;
            }
            
            //printf("Saving %ld bytes for block %ld...\n", outlen, cnt);
            c->outbuf.insert(c->outbuf.end(), outbuf, outbuf + outlen);

            inlen -= in->len + sizeof(DWORD);
            inbuf += in->len + sizeof(DWORD);
            cnt++;
        }
    } while (0);
    
    if (RecvContext) free(RecvContext);
    return result;
}

int
main(int argc, char *argv[]) {
    if ((argc != 4) || ((argv[1][0] != 'e') && (argv[1][0] != 'd'))) {
        printf("\nusage: rdp_pack [e/d]x <infile> <outfile>\n\n");
        printf(" x denotes algorithm:\n\n");
        printf("       0 - MPPC-8K\n");
        printf("       1 - MPPC-64K\n");
        printf("       2 - NCRUSH\n");
        printf("       3 - XCRUSH\n\n");
        printf("example compressing infile.txt with XCRUSH: rdp_pack e3 infile.txt outfile.bin\n");
        return 0;
    }
    
    bool result, encode = (argv[1][0] == 'e');
    int alg = argv[1][1] - '0';
    
    const char* infile = argv[2];
    const char* outfile = argv[3];
    
    RDP_pack_ctx c;
    
    if (!init_pack_ctx(&c)) {
        printf("Unable to initialise RDP API.\n");
        return 0;
    }
    
    switch(alg) {
        case 0:
            c.type = PACKET_COMPR_TYPE_8K;
            c.name = "MPPC-8K";
            c.blk  = 4096;
            break;
        case 1:
            c.type = PACKET_COMPR_TYPE_64K;
            c.name = "MPPC-64K";
            c.blk  = 8192;
            break;
        case 2:
            c.type = PACKET_COMPR_TYPE_RDP6;
            c.name = "NCRUSH";
            c.blk  = 8192;
            break;
        case 3:
            c.type = PACKET_COMPR_TYPE_RDP61;
            c.name = "XCRUSH";
            c.blk  = 8192;
            break;
        default:
            printf("Invalid compression algorithm specified.\n");
            return 0;
    }
    
    c.inbuf = ReadFileData(infile);
    
    if (encode) {
        printf("Compressing %zd bytes in %s with %s\n", 
            c.inbuf.size(), 
            infile,
            c.name
            );
        result = rdp_encode(&c);
    } else {
        printf("Decompressing %zd bytes in %s with %s\n", 
            c.inbuf.size(), 
            infile,
            c.name
            );
        result = rdp_decode(&c);
    }
    
    if (result) {
        printf("Saving %zd bytes to %s.\n", c.outbuf.size(), outfile);
        WriteFileData(outfile, c.outbuf);
    }
    printf("Status : %s\n", result ? "OK" : "FAILED");
    return 0;
}