AES 128 - encrypt/decrypt using Windows Crypto API

aes_crypt.cpp

#include <Windows.h>
#include <wincrypt.h>
#include <stdio.h>
#pragma comment(lib, "advapi32.lib")

#define AES_KEY_SIZE 16
#define IN_CHUNK_SIZE (AES_KEY_SIZE * 10) // a buffer must be a multiple of the key size
#define OUT_CHUNK_SIZE (IN_CHUNK_SIZE * 2) // an output buffer (for encryption) must be twice as big

//params: <input file> <output file> <is decrypt mode> <key>
int wmain(int argc, wchar_t *argv[])
{
    if (argc < 4) {
        printf("params: <input file> <output file> <is decrypt mode> [*key]\n");
        system("pause");
        return 0;
    }
    wchar_t *filename = argv[1];
    wchar_t *filename2 = argv[2];

    wchar_t default_key[] = L"3igcZhRdWq96m3GUmTAiv9";
    wchar_t *key_str = default_key;

    BOOL isDecrypt = FALSE;
    if (argv[3][0] > '0') {
        printf("Decrypt mode\n");
        isDecrypt = TRUE;
    }
    if (argc >= 5) {
        key_str = argv[4];
    }
    const size_t len = lstrlenW(key_str);
    const size_t key_size = len * sizeof(key_str[0]); // size in bytes

    printf("Key: %S\n", key_str);
    printf("Key len: %#x\n", len);
    printf("Key size: %#x\n", key_size);
    printf("Input File: %S\n", filename);
    printf("Output File: %S\n", filename2);
    printf("----\n");

    HANDLE hInpFile = CreateFileW(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
    if (hInpFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open input file!\n");
        system("pause");
        return (-1);
    }
    HANDLE hOutFile = CreateFileW(filename2, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    if (hOutFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open output file!\n");
        system("pause");
        return (-1);
    }

    if (isDecrypt) {
        printf("DECRYPTING\n");
    }
    else {
        printf("ENCRYPTING\n");
    }

    DWORD dwStatus = 0;
    BOOL bResult = FALSE;
    wchar_t info[] = L"Microsoft Enhanced RSA and AES Cryptographic Provider";
    HCRYPTPROV hProv;
    if (!CryptAcquireContextW(&hProv, NULL, info, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        dwStatus = GetLastError();
        printf("CryptAcquireContext failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    HCRYPTHASH hHash;
    if (!CryptCreateHash(hProv, CALG_SHA_256, 0, 0, &hHash)) {
        dwStatus = GetLastError();
        printf("CryptCreateHash failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }

    if (!CryptHashData(hHash, (BYTE*)key_str, key_size, 0)) {
        DWORD err = GetLastError();
        printf("CryptHashData Failed : %#x\n", err);
        system("pause");
        return (-1);
    }
    printf("[+] CryptHashData Success\n");

    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hProv, CALG_AES_128, hHash, 0, &hKey)) {
        dwStatus = GetLastError();
        printf("CryptDeriveKey failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    printf("[+] CryptDeriveKey Success\n");

    const size_t chunk_size = isDecrypt ? IN_CHUNK_SIZE: OUT_CHUNK_SIZE;
    BYTE *chunk = new BYTE[chunk_size];
    DWORD out_len = 0;

    BOOL isFinal = FALSE;
    DWORD readTotalSize = 0;

    DWORD inputSize = GetFileSize(hInpFile, NULL);

    while (bResult = ReadFile(hInpFile, chunk, IN_CHUNK_SIZE, &out_len, NULL)) {
        if (0 == out_len) {
            break;
        }
        readTotalSize += out_len;
        if (readTotalSize >= inputSize) {
            isFinal = TRUE;
            printf("Final chunk set, len: %d = %x\n", out_len, out_len);
        }

        if (isDecrypt) {
            if (!CryptDecrypt(hKey, NULL, isFinal, 0, chunk, &out_len)) {
                printf("[-] CryptDecrypt failed: %x\n", GetLastError());
                break;
            }
        }
        else {
            if (!CryptEncrypt(hKey, NULL, isFinal, 0, chunk, &out_len, chunk_size)) {
                printf("[-] CryptEncrypt failed: %x\n", GetLastError());
                break;
            }
        }
        DWORD written = 0;
        if (!WriteFile(hOutFile, chunk, out_len, &written, NULL)) {
            printf("writing failed!\n");
            break;
        }
        memset(chunk, 0, chunk_size);
    }
    
    delete[]chunk; chunk = NULL;

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

    CloseHandle(hInpFile);
    CloseHandle(hOutFile);
    printf("Finished. Processed %#x bytes.\n", readTotalSize);
    return 0;
}

What should I put at "is decrypt mode" param?

"is decrypt mode" is a flag that switches between decryption and encryption. if it is set to 0, the supplied file will be encrypted, otherwise it will be decrypted.

Wrong order for releasing
right order:

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

ref: https://docs.microsoft.com/en-us/windows/win32/seccrypto/example-c-program-encrypting-a-file

Very nice source and example, but I noticed a flaw - The user-supplied key is treated as a wchar_t, and its length is calculated appropriately by lstrlenW, but CryptHashData treats its argument as a byte array and treats the len parameter as the number of bytes (https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-crypthashdata), thus hashes only half of the key.
For example, if the user typed abcd, the actual data in memory is a\x00b\x00c\x00d\x00 with length 4 from lstrlenW, but CryptHashData gets passed 4 as the number of bytes, and will only hash a\x00b\x00.
This weakens the security of the encryption, as it halves the number of bits needed to bruteforce the user-supplied key (exponential factor).

@YonPog - you are right, thank you for noticing it! fixed

It works fine but it didn't work with RSA key pair. Any ideas?

maybe it will be useful to someone: an example of decryption in python:

import hashlib
from Crypto.Cipher import AES

def hash_md5(data):
    md5_obj = hashlib.md5()
    md5_obj.update(data)
    return md5_obj.digest()

def mscrypt_derive_key_md5(password):
    buf1 = bytearray([0x36] * 64)
    buf2 = bytearray([0x5C] * 64)

    hash_obj = hashlib.md5()
    hash_obj.update(password)
    hash_result = hash_obj.digest()

    buf1[:len(hash_result)] = [buf1[i] ^ hash_result[i] for i in range(len(hash_result))]
    buf2[:len(hash_result)] = [buf2[i] ^ hash_result[i] for i in range(len(hash_result))]

    hash_buf1 = hash_md5(buf1)
    hash_buf2 = hash_md5(buf2)
    
    return hash_buf1 + hash_buf2

input_file = r'D:\enc.dat'
output_file = r'D:\dec.dat'

with open(input_file, 'rb') as f:
    encrypted_data = f.read()

base_password = "3igcZhRdWq96m3GUmTAiv9"
my_derived_key = mscrypt_derive_key_md5(base_password.encode('utf-8'))

print("My Derived Key:\t\t", my_derived_key.hex())

cipher = AES.new(my_derived_key, AES.MODE_CBC, iv=b'\x00'*16)
decrypted_data = cipher.decrypt(encrypted_data)

with open(output_file, 'wb') as f:
    f.write(decrypted_data)

P.S in example used md5 hash.