AES Encrypt a file using a random generated 256-bit key and encrypt the AES key using RSA public key, then decrypt AES key using private RSA key and decrypt file.

rsaEncDecOpenSSL.c

#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <openssl/evp.h>
#include <openssl/aes.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/rsa.h>

void encrypt(EVP_PKEY **pub_key, FILE *ifp, FILE *ofp, unsigned char **key, int *ekl, unsigned char *ivec) {
    const unsigned BUFSIZE=4096;
    unsigned char *read_buf = malloc(BUFSIZE);
    unsigned char *cipher_buf;
    unsigned blocksize;
    int out_len;
    EVP_CIPHER_CTX ctx;

    /* Encryption */

    EVP_SealInit(&ctx, EVP_aes_256_cbc(), key,ekl, ivec, pub_key,1);

    blocksize = EVP_CIPHER_CTX_block_size(&ctx);
    cipher_buf = malloc(BUFSIZE + blocksize);

    while (1) {

        // Read in data in blocks until EOF. Update the ciphering with each read.

        int numRead = fread(read_buf, sizeof(unsigned char), BUFSIZE, ifp);
        EVP_SealUpdate(&ctx, cipher_buf, &out_len, read_buf, numRead);
        fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);
        if (numRead < BUFSIZE) { // EOF
            break;
        }
    }

    // Now cipher the final block and write it out.

    EVP_SealFinal(&ctx, cipher_buf, &out_len);
    fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);

    // Free memory
    free(cipher_buf);
    free(read_buf);

}

void decrypt(EVP_PKEY *priv_key, FILE *ifp, FILE *ofp, unsigned char *key, int ekl, unsigned char *ivec) {
    /* Decryption */
    const unsigned BUFSIZE=4096;
    unsigned char *read_buf = malloc(BUFSIZE);
    unsigned char *cipher_buf;
    unsigned blocksize;
    int out_len;
    EVP_CIPHER_CTX ctx;

    EVP_OpenInit(&ctx, EVP_aes_256_cbc(), key,ekl, ivec, priv_key);

    blocksize = EVP_CIPHER_CTX_block_size(&ctx);
    cipher_buf = malloc(BUFSIZE + blocksize);

    while (1) {
        // Read in data in blocks until EOF

        int numRead = fread(read_buf, sizeof(unsigned char), BUFSIZE, ifp);
        EVP_OpenUpdate(&ctx, cipher_buf, &out_len, read_buf, numRead);
        fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);
        if (numRead < BUFSIZE) { // EOF
            break;
        }
    }

    // Now cipher the final block and write it out.

    EVP_OpenFinal(&ctx, cipher_buf, &out_len);
    fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);

    // Free memory
    free(cipher_buf);
    free(read_buf);
}

int main(int argc, char *argv[]) {
    if (argc < 5) {
        printf("Not enough params");
        return 1;
    }

    unsigned char **key = (unsigned char **)malloc(sizeof(unsigned char *) * 1);
    unsigned char iv[16];

    /* Get RSA public key */
    FILE *rsa_public = fopen(argv[4],"rb");
    RSA* pubkey = RSA_new();
    pubkey = PEM_read_RSA_PUBKEY(rsa_public,&pubkey,NULL,NULL);
    EVP_PKEY **pubk = (EVP_PKEY **)malloc(sizeof(EVP_PKEY *) * 1);
    pubk[0] = EVP_PKEY_new();
    EVP_PKEY_set1_RSA(pubk[0], pubkey);

    FILE *rsa_private = fopen(argv[5],"rb");
    RSA* privkey = RSA_new();
    privkey = PEM_read_RSAPrivateKey(rsa_private,&privkey,NULL,NULL);
    EVP_PKEY **privk = (EVP_PKEY **)malloc(sizeof(EVP_PKEY *) * 1);
    privk[0] = EVP_PKEY_new();
    EVP_PKEY_assign_RSA(privk[0], privkey);


    key[0] = (unsigned char *)malloc(EVP_PKEY_size(pubk[0]));
    int *ekl = (int *)malloc(sizeof(int));

    FILE *fIN, *fOUT;

    // First encrypt the file

    fIN = fopen(argv[1], "rb"); //File to be encrypted; plain text
    fOUT = fopen(argv[2], "wb"); //File to be written; cipher text

    encrypt(pubk, fIN, fOUT, key,ekl, iv);
    printf("%s\n",key[0]);

    fclose(fIN);
    fclose(fOUT);


    //Decrypt file now

    fIN = fopen(argv[2], "rb"); //File to be read; cipher text
    fOUT = fopen(argv[3], "wb"); //File to be written; clear text

    decrypt(privk[0], fIN, fOUT, key[0], ekl[0],iv);

    return 0;
}