using the openssl library and some helper function to encrypt an array of bytes with many public keys and decrypt with a single private key in an smime envelope

openssl_smime_c_abstraction.c

// So the problem to solve is that I needed a function
// that encrypts an array of ubytes with multiple public
// keys. The resulting, encrypted, array of bytes, when saved
// to a file, should be decrypted-able by any private key
// matching any of the used public key.
// This should be possible with the openssl cli.
// smime is used as bundle everything together, properly
// not the best approach, but hey it works.
// The asymmetrically keys are used to encrypt a symmetry
// AES256 key that is in turn used to encrypt the array of
// bytes.
//
// The following three functions is where the work happens
//
// Buffer smime_main_decryption(Buffer inFile, char* certFile);
// Buffer smime_main_encryption(Buffer buf, char** certs, size_t numCerts);
// Buffer smime_main_encryption_with_certs(Buffer buf, X509** certs, size_t numCerts);
//
// # Example:
//
// ## Build the test key pairs
// openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -subj "/C=US/ST=*/L=*/O=*/OU=*/CN=Alice/" -keyout alice.key -out alice.pub
// openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -subj "/C=US/ST=*/L=*/O=*/OU=*/CN=Bob/" -keyout bob.key -out bob.pub
// openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -subj "/C=US/ST=*/L=*/O=*/OU=*/CN=Frank/" -keyout frank.key -out frank.pub
//
// ## Create the secret message
// echo 'All our secretz are belong to us' > secrets.txt
//
// ## Compile the program
// gcc -Wall -ggdb smime.c -o smime -I /usr/local/ssl/include -L /usr/local/ssl/lib -lssl -lcrypto
//
// ## Encrypt secrets.txt into secrets.txt.enc
// ./smime enc
//
// you should see a new file secrets.txt.enc
//
// ## Decrypt secrets.txt.enc with openssl
//
// openssl smime -decrypt -in secrets.txt.enc -inform PEM -inkey bob.key
//
// ## The compiled program can also decrypt
//
// ./smime 0
//
// this should use the first private key listed in function main to decrypt secrets.txt.enc
// the decrypted text is saved in file secrets_new.txt
//
// To see the usage have a look at the main function

#include <stdio.h>
#include <string.h>
#include <openssl/crypto.h>
#include <openssl/pem.h>
#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/x509_vfy.h>
#include <openssl/x509v3.h>
#include <openssl/pkcs12.h>

#define SMIME_OP 0x10
#define SMIME_SIGNERS 0x40
#define SMIME_ENCRYPT (1 | SMIME_OP)

# define B_FORMAT_TEXT 0x8000
# define FORMAT_ASN1 4					  /* ASN.1/DER */
# define FORMAT_PEM (5 | B_FORMAT_TEXT)
# define FORMAT_PKCS12 6
# define FORMAT_SMIME (7 | B_FORMAT_TEXT)
# define FORMAT_ENGINE 8					  /* Not really a file format */
# define FORMAT_MSBLOB 11					 /* MS Key blob format */
# define FORMAT_PVK	12					 /* MS PVK file format */

BIO *bio_err;

EVP_PKEY *load_key(const char *file, int format, int maybe_stdin
	, const char *pass, ENGINE *e, const char *key_descrip);

typedef struct BufferImpl {
	unsigned char* source;
	long len;
} Buffer;

static Buffer smime_main_encryptionImpl(Buffer buf, STACK_OF(X509) *encerts);
Buffer smime_main_encryption_with_certs(Buffer buf, X509** certs, size_t numCerts);

static int istext(int format) {
	return (format & B_FORMAT_TEXT) == B_FORMAT_TEXT;
}

static const char *modeverb(char mode) {
	switch (mode) {
	case 'a':
		return "appending";
	case 'r':
		return "reading";
	case 'w':
		return "writing";
	}
	return "(doing something)";
}

static const char *modestr(char mode, int format) {
	OPENSSL_assert(mode == 'a' || mode == 'r' || mode == 'w');

	switch (mode) {
	case 'a':
		return istext(format) ? "a" : "ab";
	case 'r':
		return istext(format) ? "r" : "rb";
	case 'w':
		return istext(format) ? "w" : "wb";
	}
	/* The assert above should make sure we never reach this point */
	return NULL;
}

static int pass_cb(char*, int , int , void*) {
	return -1;
}

static int load_pkcs12(BIO *in, const char *desc
	, pem_password_cb *, void *, EVP_PKEY **pkey, X509 **cert
	, STACK_OF(X509) **ca)
{
	const char *pass;
	PKCS12 *p12 = d2i_PKCS12_bio(in, NULL);
	int ret = -1;
	if(p12 == NULL) {
		BIO_printf(bio_err, "Error loading PKCS12 file for %s\n", desc);
		goto die;
	}
	/* See if an empty password will do */
	if(PKCS12_verify_mac(p12, "", 0) || PKCS12_verify_mac(p12, NULL, 0)) {
		pass = "";
	}
	ret = PKCS12_parse(p12, pass, pkey, cert, ca);
 die:
	PKCS12_free(p12);
	return ret;
}

typedef struct pw_cb_data {
	const void *password;
	const char *prompt_info;
} PW_CB_DATA;

void freeBuffer(Buffer buf) {
	if(buf.len > 0 && buf.source != NULL) {
		free(buf.source);
	}
}

Buffer readFile(const char* file) {
	Buffer ret;
	ret.len = -1;
	FILE *fp = fopen(file, "r");
	if(fp != NULL) {
		/* Go to the end of the file. */
		if(fseek(fp, 0L, SEEK_END) == 0) {
			/* Get the size of the file. */
			ret.len = ftell(fp);
			if(ret.len == -1) { 
				ret.len = -1;
				return ret;
			}

			/* Allocate our buffer to that size. */
			ret.source = malloc(sizeof(char) * ((unsigned long)ret.len + 1));

			/* Go back to the start of the file. */
			if(fseek(fp, 0L, SEEK_SET) != 0) { 
				ret.len = -1;
				return ret;
			}

			/* Read the entire file into memory. */
			size_t newLen = fread(ret.source, sizeof(char), (unsigned long)ret.len, fp);
			if(ferror( fp ) != 0) {
				fputs("Error reading file", stderr);
			} else {
				ret.source[newLen++] = '\0'; /* Just to be safe. */
			}
		}
	}
	return ret;
}

EVP_PKEY *load_key(const char *file, int format, int maybe_stdin
	, const char *pass, ENGINE *e, const char *key_descrip)
{
	BIO *key = NULL;
	EVP_PKEY *pkey = NULL;
	PW_CB_DATA cb_data;

	cb_data.password = pass;
	cb_data.prompt_info = file;

	if(file == NULL && (!maybe_stdin || format == FORMAT_ENGINE)) {
		BIO_printf(bio_err, "no keyfile specified\n");
		goto end;
	}
	if(format == FORMAT_ENGINE) {
		if(e == NULL) {
			BIO_printf(bio_err, "no engine specified\n");
		} else {
			//if(ENGINE_init(e)) {
			FILE *f = fopen("key.pem", "rb");
			//pkey = //ENGINE_load_private_key(e, file, ui_method, &cb_data);
			PEM_read_PrivateKey(f, &pkey, NULL, NULL);
			fclose(f);
			//}
			if(pkey == NULL) {
				BIO_printf(bio_err, "cannot load %s from engine\n", key_descrip);
				ERR_print_errors(bio_err);
			}
		}
		goto end;
	}
	key = BIO_new_file(file, modestr('r', format));
	if(key == NULL) {
		goto end;
	}
	if(format == FORMAT_ASN1) {
		pkey = d2i_PrivateKey_bio(key, NULL);
	} else if(format == FORMAT_PEM) {
		pkey = PEM_read_bio_PrivateKey(key, NULL,
									   pass_cb,
									   &cb_data);
	} else if(format == FORMAT_PKCS12) {
		if(!load_pkcs12(key, key_descrip,
						 pass_cb, &cb_data,
						 &pkey, NULL, NULL))
		{
			goto end;
		}
#if !defined(OPENSSL_NO_RSA) && !defined(OPENSSL_NO_DSA) && !defined (OPENSSL_NO_RC4)
	} else if(format == FORMAT_MSBLOB) {
		pkey = b2i_PrivateKey_bio(key);
	} else if(format == FORMAT_PVK) {
		pkey = b2i_PVK_bio(key, pass_cb, &cb_data);
#endif
	} else {
		BIO_printf(bio_err, "bad input format specified for key file\n");
		goto end;
	}
 end:
	BIO_free(key);
	if(pkey == NULL) {
		BIO_printf(bio_err, "unable to load %s\n", key_descrip);
		ERR_print_errors(bio_err);
	}
	return pkey;
}

static BIO *bio_open_default_(const char *filename, char mode, int format
	, int quiet)
{
	BIO *ret = BIO_new_file(filename, modestr(mode, format));
	if(quiet) {
		ERR_clear_error();
		return ret;
	}
	if(ret != NULL)
		return ret;
	BIO_printf(bio_err,
			   "Can't open %s for %s, %s\n",
			   filename, modeverb(mode), strerror(errno));
	
	ERR_print_errors(bio_err);
	return NULL;
}

static BIO *bio_open_default(const char *filename, char mode, int format) {
	return bio_open_default_(filename, mode, format, 0);
}

static X509 *load_certImpl(const char *file, int format
	, const char *cert_descrip) 
{
	X509 *x = NULL;
	BIO *cert = bio_open_default(file, 'r', format);
	if(cert == NULL)
		goto end;

	if(format == FORMAT_ASN1) {
		x = d2i_X509_bio(cert, NULL);
	} else if(format == FORMAT_PEM) {
		x = PEM_read_bio_X509_AUX(cert, NULL, 0, NULL);
	} else if(format == FORMAT_PKCS12) {
		if(!load_pkcs12(cert, cert_descrip, NULL, NULL, NULL, &x, NULL))
			goto end;
	} else {
		BIO_printf(bio_err, "bad input format specified for %s\n", cert_descrip);
		goto end;
	}
 end:
	if(x == NULL) {
		BIO_printf(bio_err, "unable to load certificate\n");
		ERR_print_errors(bio_err);
	}
	BIO_free(cert);
	return x;
}

X509 *load_cert(const char *file) {
	return load_certImpl(file, FORMAT_PEM, "recipient certificate file");
}

Buffer smime_main_encryption(Buffer buf
	, char** certs, size_t numCerts)
{
	// The public keys/certs are the leftover arguments 
	// now we pass them in as certs
	Buffer ret;
	ret.len = -1;
	X509 **certsLoaded = (X509**)malloc(sizeof(X509*) * numCerts);
	for(size_t i = 0; i < numCerts; ++i) {
		certsLoaded[i] = load_cert(certs[i]);
		if(certsLoaded == NULL) {
			goto end;
		}
	}

	ret = smime_main_encryption_with_certs(buf, certsLoaded, numCerts);
end:
	free(certsLoaded);
	return ret;
}

Buffer smime_main_encryption_with_certs(Buffer buf, X509** certs
	, size_t numCerts) 
{
	STACK_OF(X509) *encerts = sk_X509_new_null();

	Buffer ret;
	ret.len = -1;

	if(encerts == NULL) {
		goto end;
	}
	for(size_t i = 0; i < numCerts; ++i) {
		sk_X509_push(encerts, certs[i]);
	}

	ret = smime_main_encryptionImpl(buf, encerts);
	sk_X509_pop_free(encerts, X509_free);
end:
	return ret;
}

Buffer smime_main_encryptionImpl(Buffer buf, STACK_OF(X509) *encerts) {
	BIO *in = NULL;
	BIO *out = NULL;
	BIO *indata = NULL;
	EVP_PKEY *key = NULL;
	PKCS7 *p7 = NULL;
	STACK_OF(OPENSSL_STRING) *sksigners = NULL;
	STACK_OF(OPENSSL_STRING) *skkeys = NULL;
	STACK_OF(X509) *other = NULL;
	X509 *recip = NULL;
	X509 *signer = NULL;
	X509_STORE *store = NULL;
	X509_VERIFY_PARAM *vpm = NULL;
	const EVP_CIPHER *cipher = EVP_aes_256_cbc();
	char *keyfile = NULL;
	char *passin = NULL;
	int flags = PKCS7_DETACHED;
	int keyform = FORMAT_PEM;
	int rv = 0;
	ENGINE *e = NULL;

	int operation = SMIME_ENCRYPT;

	Buffer ret;
	ret.len = 0;

	if((vpm = X509_VERIFY_PARAM_new()) == NULL) {
		ret.len = 1;
		return ret;
	}

	if(!(operation & SMIME_SIGNERS) && (skkeys != NULL || sksigners != NULL)) {
		BIO_puts(bio_err, "Multiple signers or keys not allowed\n");
		printf("%d\n", __LINE__);
		goto end;
	}

	ret.len = 2;

	if(cipher == NULL) {
		BIO_printf(bio_err, "No cipher selected\n");
		goto end;
	}

	keyfile = NULL;

	if(keyfile != NULL) {
		key = load_key(keyfile, keyform, 0, passin, e, "signing key file");
		if(key == NULL) {
			goto end;
		}
	}

	//in = bio_open_default(infile, 'r', informat);
	in = BIO_new(BIO_s_mem());
	BIO_write(in, buf.source, (int)buf.len);
	if(in == NULL) {
		goto end;
	}

	out = BIO_new(BIO_s_mem());
	if(out == NULL) {
		goto end;
	}

	ret.len = 3;

	p7 = PKCS7_encrypt(encerts, in, cipher, flags);

	if(p7 == NULL) {
		BIO_printf(bio_err, "Error creating PKCS#7 structure\n");
		goto end;
	}

	ret.len = 4;
	rv = PEM_write_bio_PKCS7_stream(out, p7, in, flags);
	if(rv == 0) {
		BIO_printf(bio_err, "Error writing output\n");
		ret.len = 3;
		goto end;
	}

	char* data;
	ret.len = BIO_get_mem_data(out, &data);
	ret.source = (unsigned char*)malloc(ret.len);
	BIO_read(out, ret.source, ret.len);

 end:
	if(ret.len)
		ERR_print_errors(bio_err);
	sk_X509_pop_free(other, X509_free);
	X509_VERIFY_PARAM_free(vpm);
	sk_OPENSSL_STRING_free(sksigners);
	sk_OPENSSL_STRING_free(skkeys);
	X509_STORE_free(store);
	X509_free(recip);
	X509_free(signer);
	EVP_PKEY_free(key);
	PKCS7_free(p7);
	OPENSSL_free(e);
	BIO_free(in);
	BIO_free(out);
	BIO_free(indata);
	OPENSSL_free(passin);
	return ret;
}

Buffer smime_main_decryption(Buffer inFile
		, char* certFile) 
{
	BIO *in = NULL;
	BIO *out = NULL;
	BIO *indata = NULL;
	EVP_PKEY *key = NULL;
	PKCS7 *p7 = NULL;
	STACK_OF(OPENSSL_STRING) *sksigners = NULL;
	STACK_OF(OPENSSL_STRING) *skkeys = NULL;
	STACK_OF(X509) *encerts = NULL;
	STACK_OF(X509) *other = NULL;
	X509 *cert = NULL;
	X509 *recip = NULL;
	X509 *signer = NULL;
	X509_STORE *store = NULL;
	X509_VERIFY_PARAM *vpm = NULL;
	char *keyfile = certFile;
	char *contfile = NULL;
	char *recipfile = NULL;
	char *passin = NULL;
	int flags = PKCS7_DETACHED;
	int informat = FORMAT_PEM;
	int keyform = FORMAT_PEM;
	ENGINE *e = NULL;

	Buffer ret;
	ret.len = 0;

	if((vpm = X509_VERIFY_PARAM_new()) == NULL) {
		ret.len = 0;
		return ret;
	}

	if(recipfile == NULL && keyfile == NULL) {
		BIO_printf(bio_err,
				   "No recipient certificate or key specified\n");
		printf("%d\n", __LINE__);
		goto end;
	}

	ret.len = 2;

	if(recipfile != NULL) {
		printf("%d\n", __LINE__);
		if((recip = load_cert(recipfile)) == NULL) {
			ERR_print_errors(bio_err);
			goto end;
		}
	}

	if(keyfile == NULL) {
		printf("%d\n", __LINE__);
		keyfile = recipfile;
	}

	if(keyfile != NULL) {
		key = load_key(keyfile, keyform, 0, passin, e, "signing key file");
		if(key == NULL) {
			printf("%d\n", __LINE__);
			goto end;
		}
	}

	in = BIO_new(BIO_s_mem());
	BIO_write(in, inFile.source, (int)inFile.len);
	if(in == NULL) {
		printf("%d\n", __LINE__);
		goto end;
	}

	if(informat == FORMAT_SMIME) {
		p7 = SMIME_read_PKCS7(in, &indata);
	} else if(informat == FORMAT_PEM) {
		p7 = PEM_read_bio_PKCS7(in, NULL, NULL, NULL);
	}

	if(p7 == NULL) {
		BIO_printf(bio_err, "Error reading S/MIME message\n");
		printf("%d\n", __LINE__);
		goto end;
	}
	if(contfile != NULL) {
		BIO_free(indata);
		if((indata = BIO_new_file(contfile, "rb")) == NULL) {
			BIO_printf(bio_err, "Can't read content file %s\n", contfile);
			printf("%d\n", __LINE__);
			goto end;
		}
	}

	out = BIO_new(BIO_s_mem());
	if(out == NULL) {
		printf("%d\n", __LINE__);
		goto end;
	}

	ret.len = 3;

	if(p7 == NULL) {
		printf("%d\n", __LINE__);
		BIO_printf(bio_err, "Error creating PKCS#7 structure\n");
		goto end;
	}

	ret.len = 4;
	if(!PKCS7_decrypt(p7, key, recip, out, flags)) {
		printf("%d\n", __LINE__);
		BIO_printf(bio_err, "Error decrypting PKCS#7 structure\n");
		goto end;
	}
	ret.len = 0;

	char* data;
	ret.len = BIO_get_mem_data(out, &data);
	ret.source = (unsigned char*)malloc(ret.len);
	BIO_read(out, ret.source, ret.len);

 end:
	if(ret.len) {
		ERR_print_errors(bio_err);
	}
	sk_X509_pop_free(encerts, X509_free);
	sk_X509_pop_free(other, X509_free);
	X509_VERIFY_PARAM_free(vpm);
	sk_OPENSSL_STRING_free(sksigners);
	sk_OPENSSL_STRING_free(skkeys);
	X509_STORE_free(store);
	X509_free(cert);
	X509_free(recip);
	X509_free(signer);
	EVP_PKEY_free(key);
	PKCS7_free(p7);
	OPENSSL_free(e);
	BIO_free(in);
	BIO_free(indata);
	BIO_free_all(out);
	OPENSSL_free(passin);
	return ret;
}

int main(int argc, char** argv) {
	if(strcmp(argv[1], "enc") == 0) {
		char* certs[3] = 
			{ "../apps/alice.pub"
			, "../apps/bob.pub"
			, "../apps/frank.pub"
			};
		Buffer buf = readFile("secrets.txt");
		Buffer ret = smime_main_encryption(buf, certs, 3);
		FILE* oFile = fopen("secrets.txt.enc", "w");
		fwrite(ret.source, 1, ret.len, oFile);
		fclose(oFile);
		freeBuffer(ret);
		freeBuffer(buf);
		return 0;
	} else {
		char* certs[3] = 
			{ "../apps/alice.key"
			, "../apps/bob.key"
			, "../apps/frank.key"
			};
		char* ptr;
		long id = strtol(argv[1], &ptr, 10);
		Buffer input = readFile("secrets.txt.enc");
		Buffer ret = smime_main_decryption(input, certs[id]);
		FILE* oFile = fopen("secrets_new.txt", "w");
		fwrite(ret.source, 1, ret.len, oFile);
		fclose(oFile);
		freeBuffer(ret);
		return 0;
	}
	ERR_print_errors(bio_err);
	return 0;
}