SaneRSA.h

#import <Foundation/Foundation.h>

/**
 SaneRSA abstracts away the gory details of the RSA algorithm and exposes a clean, minimal interface
 to the outside world. The only data that needs to be fed into a SaneRSA instance is the desired RSA
 key size and the remote end's RSA public key (which obviously needs to have the same key size).
 */
@interface SaneRSA : NSObject

@property (nonatomic, readonly) NSData* publicKey;
@property (nonatomic, readonly) BOOL hasRemotePublicKey;

@property (nonatomic, assign) NSUInteger modulusLength;
@property (nonatomic, assign) NSUInteger exponentLength;
@property (nonatomic, assign) NSUInteger chunkLength;
@property (nonatomic, assign) NSUInteger encryptedChunkLength;

- (id)initWithKeySize:(NSUInteger)keySize;
- (void)importRemotePublicKey:(NSData*)keyData;
- (NSData*)encrypt:(NSData*)payload;
- (NSData*)decrypt:(NSData*)payload;

@end

SaneRSA.m

#import "SaneRSA.h"
#import <openssl/rsa.h>
#import <openssl/engine.h>

#define DEFAULT_KEY_SIZE 1024
#define OAEP_RESERVED_BYTES 42
#define EXPONENT_LENGTH 3
#define RSA_EXPONENT 65537

@interface SaneRSA ()

@property (nonatomic, assign) NSUInteger keySize;
@property (nonatomic, assign) RSA* remoteRSA;
@property (nonatomic, assign) RSA* localRSA;

@end

@implementation SaneRSA

/**
 This is the default initializer. It initializes all internal structures to be used with the RSA key of the given size.
 @param size The desired key size. Only 1024 is supported at this moment.
 @return An initialized SaneRSA object.
 */
- (id)initWithKeySize:(NSUInteger)size {
  self = [super init];
	if (self) {
		self.keySize = size;
		if (self.keySize != 1024 /* && self.keySize != ... add more conditions when more keysizes are supported */) {
			DLog(@"Unsupported key size, using %d", DEFAULT_KEY_SIZE);
			self.keySize = DEFAULT_KEY_SIZE;
		}
		
		// Init lengths
		self.modulusLength = self.keySize / 8;
		self.exponentLength = EXPONENT_LENGTH;
		self.chunkLength = self.keySize - OAEP_RESERVED_BYTES;
		self.encryptedChunkLength = self.modulusLength;
		
		// OpenSSL keeps an internal table of digest algorithms and ciphers.
		// It uses this table to lookup ciphers via functions such as EVP_get_cipher_byname().
		// OpenSSL_add_all_digests() adds all digest algorithms to the table.
		// OpenSSL_add_all_ciphers() adds all cipher algorithms to the table.
		// OpenSSL_add_all_algorithms() adds all algorithms to the table (digests and ciphers).
		// EVP_cleanup() removes all ciphers and digests from the table.
		//
		// A typical application will call OpenSSL_add_all_algorithms() initially and EVP_cleanup() before exiting.
		OpenSSL_add_all_algorithms();

		// ERR_load_crypto_strings() registers the error strings for all libcrypto functions.
		// SSL_load_error_strings() does the same, but also registers the libssl error strings.
		// ERR_free_strings() frees all previously loaded error strings.
		//
		// One of these functions should be called before generating textual error messages.
		// However, this is not required when memory usage is an issue.
		ERR_load_crypto_strings();
		
		// Generate a local RSA keypair
		self.localRSA = RSA_generate_key(self.keySize, RSA_EXPONENT, nil, nil);
		if (!self.localRSA) {
			DLog(@"ERROR: Could not generate a local RSA keypair!");
		}
	}
	
	return self;
}

/**
 Calls the default initializer with the default key size set (currently 1024).
 @return An initialized SaneRSA object.
 */
- (id)init {
	return [self initWithKeySize:DEFAULT_KEY_SIZE];
}

- (void)dealloc {
	// EVP_cleanup() removes all ciphers and digests from the table.
	EVP_cleanup();

	// ERR_free_strings() frees all previously loaded error strings.
	ERR_free_strings();
	
	if (self.remoteRSA) {
		RSA_free(self.remoteRSA);
		self.remoteRSA = nil;
	}
	
	if (self.localRSA) {
		RSA_free(self.localRSA);
		self.localRSA = nil;
	}
	
	[super dealloc];
}

#pragma mark - Public interface

/**
 Imports the remote end's public key, and prepares the current instance for encryption.
 @param keyData An encoded representation of the remote end's public key. Modulus comes first, followed by
   exponent. For the 1024-bit implementation, this equals 131 bytes (128 for the modulus, 3 for the exponent).
 */
- (void)importRemotePublicKey:(NSData*)keyData {
	uint8_t* modulus = malloc(self.modulusLength);
	[keyData getBytes:modulus length:self.modulusLength];
	
	uint8_t* exponent = malloc(self.exponentLength);
	[keyData getBytes:exponent range:NSMakeRange(self.modulusLength, self.exponentLength)];
	
	RSA* remoteRSA = RSA_new();
	if (remoteRSA != nil) {
		remoteRSA->n = BN_bin2bn(modulus, self.modulusLength, nil);
		remoteRSA->e = BN_bin2bn(exponent, self.exponentLength, nil);
		remoteRSA->d = nil;
		remoteRSA->p = nil;
		remoteRSA->q = nil;
	}
	else {
		DLog(@"Could not create a RSA* object for the remote key, setting it to nil!");
	}
	
	if (self.remoteRSA) {
		RSA_free(self.remoteRSA);
	}
	self.remoteRSA = remoteRSA;
	
	free(exponent);
	free(modulus);
}

/**
 Encrypts the given payload using the remote end's public key. If the public key is missing or any other
 encryption error is encountered, it returns nil.
 @param payload The payload to encrypt.
 @return An encrypted payload.
 */
- (NSData*)encrypt:(NSData*)payload {
	if (!self.remoteRSA) {
		DLog(@"ERROR: Couldn't encrypt the data, remote public key is missing!");
		return nil;
	}
	
	uint8_t* plaintextChunk = malloc(self.chunkLength);
	uint8_t* encryptedChunk = malloc(self.encryptedChunkLength);
	NSMutableData* encryptedData = [NSMutableData data];
	int totalBytesEncrypted = 0;
	
	while (totalBytesEncrypted < payload.length) {
		// Get a chunk of the payload
		NSRange range = NSMakeRange(totalBytesEncrypted, MIN(payload.length - totalBytesEncrypted, self.chunkLength));
		[payload getBytes:plaintextChunk range:range];
		
		// Encrypt it
		int bytesEncrypted = RSA_public_encrypt(range.length, plaintextChunk, encryptedChunk, self.remoteRSA, RSA_PKCS1_OAEP_PADDING);
		
		if (bytesEncrypted != -1) {
			// Append it to the result
			totalBytesEncrypted += bytesEncrypted;
			[encryptedData appendBytes:encryptedChunk length:bytesEncrypted];
		}
		else {
			DLog(@"Encryption error: %s (%s)", ERR_error_string(ERR_get_error(), nil), ERR_reason_error_string(ERR_get_error()));
			free(plaintextChunk);
			free(encryptedChunk);
			return nil;
		}
	}
	
	free(plaintextChunk);
	free(encryptedChunk);
	
	return encryptedData;
}

/**
 Decrypts the given payload using the local private key. If the local key is missing or invalid, or if any
 other encryption error is encountered, it returns nil.
 @param payload The payload to decrypt.
 @return A decrypted payload.
 */
- (NSData*)decrypt:(NSData*)payload {
	if (!RSA_check_key(self.localRSA)) {
		DLog(@"ERROR: Local RSA keypair is not valid, could not decrypt!");
		return nil;
	}
	
	uint8_t* encryptedChunk = malloc(self.encryptedChunkLength);
	uint8_t* decryptedChunk = malloc(self.encryptedChunkLength);
	NSMutableData* decryptedData = [NSMutableData data];
	int totalBytesDecrypted = 0;
	
	while (totalBytesDecrypted < payload.length) {
		// Get a chunk of the encrypted payload
		NSRange range = NSMakeRange(totalBytesDecrypted, MIN(payload.length - totalBytesDecrypted, self.encryptedChunkLength));
		[payload getBytes:encryptedChunk range:range];
		
		// Decrypt it
		int bytesDecrypted = RSA_private_decrypt(range.length, encryptedChunk, decryptedChunk, self.localRSA, RSA_PKCS1_OAEP_PADDING);
		
		if (bytesDecrypted != -1) {
			// Append it to the result
			totalBytesDecrypted += self.encryptedChunkLength;
			[decryptedData appendBytes:decryptedChunk length:bytesDecrypted];
		}
		else {
			DLog(@"Decryption error: %s (%s)", ERR_error_string(ERR_get_error(), nil), ERR_reason_error_string(ERR_get_error()));
			free(encryptedChunk);
			free(decryptedChunk);
			return nil;
		}
	}
	
	free(encryptedChunk);
	free(decryptedChunk);
	
	return decryptedData;
}

/**
 Returns a packed representation of the local public key. Modulus is packed first, then the exponent.
 For the 1024-bit key size, this equals 131 bytes (128 for the modulus, 3 for the exponent).
 @return A packed local public key.
 */
- (NSData*)publicKey {
	if (self.localRSA) {
		BIGNUM* modulus = self.localRSA->n;
		uint8_t* modulusBuffer = malloc(BN_num_bytes(modulus));
		BN_bn2bin(modulus, modulusBuffer);
		
		BIGNUM* exponent = self.localRSA->e;
		uint8_t* exponentBuffer = malloc(BN_num_bytes(exponent));
		BN_bn2bin(exponent, exponentBuffer);
		
		NSMutableData* publicKey = [NSMutableData dataWithBytes:modulusBuffer length:BN_num_bytes(modulus)];
		[publicKey appendBytes:exponentBuffer length:BN_num_bytes(exponent)];
		
		free(exponentBuffer);
		free(modulusBuffer);
		
		return publicKey;
	}
	
	return nil;
}

/**
 Checks if the remote public key has been set.
 @return True if the remote public key is present, false otherwise.
 */
- (BOOL)hasRemotePublicKey {
	return self.remoteRSA != nil;
}

@end