split-input.py

#!/usr/bin/python
#
# As I demonstred here https://gist.github.com/gsilos/879c28279761cd8d38301f7a54bed78a
# openssl smime can't decrypt file with more than 2GB.
#
# This script is a workaround to handle the openssl decrypt limit. It will split the input 
# stream into chunks of customized size. the splitted chunk will be used as input for openssl
# smime encryption.
# The outcome of this script will be a tar package with all the chunks in it.
#

import sys, os, shlex, subprocess, tarfile

BLOCKSIZE=4096

# for testing purposes, generate file with:
# dd if=/dev/zero of=./file bs=4k iflag=fullblock,count_bytes count=5G
# or just cat whatever bigger than 5G you have in your filesystem :)

dump_cmd = shlex.split('cat file')
dump = subprocess.Popen(dump_cmd, stdout=subprocess.PIPE, stderr=sys.stdout)

# start the first chunk

chunkcount = 0
chunkname = "out{}".format(chunkcount)
chunkmerge = "out"
byte_count = 0
# set this to the maximum openssl decrypt can handle
# according to this test: https://gist.github.com/gsilos/879c28279761cd8d38301f7a54bed78a
# the maximum is 2004626079
chunksize = 2004626079
encrypt_cmd = shlex.split('openssl smime -encrypt -aes256 -stream -binary -text -outform DEM -out {} public.key'.format(chunkname))
encrypt = subprocess.Popen(encrypt_cmd , stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr)

block = dump.stdout.read(BLOCKSIZE)

# each chunk will be added to this tar package.

tar = tarfile.open(chunkmerge, "w")

while len(block) != 0:

    byte_count += len(block)

    if byte_count <= chunksize:

        encrypt.stdin.write(block)

    elif byte_count > chunksize:

        # finish the last chunk

        encrypt.stdin.close()

        encrypt.wait()

        print "encrypt returncode: {}".format(encrypt.returncode)

        tar.add(chunkname)

        os.remove(chunkname)

        # prepare for the next chunk

        byte_count = 0
        byte_count += len(block)
        chunkcount += 1
        chunkname = "out{}".format(chunkcount)
        encrypt_cmd = shlex.split('openssl smime -encrypt -aes256 -stream -binary -text -outform DEM -out {} public.key'.format(chunkname))
        encrypt = subprocess.Popen(encrypt_cmd, stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr)
        encrypt.stdin.write(block)

    block = dump.stdout.read(BLOCKSIZE)

# finish the last chunk

encrypt.stdin.close()

encrypt.wait()

print "encrypt returncode: {}".format(encrypt.returncode)

dump.wait()

if dump.returncode != 0:
    print "dump returned error: {}".format(dump.returncode)

tar.add(chunkname)

tar.close()

os.remove(chunkname)

to decrypt, you can you can use something like this:

#!/usr/bin/bash

tar xvf out
seq=0
while true
do
	if [ -f out${seq} ]; then
		openssl smime -decrypt -indef -binary -inform DER -in out${seq} -inkey private.key -out out${seq}.d
		cat out${seq}.d >> out.d
		rm out${seq}.d out${seq}
		echo $seq
	else
		break
	fi
	seq=$((seq + 1))
        echo compare out.d with the original file using md5
done