openssl-smime-limits.sh

#!/usr/bin/env bash
#
# Test smime encryption/decryption limits.
#
# There is something in smime design break it when decrypting files that have more than 2GB in size. 
# By playing with ulimit max virtual memory and the size of the message that will be encrypted/decrypted
# I found the maximum size in bytes a original message can have before decrypt throwns an error.
#
# This is a test script that make easier talk about this problem with people.
#
# Tested with: OpenSSL 1.0.2g  1 Mar 2016
# 
# To generate private.key and public.key, use:
# openssl req -x509 -nodes -days 1000000 -newkey rsa:4096 -keyout private.key  -out public.key
#
# The whole test will generate ~ 6GB of space in your filesystem
#
# Test scenarios:
#
# Use Case 1:
# ./openssl-smime-limits.sh 5399368 2004626079
# Result: Test OK
#
# Use Case 2:
# ./openssl-smime-limits.sh 5399367 2004626079
# Result: Fail
#
# Use Case 3:
# ./openssl-smime-limits.sh 5399368 2004626080
# Result: Fail
#
# Conclusion (tested on my machine and few EC2 instances):
# Openssl smime can't decrypt message that has more than 2004626079 bytes. 
# Virtual memory set to 5399368 bytes, is the minimum size before openssl crashes.
#
# both Use case 2 and 3 will generate an error like that:
#Error reading S/MIME message
#140510123636376:error:07069041:memory buffer routines:BUF_MEM_grow_clean:malloc failure:buffer.c:159:
#140510123636376:error:0D08C041:asn1 encoding routines:COLLECT_DATA:malloc failure:tasn_dec.c:1134:
#140510123636376:error:0D08303A:asn1 encoding routines:ASN1_TEMPLATE_NOEXP_D2I:nested asn1 error:tasn_dec.c:707:Field=enc_data, Type=PKCS7_ENC_CONTENT
#140510123636376:error:0D08303A:asn1 encoding routines:ASN1_TEMPLATE_NOEXP_D2I:nested asn1 error:tasn_dec.c:717:Field=enc_data, Type=PKCS7_ENVELOPE
#140510123636376:error:0D08303A:asn1 encoding routines:ASN1_TEMPLATE_NOEXP_D2I:nested asn1 error:tasn_dec.c:717:
#140510123636376:error:0D08403A:asn1 encoding routines:ASN1_TEMPLATE_EX_D2I:nested asn1 error:tasn_dec.c:580:Field=d.enveloped, Type=PKCS7

# if you are lazy , here are pub and priv keys.
# if you generate the keys, just coment the these out :D

cat <<EOF > private.key
-----BEGIN PRIVATE KEY-----
MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCqREhNdEZG2Ln3
5K++nnZZHrsc1HxjojEozfYTrKsxVvHJdu0pbFlMKYxVmP2DVeZbD5fYYqjsZTHl
O6O1zK69rDBZWdFSJ+saH04umUkce/WE5ra9QR48ChKvM3zTUY1SFBTUKhFffDvV
NX0W57O2fU4gMKnCvy42pD7qkm9e4vc6yieAOKRdNUAfTWu67w9JV8MknSBfQL1p
gudrboOG+ueQEZshgVo+ZcR5JXByb7pT2Nx6Ag59eNK2rqcgVNFGdNUcGy+NjDUr
U+NufdiBEcrkwMl0u5nONDepi8rnI5Xrv8moLwqky93CsMAl45DOnSFgYudpkDv3
76B5LlKqGOzxP3wqfp8hGCCPY+IhdG1tIt7aZtxlhgVQVFpYP2Ppg8+BY2/Ho1NO
lY+SPRZmryTW3kGatU8PQrBbeld2PXWtzSkfRytxwkbissyB3kEnSGjmN4uMWtfe
+bqk+Ukte2D1muqtBf/fZDgDKVOGeLMvD0KUZ0NcDbt2seoimEC5sZatsbGC/gjA
Dexq9gCroWkmbhZpp2mAU5DajPGMDM7eFuvUyXE6F0pMeX4yD0WTwCXoHKW01lat
8E9DDtV3lxpcQ/ZnWadFpw3b7HCEtET5wUQV/wI5Fdurs9lZS5v1t5MQIjquDBr/
+DkjNcpv8WawNfZisy7rsRSYgF/7XQIDAQABAoICAQCRDDZj9UfhFsPmUCT/SFMd
vLY96y5EWmlyj3KwdAoqsByuOefzv9+aqFJqoThmaAkjTfdXWUT6jYKh+9qNB19d
KZDOrVtKl8MKr9yUifTklR0lKL5IH284SoR8SGpwQysW3xCXZXcDVvxJt5Vmq+BG
+BxUA+C6jM87sVP/1NuXg9aFeK2J57btyG1lvTinOPxWyLLvQAPR4X76oY+vz6ew
l5IpAzaapm1QdrWXVrOiaVfcEgRQ0+uwK/ulsMejHV5inSjnFqm99dFdpcs7121X
Fuc3VJzKGv790TIR94rUuVYATZPm+Jwio6G5N4WZQXGc8fL6/4aQQVU09ChK05GK
/c3vecUL7j/9pGAkcvTEj7M2W9euZFIK/uckZqoWuqUOnq4gu5o/4RgKMGtLhAzo
NzsuFhzB5enxfV1SJKNs1GfwaBfMKs00CS29wHBtooe4oHgmexCtpkB+3eAQUA8Q
eSG43F4TaFFxJHQIotRuhzmP+MWPoGrJ8Vm/bbJlNMF89laFoaF5fnMOc5dCnUTB
dA3MDIpAu/jpDNWY43OXFbHntaTKX7TytTO30kEG/NqXTy0tg5p/14b5Xxo8tyFX
P44nM99P/ybhKpf9WzwMu9jV0dr29AQWkK9NxXl8mGiOx3GZcfkBeUqswsiGi5yN
9m83khAwrYRou/I2RzivgQKCAQEA3xOERNvUbaidU6ldNsrNwj56r7QxMifzZlOc
+Olg+yuiMPpa5KR+UYUXT0fNzHNuoqWUXBUTUgzX0e20fYpxNTbdeX16XOvdvf2X
REH+IXyg8fjUG5y3kufIoLgfccJmFx39y4jOhBEa+twchix7/q4jm5L+qC08+ph4
xDLsy0vTXEt3JSLikpDZ/W/zasjoQQh0Syspb7odywutt+NIFa3TG84lgauQ5nJ2
SWvQ7QHSk3avxyeZOVcOtF+pmBJLTkWx/i6vdXtMwIs+p0iZY5ufysmIg0OBqT+W
D4vkzWtUi61q1xyqwCMx+gyhtBwsICqIT3J81fBr/DjbkL//cQKCAQEAw2V3LZzt
Ev1ywQ4a7rvltbAIlUW0SxH4wSTEvueHBeDgRg3tyPGfTpid0bRuZPOBOTT/pe+P
4HB8R2WUxlHZuEOiQyTaI5YgnGbURq80yqeyKd1YKuv1HNSPl36lZ/ttfYtX2rjf
Ytlrml731mvE4b1oLN804F/HXZX4dMqqjoBCSd6WGQwRQ/xpP40xgTmC8H+n2GfY
5kA9k5hvUvxMMdqKkyxIy6i1BTKj5VOkPtxfYRoXmaPWsgOWu4eWFTRKsxgmGF+o
ti+VhR8MXhWpFbSBJ7serbPBPhZdUP/VfesxWmi6QBw+1zluD4mJWM1IO6ooiVnp
OzFkjOgNobAcrQKCAQA1ZHB2Ir+Xd5IgYhsjcCsLYPTwdQqJXLKPPVgG46UIuQra
+jEjJJPvYxGPMjp5wu8qEsONvqWfL9/AKUrX+XjbKcScvYG6F+I9BSNdfQYccb37
7HioGzx89uk4PpmYGtVglxcVzirPxWBVxrU1EBSlgnyXUfmNuzYrLK9LFn1Cs2Vl
Y9KimtTsC6uBU3hJiMi+OA3NqAkE6uESpjrwFoA8YBx1PUQQ40WNQrw5GospDxSd
ovyziDDOXp3wYi0vbsGDfH77vaTVU6utMibO1zuPccIvQUlhUZkJL2T1V6Bt4lCc
FefKkNv1nPbrSHazq0KMeGGj0l06v465EcrLvIihAoIBACLeRXE3UrsTzVmUGcxG
bYyJmE6C/NR0pY5Z8F4c1OSAp6T4sVktfP86jtcwOo77QTEjOweyg1tK13KqNu1Z
B+Aw+ZC/z8mP7cvbMcBXXbVYTZvY6vXl23zElcc0C3JN1NEVf50BZqOuJtyKW/HA
k4axFRkEz9vGW75gl9DHpb3+N1YVSXW4yI/4SBPWL2JfD/W44gvlJDiaeCKRjjQh
37r5w8bVnTvypDMEQWkYDvREwSSD0nhRW2uy5kAaxvaLuVTSV2H8GBBC14zE6KEV
fnHoHzAm3SekLzTUKtzceo2Y3drDW5E7OR+RGQnewkq9f0leaRNxnwQ4IThf5KbY
1ZUCggEAO0hHLfrmm01fpFuZEoeawuoDbMestv3oM4q/bupxaTyEx2oxx5wbAcJ3
fU3wHwpnJr09V/eBmrZy3fOx53+Q6L3pvfIQ9TANqOfna11uPTX7IksAjZdZFRnN
1Z4OD2ysASLDE6fQY9tGo/2qtxiIo58y2YAiXoCXuKJ0mG7bs6BUOCnxZ3Hvr1+T
f5/tPng0Wuhb2FRkP7FnZrxfua1yOQwKRdBWKOj4avaKQx/h2QKlbpQ5Wq+e9BoZ
eIODqzq4i/8BTpD/D2eDIrQOqnvfz7Qxno/73jiWe7kIY95idr6+k/XHadbStqaL
DAyIVXWTbikfoCnRiEma1WbANJiTRQ==
-----END PRIVATE KEY-----
EOF

cat <<EOF > public.key
-----BEGIN CERTIFICATE-----
MIIEzjCCArYCCQDBLysJu6HnfjANBgkqhkiG9w0BAQsFADAoMSYwJAYJKoZIhvcN
AQkBFhdkYXJrcmlzay12QGRhcmtzdGFyLm9yZzAgFw0xODA5MTQxNzIwMzNaGA80
NzU2MDgxMTE3MjAzM1owKDEmMCQGCSqGSIb3DQEJARYXZGFya3Jpc2stdkBkYXJr
c3Rhci5vcmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCqREhNdEZG
2Ln35K++nnZZHrsc1HxjojEozfYTrKsxVvHJdu0pbFlMKYxVmP2DVeZbD5fYYqjs
ZTHlO6O1zK69rDBZWdFSJ+saH04umUkce/WE5ra9QR48ChKvM3zTUY1SFBTUKhFf
fDvVNX0W57O2fU4gMKnCvy42pD7qkm9e4vc6yieAOKRdNUAfTWu67w9JV8MknSBf
QL1pgudrboOG+ueQEZshgVo+ZcR5JXByb7pT2Nx6Ag59eNK2rqcgVNFGdNUcGy+N
jDUrU+NufdiBEcrkwMl0u5nONDepi8rnI5Xrv8moLwqky93CsMAl45DOnSFgYudp
kDv376B5LlKqGOzxP3wqfp8hGCCPY+IhdG1tIt7aZtxlhgVQVFpYP2Ppg8+BY2/H
o1NOlY+SPRZmryTW3kGatU8PQrBbeld2PXWtzSkfRytxwkbissyB3kEnSGjmN4uM
Wtfe+bqk+Ukte2D1muqtBf/fZDgDKVOGeLMvD0KUZ0NcDbt2seoimEC5sZatsbGC
/gjADexq9gCroWkmbhZpp2mAU5DajPGMDM7eFuvUyXE6F0pMeX4yD0WTwCXoHKW0
1lat8E9DDtV3lxpcQ/ZnWadFpw3b7HCEtET5wUQV/wI5Fdurs9lZS5v1t5MQIjqu
DBr/+DkjNcpv8WawNfZisy7rsRSYgF/7XQIDAQABMA0GCSqGSIb3DQEBCwUAA4IC
AQAQHgPvrPLXkkvWdrTvo4RlEFUcVHEFQdUpAVGDznxrkz4GEyQ57jDsY3YEC2Fk
+nIP+J12sy9x5KbDe3NWNwu1BYX3YZXnW/+RqH/2ZPIsdvszmj0VyBWaK56CyjfY
PfUEy6oinPwXZz8oFjDW28IpwulB8CJQEailrDg4PGVd4jOok5SHgaGMtX0FkU4p
XDEBSYzE+TsC00e+O5QyD0h1O6SRDlIdG5elvUR0hqXdy0Df1yu3DzSmBWq0eUL1
m3FMD7rvEIYX9YdGKF6I/C3BzCAFaOfRjtM+GhFUB1yrv44eAe6A5DSIUPszuoHi
ivGn4tBWVMhyTowRR2enWyLA06rGPcoh1n7XGk2qhVHJQIanBTCMtrf8qAbexlsQ
oXOzJNINCB9hYYNkm9Ig5HowKB/nimecQ4LHgSl6PQJ81RecM/qRnj/fZB55N9qt
sUwOITW7llwzuTAHvFQXf1Td385DkF/58WmD8yr3gQwp59ob/C8Jb2SScSl6fz5C
ZjXVRYpfzhvkgEAZMcrgCR4rXJ7Xi/Pi89QuFpzGgOyswySxv88r4g5A0DMFubLN
Pidhw9sWLTgeQ8tcDFmepmjq1FyC3aY19sDHAhabGVyzM3RgcYr0PXd0tq7pWgO6
NxtLQYQESRw1IaQuqpdmM09/bLMfjam/minFNImXuR78eA==
-----END CERTIFICATE-----
EOF

openssl_version=$(openssl version)
if [ $? -eq 0 ]; then
    echo $openssl_version
else
    echo no openssl found ?... I need at lest OpenSSL 1.0.2g  1 Mar 2016 installed...
    exit
fi

# limit virtual memory
# magic virtual memory limit: 5399368
if [ -z "$1" ]; then
    max_virtual_memory=5399368
else
    max_virtual_memory=$1
fi
echo testing with max_virtual_memory=$max_virtual_memory

ulimit -v ${max_virtual_memory}
# create a bigfile
# magic file size limit: 2004626079
if [ -z "$2" ]; then
    max_file_size=2004626079
else
    max_file_size=$2
fi
echo testing with max_file_size=$max_file_size

if [ -f bigfile ]; then
    rm bigfile
fi

dd if=/dev/zero of=bigfile bs=1 count=0 seek=${max_file_size}
ls -l bigfile

# encrypt
openssl smime -encrypt -aes256 -stream -binary -text -outform DEM -in bigfile -out bigfile.crypted public.key
echo encrypt returned $?
ls -l bigfile.crypted
# decrypt
openssl smime -decrypt -indef -binary -inform DER -in bigfile.crypted -inkey private.key -out bigfile.decrypted
echo decrypt returned $?
ls -l bigfile.decrypted