Docker Save - export docker image, calculate md5 / sha 256 on the fly and write to disk

dockersave.py

# Small util to avoid hammering the disk by reading and writing files
# all the time. Attempts to export the docker container, calculate md5
# and sha256 sums on the fly, compress, gpg encrypt and write it to disk
#
# Some credit to some Stack Overflow posts is due!

import hashlib
import os
import resource
import subprocess
import shlex
import sys
import time


def report_usage():
    print "pid: ", os.getpid()
    print "Max RSS used = {}".format(
            resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)


def report_status(c, from_time, to_time):
    print "wrote {} bytes ({:.2f} MB/s)".format(c, long(c)/(to_time-from_time)/1024.0/1024.0)


def check_args():
    if len(sys.argv) != 3:
        print "Usage: {} <container name> <tag>".format(sys.argv[0])
        sys.exit(1)


# From here on this code could be converted to a class eg Packager/Exporter

# buffer size
bufsz = 4096 * 10
# counter
c = 0
now = time.time()

report_usage()
md5sum = hashlib.md5()
sha256 = hashlib.sha256()
check_args()
(container, tag) = sys.argv[1:]

dockercmd = 'docker save "{}:{}"'.format(container, tag)
print dockercmd

proc = subprocess.Popen(shlex.split(dockercmd), stdout=subprocess.PIPE)
output = proc.stdout

file_out = open("output.tar", "wb")
#gz_cmd = "gzip -c - "
#gz_proc = subprocess.Popen(shlex.split(gz_cmd), stdin=subprocess.PIPE, stdout=subprocess.PIPE)

gpg_stdin_pass = True
if gpg_stdin_pass:
    gpg_cmd = "gpg --passphrase-fd 0 -c -"
else:
    gpg_cmd = "gpg --passphrase PASSPHRASE -c -"

gpg_proc = subprocess.Popen(shlex.split(gpg_cmd), stdin=subprocess.PIPE, stdout=file_out)

#f = open("foo", "wb")
#gpg_proc = subprocess.Popen(shlex.split(gpg_cmd), stdin=subprocess.PIPE, stdout=subprocess.PIPE)

if gpg_stdin_pass:
    gpg_proc.stdin.write("PASSPHRASE\n")
    gpg_proc.stdin.flush()

#f = open("foo", "wb")

begin = time.time()
for chunk in iter(lambda: output.read(bufsz), b''):
    # count the bytes
    c += len(chunk)
    # calculate the sum
    md5sum.update(chunk)
    sha256.update(chunk)
    # write to gpg
    gpg_proc.stdin.write(chunk)
    # write to disk
    #f.write(chunk)
    if (time.time() - now) > 6:
        now = time.time()
        report_status(c, begin, now)

# report status if all done
report_status(c, begin, time.time())


# Ensure buffer is flushed
gpg_proc.stdin.flush()

#f.close()
manifest = {
            "md5": md5sum.hexdigest(),
            "sha256": sha256.hexdigest(),
            "container": container,
            "tag": tag,
            "c_size": c
           }

print manifest

report_usage()

new output:

$ time python docker-package.py tomcat API_1.7.0
pid:  79066
Max RSS used = 5292032
docker save "tomcat:API_1.7.0"
Reading passphrase from file descriptor 0
wrote 24903680 bytes (3.96 MB/s)
wrote 144261120 bytes (11.47 MB/s)
wrote 257269760 bytes (13.64 MB/s)
wrote 359628800 bytes (14.29 MB/s)
wrote 399212032 bytes (14.63 MB/s)
{'tag': 'API_1.7.0', 'sha256': '1c947aaf70cfeb66a664ff7fa3ddd5d8673a458e26ade06504bad1b7eab2b82d', 'container': 'tomcat', 'c_size': 399212032, 'md5': 'edfe5d76bb9836d35e703dd7d1c874f9'}
pid:  79066
Max RSS used = 5627904

real	0m26.196s
user	0m5.237s
sys	0m1.232s