preprocessor-streaming.py

#!/usr/bin/python

import resource
import timeit


init = """

import hashlib
import subprocess

cmd = ["g++", "-std=c++11", "-E", "botan_all.cpp"]

"""


t = timeit.Timer(
"""
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()

hasher = hashlib.md5()
hasher.update(stdout)
hash = hasher.hexdigest()
"""
, init)

print("Read entire file: {}".format(t.timeit(number=10)))

t = timeit.Timer(
"""
hasher = hashlib.md5()

proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in proc.stdout:
	hasher.update(line)

for line in proc.stderr:
	print(line)

hash = hasher.hexdigest()
"""
, init)

print("Stream by line: {}".format(t.timeit(number=10)))

for partsize in [16, 64, 256, 1024, 2048, 4096, 8192, 16384, 32768, 65536]:
    t = timeit.Timer(
"""
hasher = hashlib.md5()

def read_part(instream, size={}):
    while True:
        data = instream.read(size)
        if not data:
            return
        else:
            yield data

proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in read_part(proc.stdout):
    hasher.update(line)

hash = hasher.hexdigest()
""".format(partsize), init)
    print("Stream buffer size {}: {}".format(partsize, t.timeit(number=10)))

# http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
print("{} kilobytes on Linux, bytes on OS X".format(
        resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))