autograd_benchmark_viz.py

%pylab inline

#!/usr/bin/env python
from stat import S_ISREG, ST_CTIME, ST_MODE
import os, sys, time

# path to the directory (relative or absolute)
dirpath = "/Users/Alex/Code/autograd/.asv/results/quincy.local/"

# get all entries in the directory w/ stats
entries = (os.path.join(dirpath, fn) for fn in os.listdir(dirpath))
entries = ((os.stat(path), path) for path in entries)

# leave only regular files, insert creation date
entries = ((stat[ST_CTIME], path)
           for stat, path in entries if S_ISREG(stat[ST_MODE]))
#NOTE: on Windows `ST_CTIME` is a creation date 
#  but on Unix it could be something else
#NOTE: use `ST_MTIME` to sort by a modification date

paths = [p[1] for p in sorted(entries,key=lambda x:time.ctime(x[0]))][::-1]

import json
results = []
for path in paths:
    with open(path,"r") as f:
        results.append(json.load(f))

manual_speed_name = "benchmarks.RNNSuite.time_manual_rnn_grad"
autograd_speed_name = "benchmarks.RNNSuite.time_rnn_grad"
manual_peakmem_name = "benchmarks.RNNSuite.peakmem_manual_rnn_grad"
autograd_peakmem_name = "benchmarks.RNNSuite.peakmem_rnn_grad"
        
results = [r for r in results 
           if "results" in r 
           and manual_name in r['results']]

manual_speed = np.array([r['results'][manual_speed_name] for r in results])
autograd_speed = np.array([r['results'][autograd_speed_name] for r in results])
manual_peakmem = np.array([r['results'][manual_peakmem_name] for r in results])
autograd_peakmem = np.array([r['results'][autograd_peakmem_name] for r in results])

figure(figsize=(8,3))
gs = GridSpec(1,2)
subplot(gs[0])
plot(manual_speed,label="Manual")
plot(autograd_speed,label="Autograd")
ylabel("Execution time (sec)")
xlabel("Commit #")
legend()

subplot(gs[1])
plot(manual_peakmem,label="Manual")
plot(autograd_peakmem,label="Autograd")
ylabel("Peak Memory (MB)")
xlabel("Commit #")
legend()