benchmark script for nose tests

speed.py

#!/usr/bin/env python

import subprocess
import sys

trials = 5
force_drop = True

branch = 'factories'
test_db = 'test_zamboni'
drop_db = 'mysql -u root -e "drop database %s"' % test_db
test_runner = 'manage.py test --settings=settings_local_mkt --noinput -s --logging-clear-handlers'
test_to_run = 'webapps'


def call(s):
    stdout, stderr = subprocess.Popen(s.split(), stdout=subprocess.PIPE,
        stderr=subprocess.PIPE).communicate()
    return stderr


def test():
    output = call(test_runner + ' ' + test_to_run)
    for line in output.split('\n'):
        if line.startswith('Ran'):
            duration = line.strip('s').split(' ')[-1]
            return line, float(duration)


def do_trials():
    total_duration = 0.0
    for x in xrange(trials):
        if force_drop:
            print '\tDropping database ...'
            call(drop_db)
        print '\tTrial %s:' % (x + 1)
        line, duration = test()
        total_duration += duration
        print '\t\t' + line
    print '\tTotal duration: %.4fs' % total_duration
    print '\tAverage time per trial: %.4fs' % (total_duration / trials)
    return total_duration


def _main():
    global branch
    global test_to_run
    if len(sys.argv) == 2:
        branch = sys.argv[1]
    if len(sys.argv) == 3:
        test_to_run = sys.argv[2]
    print 'Benchmarks for %s:' % test_to_run

    call('git checkout %s' % branch)
    print 'After:'
    after = do_trials()

    call('git checkout master')
    print 'Before:'
    before = do_trials()

    stats = {
        'before': before,
        'after': after,
        'trials': trials,
        'speedup': before / after,
        'diff': (before - after) / trials,
    }
    print ('Before (%(before).4f / %(trials)s) vs. '
           'After (%(after).4f / %(trials)s) => %(diff).4fs faster '
           '(%(speedup).4fx speedup)') % stats


if __name__ == '__main__':
    _main()