Testing only Python code that has changed using trace

fib.py

def fib(n):
    # just an example function
    if n == 0:
        return 0
    if n == 1:
        return 1
    return fib(n-1) + fib(n-2)

fingerprints.json

{
    "test_fib.test_fib_24": {
        "fib:fib": "8067c3e3f3f7996416aaf6fce74413a0",
        "test_fib:test_fib_24": "1c1a892033920dfc008d4b3ae74b0c00"
    },
    "test_fib.test_fib_25": {
        "fib:fib": "8067c3e3f3f7996416aaf6fce74413a0",
        "test_fib:test_fib_25": "06922ab7715791720b8d308b84ab1851"
    }
}

test_collector.py

all_tests = {}

def test(f):
    name = '{}.{}'.format(f.__module__, f.__name__)
    all_tests[name] = f
    return f

test_fib.py

from fib import fib
from test_collector import test
from time import time

@test
def test_fib_24():
    t = time()
    assert fib(24) == 46368
    print('fib(24) took {:.3f} s'.format(time() - t))

@test
def test_fib_25():
    t = time()
    assert fib(25) == 75025
    print('fib(25) took {:.3f} s'.format(time() - t))

test_runner.py

#!/usr/bin/env python3

import argparse
import sys
import trace
import hashlib
import json

import test_fib
from test_collector import all_tests

def main():
    p = argparse.ArgumentParser()
    p.add_argument('--no-fingerprinting', '-n', action='store_true')
    p.add_argument('--fingerprints-file', '-f', default='fingerprints.json', metavar='FILE')
    args = p.parse_args()
    if args.no_fingerprinting:
        # run tests
        for test_name, test_f in sorted(all_tests.items()):
            print('Running test {}'.format(test_name))
            test_f()
    else:
        # load fingerprints
        try:
            fingerprints = json.load(open(args.fingerprints_file))
        except (FileNotFoundError, ValueError) as e:
            fingerprints = {}
        # run tests
        for test_name, test_f in sorted(all_tests.items()):
            if fingerprint_is_actual(fingerprints.get(test_name)):
                print('Skipping test {}'.format(test_name))
                continue
            print('Running test {}'.format(test_name))
            tracer = trace.Trace(countfuncs=True)
            tracer.runfunc(test_f)
            results = tracer.results()
            fingerprints[test_name] = fingerprint_from_tracer_results(tracer.results())
        # save fingerprints
        with open(args.fingerprints_file, 'w') as f:
            json.dump(fingerprints, f, indent=4, sort_keys=True)

def func_code_md5(f):
    to_hash = f.__code__.co_code, f.__code__.co_consts
    return hashlib.md5(str(to_hash).encode('UTF-8')).hexdigest()

def fingerprint_is_actual(fp):
    if not fp:
        return False
    for func_loc, func_md5 in sorted(fp.items()):
        modname, funcname = func_loc.split(':')
        mod = sys.modules[modname]
        func = getattr(mod, funcname)
        func_real_md5 = func_code_md5(func)
        if func_real_md5 != func_md5:
            return False
    return True

def fingerprint_from_tracer_results(results):
    fp = {}
    for filename, modname, funcname in results.calledfuncs:
        mod = sys.modules[modname]
        assert mod.__file__ == filename
        func = getattr(mod, funcname)
        func_md5 = func_code_md5(func)
        fp[modname + ':' + funcname] = func_md5
    return fp

if __name__ == '__main__':
    main()