Compare pytest benchmarks

compare.py

import json
import pandas as pd
from sys import argv, exit


def benchmark_to_df(json_file):
    with open(json_file) as jf:
        content = json.load(jf)
        df = pd.DataFrame(columns=("test", "time [ms]"))
        for b in content["benchmarks"]:
            df = df.append({"test": b["name"],
                            "time [ms]": b["stats"]["mean"]*1000.0},
                           ignore_index=True)
        return df


if __name__ == "__main__":
    if len(argv) != 3:
        print("Usage: compare.py before.json after.json")
        exit(1)

    before = benchmark_to_df(argv[1])
    after = benchmark_to_df(argv[2])
    both = pd.merge(before, after, how="inner", on="test",
                    suffixes=(" before", " after"))
    both["fraction"] = both.iloc[:, 1] / both.iloc[:, 2]

    print(both.sort("fraction"))
    exit(0)