Code benchmarking approach with correctness and usefulness ambitions

bench.py

"""Describes "a better way" to benchmark code and present results.

Key points:
- repeat benchmark 5-7 times, take only best time, translate to single operation speed/duration
- calculate relative delta (new/old-1) of single op speed/duration
- translate delta crossing of boundary values to human friendly labels

Take it as is or add to your awesome benchmarking library.
"""
import timeit

# concrete code of benchmarked functions is not important here

def f1(d, key, value):
    d.setdefault(key, []).append(value)


def f2(d, key, value):
    if key not in d:
        d[key] = []
    d[key].append(value)


def bench(fun, n=None):
    gs = dict(
        fun=fun,
        state={},
        key="kkey",
        value=42,
    )
    t = timeit.Timer("""fun(state, key, value)""", globals=gs)
    if not n:
        n = t.autorange()[0] * 2
    # Important piece #1: timeit.repeat, take only best time, translate to single operation speed/duration
    raw = t.repeat(7, n)
    op_ns = int(1e9 / n * min(raw))
    return fun, n, op_ns


rs = [bench(fun) for fun in (f1, f2)]
for fun, n, op_ns in rs:
    print(f"{fun.__name__}\t{n}\t{op_ns} ns/op")
# Important piece #2: change = new / old - 1 or (new - old) / old
delta_rel = rs[1][2] / rs[0][2] - 1  # + slower, - faster
ignore = .02
important = .1
# Important piece #3: translate delta crossing drop-off values to human friendly label
label = {
    (-delta_rel) >= important: "+2/awesome",
    ignore <= (-delta_rel) < important: "+1/improved",
    0 <= abs(delta_rel) < ignore: "0/neutral",
    ignore <= delta_rel < important: "-1/slowed",
    delta_rel >= important: "-2/issue",
}.get(True, "?/unlabeled")
print(f"{label} {int(delta_rel*100)}%")