Compare perf stat between programs

perf-cmp.py

#!/usr/bin/env python3

import argparse
import json
import os
import re
from scipy.stats import ttest_ind_from_stats
from subprocess import DEVNULL, Popen
import sys


class ArgParser(argparse.ArgumentParser):
    def __init__(self):
        super().__init__()
        self.add_argument("-r", "--repeat", type=int, default=1)

    def parse_args(self, args=None, namespace=None):
        if args is None:
            args = sys.argv[1:]

        perf_args = []
        separator = None
        commands = [[]]

        for arg in args:
            if separator is None:
                if arg.count('-') == len(arg) >= 2:
                    separator = arg
                else:
                    perf_args.append(arg)
            elif arg == separator:
                commands.append([])
            else:
                commands[-1].append(arg)

        ret, _ = self.parse_known_args(perf_args, namespace)
        ret.perf_args = perf_args
        ret.commands = commands

        if [] in commands:
            self.error("Empty command")

        return ret

    def print_help(self, file=None):
        if file is None:
            file = sys.stdout

        prog = self.prog
        text = f"""
 Usage: {prog} [<perf args>...] -- first command -- second command ...

 <perf args> are passed directly to perf stat.

    -r, --repeat <n>      repeat each command <n> times

 Commands are separated with two or more dashes (--).  The first command is the
 baseline, and all others will be compared to it.  If you need to pass -- to a
 command, use more dashes to separate them.  For example:

    $ {prog} -d -r10 --- foo -- bar --- foo -- baz
"""
        print(text, file=file)

    def print_usage(self, file=None):
        self.print_help(file)


def perf_stat(args, command):
    r, w = os.pipe()
    os.set_inheritable(w, True)

    proc = Popen(
        ["perf", "stat", "--json-output", f"--log-fd={w}"] + args.perf_args + ["--"] + command,
        stdin=DEVNULL, stdout=DEVNULL,
        close_fds=False,
    )
    with proc:
        os.close(w)
        with os.fdopen(r, "r") as r:
            metrics = []
            for line in r:
                metrics.append(json.loads(line))
            return metrics


class Table:
    def __init__(self):
        self.rows = []
        self.numeric_cols = set()
        self.col_padding = {}
        self.group_rows = 0

    def append(self, row):
        self.rows.append(row)

    def justify(self):
        ncols = max(len(row) for row in self.rows)

        for row in self.rows:
            while len(row) < ncols:
                row.append("")

        for i in range(ncols):
            for row in self.rows:
                if row[i] is None:
                    row[i] = ""

            width = max(len(row[i]) for row in self.rows)
            if i in self.col_padding:
                width += self.col_padding[i]

            for row in self.rows:
                if i in self.numeric_cols:
                    row[i] = row[i].rjust(width)
                else:
                    row[i] = row[i].ljust(width)

    def print(self):
        self.justify()

        for i, row in enumerate(self.rows):
            string = " ".join(row)
            if self.group_rows:
                if (i // self.group_rows) % 2 == 1:
                    string = f"\033[7m{string}\033[m"
            print(string)


def to_number(value):
    if not isinstance(value, str):
        return value

    v = re.sub(r"\.0+$", "", value)
    try:
        return int(v)
    except ValueError:
        try:
            return float(v)
        except ValueError:
            return value


def is_number(*args):
    return all(type(arg) in [int, float] for arg in args)


def tabulate(args, metric, baseline=None):
    if baseline is None:
        baseline = {}

    row = [""]

    value = to_number(metric.get("counter-value"))
    base_val = to_number(baseline.get("counter-value"))
    if is_number(value):
        if value == base_val:
            row.append("॥")
        elif isinstance(value, int):
            row.append(f"{value:,}")
        else:
            row.append(f"{value:.2f}")
    else:
        row.append(value)

    if is_number(value, base_val) and value != base_val:
        try:
            diff = 100 * (value / base_val - 1)
        except ZeroDivisionError:
            diff = float("inf")
        row[0] = f"[{diff:+.2f}%]"

    cv = to_number(metric.get("variance"))
    if is_number(value, cv):
        row += ["(±", f"{cv:.2f}%)"]
    else:
        row += ["", ""]

    if baseline:
        row.append("")
    else:
        row.append(metric.get("unit"))

    base_cv = to_number(baseline.get("variance"))
    if is_number(value, cv, base_val, base_cv):
        mean1 = value
        std1 = cv * value / 100
        mean2 = base_val
        cv2 = base_cv
        std2 = cv2 * mean2 / 100
        ttest = ttest_ind_from_stats(
            mean1, std1, args.repeat,
            mean2, std2, args.repeat,
            equal_var=False, alternative="less",
        )
        p = ttest.pvalue
        if p < 0.05:
            row.append(f"< base (p={p:.2})")
        else:
            row.append(f"≳ base (p={p:.2})")
    elif baseline:
        row.append("")
    else:
        row.append(metric.get("event"))

    row.append("  #  ")

    metric_val = to_number(metric.get("metric-value"))
    base_mval = to_number(baseline.get("metric-value"))
    if is_number(metric_val):
        if metric_val == base_mval:
            row.append("॥")
        else:
            row.append(f"{metric_val:.2f}")
    else:
        row.append(metric_val)

    if baseline:
        row.append("")
    else:
        row.append(metric.get("metric-unit"))

    return row


if __name__ == "__main__":
    args = ArgParser().parse_args()
    results = []

    print()
    for i, cmd in enumerate(args.commands):
        string = " ".join(cmd)
        if i == 0:
            print(f" Performance counter stats for '{string}' ({args.repeat} runs)")
        else:
            print(f"                           vs. '{string}' ({args.repeat} runs)")
        results.append(perf_stat(args, cmd))
    print()

    table = Table()
    table.numeric_cols = {0, 1, 3, 7}
    table.col_padding[0] = 4
    table.col_padding[1] = 4
    if len(results) > 1 and sys.stdout.isatty():
        table.group_rows = len(results)

    for i, baseline in enumerate(results[0]):
        table.append(tabulate(args, baseline))

        for metrics in results[1:]:
            metric = metrics[i]
            assert baseline.get("event") == metric.get("event")
            table.append(tabulate(args, metric, baseline))

    table.print()
    print()