|
#!/usr/bin/env python3 |
|
from __future__ import annotations |
|
|
|
import csv |
|
import json |
|
import os |
|
import random |
|
import statistics |
|
import subprocess |
|
import sys |
|
import time |
|
from dataclasses import dataclass |
|
from pathlib import Path |
|
|
|
import matplotlib.pyplot as plt |
|
import matplotlib.ticker as mticker |
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2] |
|
OUT = ROOT / ".generated" / "blog-performance" |
|
RESULTS = OUT / "results" |
|
FIGURES = OUT / "figures" |
|
|
|
BASELINE = "0025299a9c892410a8bb3d04ee1eb564cc420e85" |
|
HEAD = "32f0b7187fd9b927bdc2efde529dfeb96fe18203" |
|
|
|
WORKTREES = { |
|
"Before": OUT / "worktrees" / "baseline", |
|
"After": OUT / "worktrees" / "head", |
|
} |
|
|
|
# `ops` changed source between these commits; exclude it so every timed crate is identical. |
|
TESTS = [ |
|
"binsearch", |
|
"closures", |
|
"collatz", |
|
"enums", |
|
"fibonacci", |
|
"fn_pointers", |
|
"generics", |
|
"impl", |
|
"is_even_plus_one", |
|
"modules", |
|
"mutable_borrow", |
|
"panic", |
|
"primes", |
|
"rsa", |
|
"structs", |
|
"traits", |
|
"unions", |
|
] |
|
|
|
|
|
@dataclass(frozen=True) |
|
class Measurement: |
|
variant: str |
|
test: str |
|
run: int |
|
seconds: float |
|
|
|
|
|
def run_command(cmd: list[str], cwd: Path, *, check: bool = True) -> subprocess.CompletedProcess: |
|
return subprocess.run( |
|
cmd, |
|
cwd=cwd, |
|
stdout=subprocess.PIPE, |
|
stderr=subprocess.PIPE, |
|
text=True, |
|
encoding="utf-8", |
|
errors="replace", |
|
check=check, |
|
) |
|
|
|
|
|
def time_clean_build(test_dir: Path) -> float: |
|
clean = run_command(["cargo", "clean"], test_dir, check=False) |
|
if clean.returncode != 0: |
|
raise RuntimeError(f"cargo clean failed in {test_dir}\n{clean.stderr}") |
|
|
|
env = os.environ.copy() |
|
env["CARGO_TERM_COLOR"] = "never" |
|
env["CARGO_INCREMENTAL"] = "0" |
|
|
|
start = time.perf_counter() |
|
build = subprocess.run( |
|
["cargo", "build"], |
|
cwd=test_dir, |
|
stdout=subprocess.PIPE, |
|
stderr=subprocess.PIPE, |
|
text=True, |
|
encoding="utf-8", |
|
errors="replace", |
|
env=env, |
|
) |
|
elapsed = time.perf_counter() - start |
|
if build.returncode != 0: |
|
fail = RESULTS / f"failed-{test_dir.parent.parent.parent.name}-{test_dir.name}.log" |
|
fail.write_text(f"STDOUT:\n{build.stdout}\n\nSTDERR:\n{build.stderr}") |
|
raise RuntimeError(f"cargo build failed in {test_dir}; wrote {fail}") |
|
return elapsed |
|
|
|
|
|
def collect(runs: int) -> list[Measurement]: |
|
RESULTS.mkdir(parents=True, exist_ok=True) |
|
jobs = [ |
|
(variant, test, run) |
|
for run in range(1, runs + 1) |
|
for test in TESTS |
|
for variant in WORKTREES |
|
] |
|
random.Random(20260706).shuffle(jobs) |
|
|
|
measurements: list[Measurement] = [] |
|
total = len(jobs) |
|
for index, (variant, test, run) in enumerate(jobs, 1): |
|
test_dir = WORKTREES[variant] / "tests" / "binary" / test |
|
print(f"[{index:03}/{total}] {variant:6} run {run}/{runs} {test}", flush=True) |
|
seconds = time_clean_build(test_dir) |
|
measurements.append(Measurement(variant, test, run, seconds)) |
|
print(f" {seconds:.3f}s", flush=True) |
|
return measurements |
|
|
|
|
|
def summarize(measurements: list[Measurement]) -> dict: |
|
by_variant_test: dict[tuple[str, str], list[float]] = {} |
|
for measurement in measurements: |
|
by_variant_test.setdefault((measurement.variant, measurement.test), []).append( |
|
measurement.seconds |
|
) |
|
|
|
rows = [] |
|
for test in TESTS: |
|
before = statistics.median(by_variant_test[("Before", test)]) |
|
after = statistics.median(by_variant_test[("After", test)]) |
|
rows.append( |
|
{ |
|
"test": test, |
|
"before_median_s": before, |
|
"after_median_s": after, |
|
"speedup": before / after, |
|
} |
|
) |
|
|
|
total_before = sum(row["before_median_s"] for row in rows) |
|
total_after = sum(row["after_median_s"] for row in rows) |
|
geo_speedup = statistics.geometric_mean(row["speedup"] for row in rows) |
|
return { |
|
"baseline_commit": BASELINE, |
|
"head_commit": HEAD, |
|
"runs_per_test": len({m.run for m in measurements}), |
|
"tests": rows, |
|
"total_before_median_s": total_before, |
|
"total_after_median_s": total_after, |
|
"total_speedup": total_before / total_after, |
|
"geomean_speedup": geo_speedup, |
|
} |
|
|
|
|
|
def write_results(measurements: list[Measurement], summary: dict) -> None: |
|
RESULTS.mkdir(parents=True, exist_ok=True) |
|
raw_path = RESULTS / "compile-improvement-raw.csv" |
|
with raw_path.open("w", newline="") as f: |
|
writer = csv.DictWriter(f, ["variant", "test", "run", "seconds"]) |
|
writer.writeheader() |
|
for m in measurements: |
|
writer.writerow( |
|
{ |
|
"variant": m.variant, |
|
"test": m.test, |
|
"run": m.run, |
|
"seconds": f"{m.seconds:.9f}", |
|
} |
|
) |
|
|
|
summary_path = RESULTS / "compile-improvement-summary.json" |
|
summary_path.write_text(json.dumps(summary, indent=2)) |
|
|
|
table_path = RESULTS / "compile-improvement-summary.csv" |
|
with table_path.open("w", newline="") as f: |
|
writer = csv.DictWriter( |
|
f, ["test", "before_median_s", "after_median_s", "speedup"] |
|
) |
|
writer.writeheader() |
|
for row in summary["tests"]: |
|
writer.writerow(row) |
|
|
|
|
|
def nice_test_name(name: str) -> str: |
|
return name.replace("_", " ") |
|
|
|
|
|
def plot(summary: dict) -> None: |
|
FIGURES.mkdir(parents=True, exist_ok=True) |
|
rows = sorted(summary["tests"], key=lambda row: row["test"]) |
|
|
|
def seconds_text(seconds: float) -> str: |
|
if seconds >= 100: |
|
return f"{seconds:.0f}s" |
|
if seconds >= 10: |
|
return f"{seconds:.1f}s" |
|
return f"{seconds:.2f}s" |
|
|
|
plt.rcParams.update( |
|
{ |
|
"font.family": "DejaVu Sans", |
|
"axes.titleweight": "bold", |
|
"axes.labelsize": 12, |
|
"xtick.labelsize": 10, |
|
"ytick.labelsize": 10, |
|
} |
|
) |
|
|
|
fig = plt.figure(figsize=(10, 14), dpi=180) |
|
bg_color = "#f6f8fc" |
|
panel_color = "#ffffff" |
|
header_color = "#e8eef7" |
|
stripe_color = "#f3f6fa" |
|
total_color = "#eaf2ff" |
|
grid_color = "#d7e0ec" |
|
old_color = "#d97706" |
|
new_color = "#2563eb" |
|
speedup_color = "#1d4ed8" |
|
text_color = "#172033" |
|
muted_text = "#64748b" |
|
fig.patch.set_facecolor(bg_color) |
|
|
|
table_rows = [ |
|
{ |
|
"test": "Total", |
|
"before_median_s": summary["total_before_median_s"], |
|
"after_median_s": summary["total_after_median_s"], |
|
"speedup": summary["total_speedup"], |
|
"is_total": True, |
|
}, |
|
*({**row, "is_total": False} for row in rows), |
|
] |
|
|
|
bar_ax = fig.add_axes([0.130, 0.720, 0.800, 0.165], zorder=2) |
|
bar_ax.set_facecolor(panel_color) |
|
totals = [summary["total_before_median_s"], summary["total_after_median_s"]] |
|
bar_ax.barh([1, 0], totals, color=[old_color, new_color], height=0.46) |
|
bar_ax.set_xlim(0, summary["total_before_median_s"] * 1.13) |
|
bar_ax.set_yticks([1, 0]) |
|
bar_ax.set_yticklabels(["Before", "After"], color=text_color, fontsize=17, weight="bold") |
|
bar_ax.set_title("Rust to JVM compile time (testsuite)", fontsize=27, color=text_color, weight="bold", pad=18) |
|
bar_ax.set_xlabel("seconds, lower is better", color=muted_text, fontsize=13, weight="bold", labelpad=8) |
|
bar_ax.grid(axis="x", color=grid_color, linewidth=1.1) |
|
bar_ax.set_axisbelow(True) |
|
bar_ax.xaxis.set_major_formatter(mticker.FuncFormatter(lambda value, _: f"{value:g}")) |
|
bar_ax.tick_params(axis="x", colors=muted_text, labelsize=12) |
|
bar_ax.tick_params(axis="y", length=0, pad=8) |
|
for spine in bar_ax.spines.values(): |
|
spine.set_color("#dbe4ef") |
|
bar_ax.text( |
|
summary["total_before_median_s"] * 1.015, |
|
1, |
|
seconds_text(summary["total_before_median_s"]), |
|
va="center", |
|
ha="left", |
|
color=text_color, |
|
fontsize=16, |
|
weight="bold", |
|
) |
|
bar_ax.text( |
|
max(summary["total_after_median_s"] * 2.05, summary["total_before_median_s"] * 0.085), |
|
0, |
|
seconds_text(summary["total_after_median_s"]), |
|
va="center", |
|
ha="left", |
|
color=text_color, |
|
fontsize=16, |
|
weight="bold", |
|
) |
|
|
|
legend_y = 0.660 |
|
fig.text(0.260, legend_y, "●", fontsize=24, color=old_color, va="center") |
|
fig.text( |
|
0.295, |
|
legend_y, |
|
f"Before ({BASELINE[:7]})", |
|
fontsize=15, |
|
color=text_color, |
|
va="center", |
|
weight="bold", |
|
) |
|
fig.text(0.560, legend_y, "●", fontsize=24, color=new_color, va="center") |
|
fig.text( |
|
0.595, |
|
legend_y, |
|
f"After ({HEAD[:7]})", |
|
fontsize=15, |
|
color=text_color, |
|
va="center", |
|
weight="bold", |
|
) |
|
|
|
table_ax = fig.add_axes([0.070, 0.070, 0.860, 0.545], zorder=2) |
|
table_ax.set_axis_off() |
|
table_ax.set_facecolor(panel_color) |
|
table_ax.add_patch( |
|
plt.Rectangle( |
|
(0, 0), |
|
1, |
|
1, |
|
transform=table_ax.transAxes, |
|
facecolor=panel_color, |
|
edgecolor="#dbe4ef", |
|
linewidth=1.5, |
|
) |
|
) |
|
|
|
col_test = 0.045 |
|
col_before = 0.525 |
|
col_after = 0.705 |
|
col_speedup = 0.945 |
|
row_h = 1.0 / (len(table_rows) + 1) |
|
|
|
table_ax.add_patch( |
|
plt.Rectangle((0, 1 - row_h), 1, row_h, transform=table_ax.transAxes, facecolor=header_color, edgecolor="none") |
|
) |
|
headers = [ |
|
("Test", col_test, "left"), |
|
("Before", col_before, "right"), |
|
("After", col_after, "right"), |
|
("Speedup", col_speedup, "right"), |
|
] |
|
for label, x, ha in headers: |
|
table_ax.text( |
|
x, |
|
1 - row_h / 2, |
|
label, |
|
transform=table_ax.transAxes, |
|
ha=ha, |
|
va="center", |
|
fontsize=14, |
|
weight="bold", |
|
color=text_color, |
|
) |
|
|
|
for idx, row in enumerate(table_rows): |
|
y0 = 1 - row_h * (idx + 2) |
|
bg = total_color if row["is_total"] else (stripe_color if idx % 2 == 0 else panel_color) |
|
table_ax.add_patch( |
|
plt.Rectangle((0, y0), 1, row_h, transform=table_ax.transAxes, facecolor=bg, edgecolor="none") |
|
) |
|
weight = "bold" if row["is_total"] else "normal" |
|
size = 13.5 if row["is_total"] else 12.3 |
|
table_ax.text( |
|
col_test, |
|
y0 + row_h / 2, |
|
nice_test_name(row["test"]), |
|
transform=table_ax.transAxes, |
|
ha="left", |
|
va="center", |
|
fontsize=size, |
|
weight=weight, |
|
color=text_color, |
|
) |
|
table_ax.text( |
|
col_before, |
|
y0 + row_h / 2, |
|
seconds_text(row["before_median_s"]), |
|
transform=table_ax.transAxes, |
|
ha="right", |
|
va="center", |
|
fontsize=size - 0.2, |
|
weight=weight, |
|
color=muted_text, |
|
) |
|
table_ax.text( |
|
col_after, |
|
y0 + row_h / 2, |
|
seconds_text(row["after_median_s"]), |
|
transform=table_ax.transAxes, |
|
ha="right", |
|
va="center", |
|
fontsize=size - 0.2, |
|
weight=weight, |
|
color=muted_text, |
|
) |
|
table_ax.text( |
|
col_speedup, |
|
y0 + row_h / 2, |
|
f"{row['speedup']:.1f}x", |
|
transform=table_ax.transAxes, |
|
ha="right", |
|
va="center", |
|
fontsize=size + 0.8, |
|
weight="bold", |
|
color=speedup_color, |
|
) |
|
|
|
for x in (0.420, 0.600, 0.780): |
|
table_ax.plot([x, x], [0, 1], transform=table_ax.transAxes, color="#dbe4ef", linewidth=1.0) |
|
|
|
table_ax.plot([0, 1], [1 - row_h, 1 - row_h], transform=table_ax.transAxes, color="#cbd5e1", linewidth=1.5) |
|
|
|
png = FIGURES / "compiler-performance-improvement.png" |
|
svg = FIGURES / "compiler-performance-improvement.svg" |
|
thumbnail_png = FIGURES / "compiler-performance-thumbnail.png" |
|
thumbnail_svg = FIGURES / "compiler-performance-thumbnail.svg" |
|
for path in (png, svg, thumbnail_png, thumbnail_svg): |
|
fig.savefig(path, facecolor=fig.get_facecolor(), pad_inches=0) |
|
print(f"Wrote {path}") |
|
|
|
|
|
def main() -> int: |
|
if len(sys.argv) > 1 and sys.argv[1] == "plot": |
|
summary = json.loads((RESULTS / "compile-improvement-summary.json").read_text()) |
|
plot(summary) |
|
return 0 |
|
|
|
runs = int(sys.argv[1]) if len(sys.argv) > 1 else 5 |
|
measurements = collect(runs) |
|
summary = summarize(measurements) |
|
write_results(measurements, summary) |
|
plot(summary) |
|
print( |
|
f"Total median clean build time: " |
|
f"{summary['total_before_median_s']:.3f}s -> " |
|
f"{summary['total_after_median_s']:.3f}s " |
|
f"({summary['total_speedup']:.2f}x faster)" |
|
) |
|
return 0 |
|
|
|
|
|
if __name__ == "__main__": |
|
raise SystemExit(main()) |
