aboutsummaryrefslogtreecommitdiff
path: root/bench/benchstat.py
diff options
context:
space:
mode:
Diffstat (limited to 'bench/benchstat.py')
-rwxr-xr-xbench/benchstat.py127
1 files changed, 127 insertions, 0 deletions
diff --git a/bench/benchstat.py b/bench/benchstat.py
new file mode 100755
index 0000000..7f8b695
--- /dev/null
+++ b/bench/benchstat.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Compare two benchmark records produced by make bench or make bench-model
+(JSON written by bench/bench.h --json).
+
+usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0]
+
+Per benchmark: median ns/op of both sample sets, the delta, and a
+Mann-Whitney U significance test. A delta only counts when it is BOTH
+significant at --alpha AND at least --threshold percent: back-to-back
+runs of an unchanged binary drift a couple of percent with machine state
+(thermals, core scheduling), so smaller shifts are below the
+reproducibility floor even when statistically clean. Everything else
+prints as "~" - treat it as noise, not improvement or regression.
+Exits 1 if any benchmark regresses past both bars.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+
+def load(path: str) -> dict:
+ with open(path) as f:
+ data = json.load(f)
+ return {b["name"]: b["ns_per_op"] for b in data["benchmarks"]}
+
+
+def median(v: list[float]) -> float:
+ s = sorted(v)
+ n = len(s)
+ return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+
+
+def mann_whitney_p(a: list[float], b: list[float]) -> float:
+ """Two-sided p-value via the normal approximation with tie correction.
+ Fine for the sample counts bench.h produces (default 10 per side)."""
+ n1, n2 = len(a), len(b)
+ pooled = sorted((x, 0) for x in a) + sorted((x, 1) for x in b)
+ pooled.sort(key=lambda t: t[0])
+
+ ranks: list[float] = [0.0] * len(pooled)
+ tie_term = 0.0
+ i = 0
+ while i < len(pooled):
+ j = i
+ while j < len(pooled) and pooled[j][0] == pooled[i][0]:
+ j += 1
+ rank = (i + j + 1) / 2.0 # average rank for the tie group (1-based)
+ for k in range(i, j):
+ ranks[k] = rank
+ t = j - i
+ tie_term += t * t * t - t
+ i = j
+
+ r1 = sum(r for r, (_, grp) in zip(ranks, pooled) if grp == 0)
+ u1 = r1 - n1 * (n1 + 1) / 2.0
+ mu = n1 * n2 / 2.0
+ n = n1 + n2
+ sigma_sq = n1 * n2 / 12.0 * ((n + 1) - tie_term / (n * (n - 1)))
+ if sigma_sq <= 0:
+ return 1.0 # all values identical
+ z = (u1 - mu - (0.5 if u1 > mu else -0.5 if u1 < mu else 0.0)) \
+ / math.sqrt(sigma_sq)
+ return math.erfc(abs(z) / math.sqrt(2.0))
+
+
+def fmt_ns(ns: float) -> str:
+ if ns < 1e3:
+ return f"{ns:.1f}ns"
+ if ns < 1e6:
+ return f"{ns / 1e3:.2f}µs"
+ if ns < 1e9:
+ return f"{ns / 1e6:.2f}ms"
+ return f"{ns / 1e9:.3f}s"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("old")
+ ap.add_argument("new")
+ ap.add_argument("--alpha", type=float, default=0.05)
+ ap.add_argument("--threshold", type=float, default=3.0,
+ help="minimum |delta| percent to report (default 3)")
+ args = ap.parse_args()
+
+ old, new = load(args.old), load(args.new)
+ common = [k for k in old if k in new]
+ if not common:
+ print("no common benchmarks", file=sys.stderr)
+ return 2
+
+ width = max(len(k) for k in common)
+ print(f"{'benchmark':<{width}} {'old':>10} {'new':>10} delta")
+ regressions = []
+ for name in common:
+ mo, mn = median(old[name]), median(new[name])
+ p = mann_whitney_p(old[name], new[name])
+ delta = (mn - mo) / mo * 100.0
+ if p < args.alpha and abs(delta) >= args.threshold:
+ mark = f"{delta:+.1f}% (p={p:.3f})"
+ if delta > 0:
+ regressions.append((name, delta))
+ else:
+ mark = "~"
+ print(f"{name:<{width}} {fmt_ns(mo):>10} {fmt_ns(mn):>10} {mark}")
+
+ for k in old:
+ if k not in new:
+ print(f"only in {Path(args.old).name}: {k}")
+ for k in new:
+ if k not in old:
+ print(f"only in {Path(args.new).name}: {k}")
+
+ if regressions:
+ worst = max(regressions, key=lambda r: r[1])
+ print(f"\n{len(regressions)} significant regression(s); "
+ f"worst: {worst[0]} {worst[1]:+.1f}%", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())