aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rwxr-xr-xtools/benchstat.py127
1 files changed, 0 insertions, 127 deletions
diff --git a/tools/benchstat.py b/tools/benchstat.py
deleted file mode 100755
index 7f8b695..0000000
--- a/tools/benchstat.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-"""Compare two benchmark records produced by make bench or make bench-model
-(JSON written by bench/bench.h --json).
-
-usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0]
-
-Per benchmark: median ns/op of both sample sets, the delta, and a
-Mann-Whitney U significance test. A delta only counts when it is BOTH
-significant at --alpha AND at least --threshold percent: back-to-back
-runs of an unchanged binary drift a couple of percent with machine state
-(thermals, core scheduling), so smaller shifts are below the
-reproducibility floor even when statistically clean. Everything else
-prints as "~" - treat it as noise, not improvement or regression.
-Exits 1 if any benchmark regresses past both bars.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import sys
-from pathlib import Path
-
-
-def load(path: str) -> dict:
- with open(path) as f:
- data = json.load(f)
- return {b["name"]: b["ns_per_op"] for b in data["benchmarks"]}
-
-
-def median(v: list[float]) -> float:
- s = sorted(v)
- n = len(s)
- return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
-
-
-def mann_whitney_p(a: list[float], b: list[float]) -> float:
- """Two-sided p-value via the normal approximation with tie correction.
- Fine for the sample counts bench.h produces (default 10 per side)."""
- n1, n2 = len(a), len(b)
- pooled = sorted((x, 0) for x in a) + sorted((x, 1) for x in b)
- pooled.sort(key=lambda t: t[0])
-
- ranks: list[float] = [0.0] * len(pooled)
- tie_term = 0.0
- i = 0
- while i < len(pooled):
- j = i
- while j < len(pooled) and pooled[j][0] == pooled[i][0]:
- j += 1
- rank = (i + j + 1) / 2.0 # average rank for the tie group (1-based)
- for k in range(i, j):
- ranks[k] = rank
- t = j - i
- tie_term += t * t * t - t
- i = j
-
- r1 = sum(r for r, (_, grp) in zip(ranks, pooled) if grp == 0)
- u1 = r1 - n1 * (n1 + 1) / 2.0
- mu = n1 * n2 / 2.0
- n = n1 + n2
- sigma_sq = n1 * n2 / 12.0 * ((n + 1) - tie_term / (n * (n - 1)))
- if sigma_sq <= 0:
- return 1.0 # all values identical
- z = (u1 - mu - (0.5 if u1 > mu else -0.5 if u1 < mu else 0.0)) \
- / math.sqrt(sigma_sq)
- return math.erfc(abs(z) / math.sqrt(2.0))
-
-
-def fmt_ns(ns: float) -> str:
- if ns < 1e3:
- return f"{ns:.1f}ns"
- if ns < 1e6:
- return f"{ns / 1e3:.2f}µs"
- if ns < 1e9:
- return f"{ns / 1e6:.2f}ms"
- return f"{ns / 1e9:.3f}s"
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("old")
- ap.add_argument("new")
- ap.add_argument("--alpha", type=float, default=0.05)
- ap.add_argument("--threshold", type=float, default=3.0,
- help="minimum |delta| percent to report (default 3)")
- args = ap.parse_args()
-
- old, new = load(args.old), load(args.new)
- common = [k for k in old if k in new]
- if not common:
- print("no common benchmarks", file=sys.stderr)
- return 2
-
- width = max(len(k) for k in common)
- print(f"{'benchmark':<{width}} {'old':>10} {'new':>10} delta")
- regressions = []
- for name in common:
- mo, mn = median(old[name]), median(new[name])
- p = mann_whitney_p(old[name], new[name])
- delta = (mn - mo) / mo * 100.0
- if p < args.alpha and abs(delta) >= args.threshold:
- mark = f"{delta:+.1f}% (p={p:.3f})"
- if delta > 0:
- regressions.append((name, delta))
- else:
- mark = "~"
- print(f"{name:<{width}} {fmt_ns(mo):>10} {fmt_ns(mn):>10} {mark}")
-
- for k in old:
- if k not in new:
- print(f"only in {Path(args.old).name}: {k}")
- for k in new:
- if k not in old:
- print(f"only in {Path(args.new).name}: {k}")
-
- if regressions:
- worst = max(regressions, key=lambda r: r[1])
- print(f"\n{len(regressions)} significant regression(s); "
- f"worst: {worst[0]} {worst[1]:+.1f}%", file=sys.stderr)
- return 1
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())