From bc8bebe78358e3a37bad0867b2d94955cac01d47 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:45:39 +0200 Subject: bench: add e2e benchmarks + new API compat fixes --- tools/benchstat.py | 127 ----------------------------------------------------- 1 file changed, 127 deletions(-) delete mode 100755 tools/benchstat.py (limited to 'tools') diff --git a/tools/benchstat.py b/tools/benchstat.py deleted file mode 100755 index 7f8b695..0000000 --- a/tools/benchstat.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -"""Compare two benchmark records produced by make bench or make bench-model -(JSON written by bench/bench.h --json). - -usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0] - -Per benchmark: median ns/op of both sample sets, the delta, and a -Mann-Whitney U significance test. A delta only counts when it is BOTH -significant at --alpha AND at least --threshold percent: back-to-back -runs of an unchanged binary drift a couple of percent with machine state -(thermals, core scheduling), so smaller shifts are below the -reproducibility floor even when statistically clean. Everything else -prints as "~" - treat it as noise, not improvement or regression. -Exits 1 if any benchmark regresses past both bars. -""" - -from __future__ import annotations - -import argparse -import json -import math -import sys -from pathlib import Path - - -def load(path: str) -> dict: - with open(path) as f: - data = json.load(f) - return {b["name"]: b["ns_per_op"] for b in data["benchmarks"]} - - -def median(v: list[float]) -> float: - s = sorted(v) - n = len(s) - return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2]) - - -def mann_whitney_p(a: list[float], b: list[float]) -> float: - """Two-sided p-value via the normal approximation with tie correction. - Fine for the sample counts bench.h produces (default 10 per side).""" - n1, n2 = len(a), len(b) - pooled = sorted((x, 0) for x in a) + sorted((x, 1) for x in b) - pooled.sort(key=lambda t: t[0]) - - ranks: list[float] = [0.0] * len(pooled) - tie_term = 0.0 - i = 0 - while i < len(pooled): - j = i - while j < len(pooled) and pooled[j][0] == pooled[i][0]: - j += 1 - rank = (i + j + 1) / 2.0 # average rank for the tie group (1-based) - for k in range(i, j): - ranks[k] = rank - t = j - i - tie_term += t * t * t - t - i = j - - r1 = sum(r for r, (_, grp) in zip(ranks, pooled) if grp == 0) - u1 = r1 - n1 * (n1 + 1) / 2.0 - mu = n1 * n2 / 2.0 - n = n1 + n2 - sigma_sq = n1 * n2 / 12.0 * ((n + 1) - tie_term / (n * (n - 1))) - if sigma_sq <= 0: - return 1.0 # all values identical - z = (u1 - mu - (0.5 if u1 > mu else -0.5 if u1 < mu else 0.0)) \ - / math.sqrt(sigma_sq) - return math.erfc(abs(z) / math.sqrt(2.0)) - - -def fmt_ns(ns: float) -> str: - if ns < 1e3: - return f"{ns:.1f}ns" - if ns < 1e6: - return f"{ns / 1e3:.2f}µs" - if ns < 1e9: - return f"{ns / 1e6:.2f}ms" - return f"{ns / 1e9:.3f}s" - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("old") - ap.add_argument("new") - ap.add_argument("--alpha", type=float, default=0.05) - ap.add_argument("--threshold", type=float, default=3.0, - help="minimum |delta| percent to report (default 3)") - args = ap.parse_args() - - old, new = load(args.old), load(args.new) - common = [k for k in old if k in new] - if not common: - print("no common benchmarks", file=sys.stderr) - return 2 - - width = max(len(k) for k in common) - print(f"{'benchmark':<{width}} {'old':>10} {'new':>10} delta") - regressions = [] - for name in common: - mo, mn = median(old[name]), median(new[name]) - p = mann_whitney_p(old[name], new[name]) - delta = (mn - mo) / mo * 100.0 - if p < args.alpha and abs(delta) >= args.threshold: - mark = f"{delta:+.1f}% (p={p:.3f})" - if delta > 0: - regressions.append((name, delta)) - else: - mark = "~" - print(f"{name:<{width}} {fmt_ns(mo):>10} {fmt_ns(mn):>10} {mark}") - - for k in old: - if k not in new: - print(f"only in {Path(args.old).name}: {k}") - for k in new: - if k not in old: - print(f"only in {Path(args.new).name}: {k}") - - if regressions: - worst = max(regressions, key=lambda r: r[1]) - print(f"\n{len(regressions)} significant regression(s); " - f"worst: {worst[0]} {worst[1]:+.1f}%", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) -- cgit v1.3.1