aboutsummaryrefslogtreecommitdiff
path: root/bench/benchstat.py
blob: 7f8b6956dd15e8c1dac2dc49cd272f620e400e0c (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
#!/usr/bin/env python3
"""Compare two benchmark records produced by make bench or make bench-model
(JSON written by bench/bench.h --json).

usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0]

Per benchmark: median ns/op of both sample sets, the delta, and a
Mann-Whitney U significance test. A delta only counts when it is BOTH
significant at --alpha AND at least --threshold percent: back-to-back
runs of an unchanged binary drift a couple of percent with machine state
(thermals, core scheduling), so smaller shifts are below the
reproducibility floor even when statistically clean. Everything else
prints as "~" - treat it as noise, not improvement or regression.
Exits 1 if any benchmark regresses past both bars.
"""

from __future__ import annotations

import argparse
import json
import math
import sys
from pathlib import Path


def load(path: str) -> dict:
    with open(path) as f:
        data = json.load(f)
    return {b["name"]: b["ns_per_op"] for b in data["benchmarks"]}


def median(v: list[float]) -> float:
    s = sorted(v)
    n = len(s)
    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])


def mann_whitney_p(a: list[float], b: list[float]) -> float:
    """Two-sided p-value via the normal approximation with tie correction.
    Fine for the sample counts bench.h produces (default 10 per side)."""
    n1, n2 = len(a), len(b)
    pooled = sorted((x, 0) for x in a) + sorted((x, 1) for x in b)
    pooled.sort(key=lambda t: t[0])

    ranks: list[float] = [0.0] * len(pooled)
    tie_term = 0.0
    i = 0
    while i < len(pooled):
        j = i
        while j < len(pooled) and pooled[j][0] == pooled[i][0]:
            j += 1
        rank = (i + j + 1) / 2.0  # average rank for the tie group (1-based)
        for k in range(i, j):
            ranks[k] = rank
        t = j - i
        tie_term += t * t * t - t
        i = j

    r1 = sum(r for r, (_, grp) in zip(ranks, pooled) if grp == 0)
    u1 = r1 - n1 * (n1 + 1) / 2.0
    mu = n1 * n2 / 2.0
    n = n1 + n2
    sigma_sq = n1 * n2 / 12.0 * ((n + 1) - tie_term / (n * (n - 1)))
    if sigma_sq <= 0:
        return 1.0  # all values identical
    z = (u1 - mu - (0.5 if u1 > mu else -0.5 if u1 < mu else 0.0)) \
        / math.sqrt(sigma_sq)
    return math.erfc(abs(z) / math.sqrt(2.0))


def fmt_ns(ns: float) -> str:
    if ns < 1e3:
        return f"{ns:.1f}ns"
    if ns < 1e6:
        return f"{ns / 1e3:.2f}µs"
    if ns < 1e9:
        return f"{ns / 1e6:.2f}ms"
    return f"{ns / 1e9:.3f}s"


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("old")
    ap.add_argument("new")
    ap.add_argument("--alpha", type=float, default=0.05)
    ap.add_argument("--threshold", type=float, default=3.0,
                    help="minimum |delta| percent to report (default 3)")
    args = ap.parse_args()

    old, new = load(args.old), load(args.new)
    common = [k for k in old if k in new]
    if not common:
        print("no common benchmarks", file=sys.stderr)
        return 2

    width = max(len(k) for k in common)
    print(f"{'benchmark':<{width}}  {'old':>10}  {'new':>10}  delta")
    regressions = []
    for name in common:
        mo, mn = median(old[name]), median(new[name])
        p = mann_whitney_p(old[name], new[name])
        delta = (mn - mo) / mo * 100.0
        if p < args.alpha and abs(delta) >= args.threshold:
            mark = f"{delta:+.1f}% (p={p:.3f})"
            if delta > 0:
                regressions.append((name, delta))
        else:
            mark = "~"
        print(f"{name:<{width}}  {fmt_ns(mo):>10}  {fmt_ns(mn):>10}  {mark}")

    for k in old:
        if k not in new:
            print(f"only in {Path(args.old).name}: {k}")
    for k in new:
        if k not in old:
            print(f"only in {Path(args.new).name}: {k}")

    if regressions:
        worst = max(regressions, key=lambda r: r[1])
        print(f"\n{len(regressions)} significant regression(s); "
              f"worst: {worst[0]} {worst[1]:+.1f}%", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())