aboutsummaryrefslogtreecommitdiff
path: root/tests/check_cuda_fast_gemm_parity.py
blob: bae4b13d4ee0c9b4dfec599b1f54ea4018a9a763 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
#!/usr/bin/env python3
"""Check CUDA reduced-precision quality against exact CUDA F32.

Covers both reduced-precision axes: compute mode (--mode tf32|bf16|16f,
selecting --gpu-gemm-mode) and weight storage (--weights bf16, selecting
--gpu-weight-dtype). The reference run always uses exact F32 weights and compute.
For BF16 weight storage the int8 byte-change rate is inherently large
(~7-15%) and informational only; cosine is the quality gate, so the int8
threshold defaults to 1.0 in that case unless overridden."""

from __future__ import annotations

import argparse
import json
import math
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "ffwd-cli"


_PASSAGE = (
    "document: Embedding models turn text into dense vectors for semantic "
    "retrieval. Long document inference stresses bidirectional attention "
    "because every token attends to every other token in the same packed "
    "sequence. "
)

DEFAULT_TEXTS = [
    "query: what is the capital of France?",
    "document: Paris is the capital of France.",
    "document: Berlin is the capital of Germany.",
    _PASSAGE * 4,
    _PASSAGE * 5,
    _PASSAGE * 6,
    _PASSAGE * 7,
    _PASSAGE * 8,
]
# COVERAGE: the batch total here clears ~1300 tokens, ABOVE the 1024-token
# threshold that gates the CUDA reduced-precision stack (bf16 activations,
# cuBLASLt fused FFN epilogue, QKV16, direct-K/V). That threshold matters: a
# cuBLASLt FFN gate-up aliasing bug shipped undetected precisely because the old
# defaults summed to <1024 tokens, so the reduced path never engaged here. Keep
# the batch >=1024 tokens. Each entry still stays under the 512-token position
# limit of short-context encoders (BERT/MiniLM/BGE max_position 512, XLM-R 514);
# larger single inputs overflow their position tables and are (correctly)
# rejected. The entries are deliberately ragged (different lengths) to exercise
# the per-sequence attention path; to also hit the equal-length batched-attention
# path (cublasGemmBatchedEx), pass >=2 identical texts as positional args.


def run_cuda(binary: str, model_dir: str, mode: str, weights: str,
             batch_size: int, texts: list[str]) -> list[list[float]]:
    cmd = [
        binary,
        "-d", model_dir,
        "--gpu-gemm-mode", mode,
        "--gpu-weight-dtype", weights,
        "--stream",
        "-b", str(batch_size),
    ]
    proc = subprocess.run(
        cmd,
        input="\n".join(texts) + "\n",
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        check=False,
    )
    if proc.returncode != 0:
        raise RuntimeError(
            f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
        )

    rows: list[list[float]] = []
    for line in proc.stdout.splitlines():
        if not line.strip():
            continue
        row = json.loads(line)
        if "error" in row:
            raise RuntimeError(f"stdin returned error: {row}")
        rows.append([float(x) for x in row["embedding"]])
    if len(rows) != len(texts):
        raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
    return rows


def cosine(a: list[float], b: list[float]) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    return dot / (na * nb)


def quantize_int8_tanh(x: float) -> int:
    v = round(math.tanh(x) * 127.0)
    if v < -128:
        return -128
    if v > 127:
        return 127
    return int(v)


def compare(name: str, ref: list[list[float]], got: list[list[float]]) -> tuple[float, float, int]:
    worst_diff = 0.0
    worst_cos = 1.0
    int8_changes = 0
    for i, (a, b) in enumerate(zip(ref, got)):
        if len(a) != len(b):
            raise RuntimeError(f"dimension mismatch at row {i}: {len(a)} vs {len(b)}")
        max_diff = max(abs(x - y) for x, y in zip(a, b))
        cos = cosine(a, b)
        changes = sum(
            quantize_int8_tanh(x) != quantize_int8_tanh(y)
            for x, y in zip(a, b)
        )
        worst_diff = max(worst_diff, max_diff)
        worst_cos = min(worst_cos, cos)
        int8_changes += changes
        print(
            f"{name} [{i}] cosine={cos:.8f} "
            f"max_abs_diff={max_diff:.8g} int8_changes={changes}"
        )
    return worst_diff, worst_cos, int8_changes


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--binary", default=str(DEFAULT_BINARY))
    ap.add_argument("--model-dir", required=True)
    ap.add_argument("--mode", choices=("f32", "tf32", "bf16", "16f"),
                    default="tf32")
    ap.add_argument("--weights", choices=("f32", "bf16", "f16"), default="f32")
    ap.add_argument("--batch-size", type=int, default=8)
    ap.add_argument("--max-diff", type=float, default=None)
    ap.add_argument("--min-cosine", type=float, default=0.999)
    ap.add_argument("--max-int8-change-rate", type=float, default=None)
    # The reduced-precision path must be DETERMINISTIC: re-run the fast batch and
    # require the runs to agree. A self-aliasing or unsynchronized GPU kernel
    # produces correct-looking averages but different output per run; that is
    # invisible to a single quality-vs-f32 comparison, so it gets its own gate.
    ap.add_argument("--determinism-runs", type=int, default=3,
                    help="repeat the fast batch this many times and require agreement")
    ap.add_argument("--determinism-min-cosine", type=float, default=0.99999)
    ap.add_argument("texts", nargs="*")
    args = ap.parse_args()

    if args.batch_size < 2:
        raise SystemExit("--batch-size must be >= 2")
    if args.mode == "f32" and args.weights == "f32":
        raise SystemExit("nothing to check: --mode and --weights are both f32")
    if args.max_diff is None:
        args.max_diff = 0.05 if args.weights == "bf16" else 0.01
    if args.max_int8_change_rate is None:
        args.max_int8_change_rate = 1.0 if args.weights == "bf16" else 0.02

    texts = args.texts or DEFAULT_TEXTS
    ref_batch = run_cuda(args.binary, args.model_dir, "f32", "f32",
                         args.batch_size, texts)
    fast_batch = run_cuda(args.binary, args.model_dir, args.mode,
                          args.weights, args.batch_size, texts)
    fast_single = run_cuda(args.binary, args.model_dir, args.mode,
                           args.weights, 1, texts)

    qdiff, qcos, qchanges = compare("quality", ref_batch, fast_batch)
    sdiff, scos, schanges = compare("shape", fast_single, fast_batch)

    # Determinism: re-run the fast batch and compare each repeat to the first.
    # The min cosine across all repeats and rows must stay essentially 1.0.
    det_cos = 1.0
    for r in range(1, max(args.determinism_runs, 1)):
        repeat = run_cuda(args.binary, args.model_dir, args.mode, args.weights,
                          args.batch_size, texts)
        _, rcos, _ = compare(f"determinism[{r}]", fast_batch, repeat)
        det_cos = min(det_cos, rcos)

    total_values = len(texts) * len(ref_batch[0])
    qrate = qchanges / total_values
    srate = schanges / total_values
    print(
        f"ok: mode={args.mode} weights={args.weights} "
        f"batch_size={args.batch_size} "
        f"quality_diff={qdiff:.8g} quality_cos={qcos:.8f} "
        f"quality_int8_change_rate={qrate:.6f} "
        f"shape_diff={sdiff:.8g} shape_cos={scos:.8f} "
        f"shape_int8_change_rate={srate:.6f} "
        f"determinism_runs={args.determinism_runs} determinism_cos={det_cos:.8f}"
    )

    failed = (
        qdiff > args.max_diff or qcos < args.min_cosine or
        sdiff > args.max_diff or scos < args.min_cosine or
        qrate > args.max_int8_change_rate or
        srate > args.max_int8_change_rate or
        det_cos < args.determinism_min_cosine
    )
    if failed:
        raise SystemExit(
            f"cuda parity failed (mode={args.mode} weights={args.weights}): "
            f"quality diff={qdiff:g} cos={qcos:g} int8_rate={qrate:g}; "
            f"shape diff={sdiff:g} cos={scos:g} int8_rate={srate:g}; "
            f"determinism cos={det_cos:g} (min {args.determinism_min_cosine:g})"
        )
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except Exception as exc:
        print(f"error: {exc}", file=sys.stderr)
        raise SystemExit(1)