diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 00:58:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 01:03:41 +0200 |
| commit | af55803d0dff9eb692c760c8d486e736c945061d (patch) | |
| tree | 0875a1112bf98304e445e0544c4309670d6f4769 /bench | |
| parent | 0f861b7f3098750ddee4f6570f3be228d1da5320 (diff) | |
| download | ffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip | |
chore: clean up comment formatting and wording
Diffstat (limited to 'bench')
| -rw-r--r-- | bench/bench.h | 2 | ||||
| -rw-r--r-- | bench/bench_kernels.c | 14 | ||||
| -rw-r--r-- | bench/bench_late_rerank.c | 10 | ||||
| -rwxr-xr-x | bench/bench_late_rerank.py | 2 | ||||
| -rwxr-xr-x | bench/bench_matrix.py | 2 | ||||
| -rw-r--r-- | bench/bench_model.c | 10 | ||||
| -rw-r--r-- | bench/bench_tokenizer.c | 5 | ||||
| -rwxr-xr-x | bench/benchstat.py | 21 |
8 files changed, 40 insertions, 26 deletions
diff --git a/bench/bench.h b/bench/bench.h index 8be685c..2c11fa4 100644 --- a/bench/bench.h +++ b/bench/bench.h @@ -1,4 +1,4 @@ -/* bench/bench.h - minimal microbenchmark harness. +/* Minimal microbenchmark harness. * * A benchmark function receives a bench_state_t and must run its measured * loop exactly b->n times. Call bench_begin(b) after setup so allocation diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c index 55f859b..d302396 100644 --- a/bench/bench_kernels.c +++ b/bench/bench_kernels.c @@ -1,10 +1,12 @@ -/* bench/bench_kernels.c - regression microbenchmarks for the hot kernels. +/* Regression microbenchmarks for the hot kernels. * * Shapes are the real pplx-embed-v1-0.6b dimensions (hidden 1024, 16 query / * 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts - * a slowdown in inference. Single-threaded by default for stable numbers; - * --threads N opts into the pool. Build and record via `make bench`, - * compare records with bench/benchstat.py. + * a slowdown in inference. + * + * Single-threaded by default for stable numbers; --threads N opts into the pool. + * + * Build and record via make bench, compare records with bench/benchstat.py. */ #include "bench.h" @@ -133,7 +135,7 @@ static void bm_bf16_widen(bench_state_t *b) { free(src); } -/* ---- attention pieces ---- */ +/* attention pieces */ static void bm_attn_scores(bench_state_t *b) { /* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */ @@ -171,7 +173,7 @@ static void bm_rope(bench_state_t *b) { free(x); } -/* ---- norms, activations, reductions ---- */ +/* norms, activations, reductions */ static void bm_rms_norm(bench_state_t *b) { float *x = alloc_f32((size_t)SEQ * HIDDEN, 23); diff --git a/bench/bench_late_rerank.c b/bench/bench_late_rerank.c index 987874c..88fc611 100644 --- a/bench/bench_late_rerank.c +++ b/bench/bench_late_rerank.c @@ -1,9 +1,13 @@ -/* bench/bench_late_rerank.c - late-interaction MaxSim rerank benchmark. - * Build via `make bench-late-rerank`; run with: +/* Late-interaction MaxSim rerank benchmark. + * + * Build via make bench-late-rerank; run with: * ./bench_late_rerank MODEL_DIR CANDIDATES DOC_REPEAT RUNS + * * Encodes a query and N candidate documents to token vectors, then times the * packed MaxSim rerank. check it against bench_late_rerank.py, which builds and - * runs this binary. */ + * runs this binary. + */ + #include "internal.h" #ifdef CHECK_MLX # include "mlx.h" diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py index 99bb817..489ea58 100755 --- a/bench/bench_late_rerank.py +++ b/bench/bench_late_rerank.py @@ -2,7 +2,7 @@ """Benchmark late-interaction MaxSim reranking shapes. Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it -against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only. +against a late model directory (e.g. pplx-embed-v1-late-0.6b). bench/bench_late_rerank.py --model-dir DIR """ diff --git a/bench/bench_matrix.py b/bench/bench_matrix.py index 7e0d72c..e0d99e2 100755 --- a/bench/bench_matrix.py +++ b/bench/bench_matrix.py @@ -15,7 +15,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -# Each backend is its own binary now: `make <backend>` builds the CLI into +# Each backend is its own binary now: make <backend> builds the CLI into # build/release/<dir>/ffwd-cli (and refreshes the ./ffwd-cli root symlink). BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"} diff --git a/bench/bench_model.c b/bench/bench_model.c index 7cb6458..a274285 100644 --- a/bench/bench_model.c +++ b/bench/bench_model.c @@ -1,11 +1,11 @@ -/* bench/bench_model.c - end-to-end embedding throughput against real - * weights, for regression tracking (kernels are covered by bench_kernels.c; - * this catches slowdowns in the orchestration between them). +/* End-to-end embedding throughput against real weights, for regression tracking. * + * Kernels are covered by bench_kernels.c; this catches slowdowns in the orchestration + * between them: * make bench-model MODEL_DIR=/path/to/pplx-embed-v1-0.6b * - * Uses deterministic synthetic token ids, so only config.json and - * model.safetensors are needed (no tokenizer). Single-threaded by default; + * Uses deterministic synthetic token ids, so only config.json and model.safetensors + * are needed (no tokenizer). Single-threaded by default; * --threads N opts into the pool. */ diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index a5a16ee..7c06bdd 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,5 +1,6 @@ -/* bench/bench_tokenizer.c - tokenizer encode-path benchmark. - * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */ +/* Tokenizer encode-path benchmark. + * Build via make bench-tokenizer; accepts a model dir or vocab.json. + */ #include "bpe.h" #include "sentencepiece.h" diff --git a/bench/benchstat.py b/bench/benchstat.py index 7f8b695..8036c97 100755 --- a/bench/benchstat.py +++ b/bench/benchstat.py @@ -4,13 +4,20 @@ usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0] -Per benchmark: median ns/op of both sample sets, the delta, and a -Mann-Whitney U significance test. A delta only counts when it is BOTH -significant at --alpha AND at least --threshold percent: back-to-back -runs of an unchanged binary drift a couple of percent with machine state -(thermals, core scheduling), so smaller shifts are below the -reproducibility floor even when statistically clean. Everything else -prints as "~" - treat it as noise, not improvement or regression. +Per benchmark: +median ns/op for both sample sets, the delta, and a Mann-Whitney U +significance test. + +A delta counts only when it is both: +- significant at --alpha +- at least --threshold percent + +Back-to-back runs of an unchanged binary can drift a few percent with machine +state, such as thermals and core scheduling. Smaller shifts are below the +reproducibility floor even when statistically clean. + +Everything else prints as "~"; treat it as noise, not improvement or regression. + Exits 1 if any benchmark regresses past both bars. """ |
