aboutsummaryrefslogtreecommitdiff
path: root/bench
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-27 00:58:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-27 01:03:41 +0200
commitaf55803d0dff9eb692c760c8d486e736c945061d (patch)
tree0875a1112bf98304e445e0544c4309670d6f4769 /bench
parent0f861b7f3098750ddee4f6570f3be228d1da5320 (diff)
downloadffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz
ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip
chore: clean up comment formatting and wording
Diffstat (limited to 'bench')
-rw-r--r--bench/bench.h2
-rw-r--r--bench/bench_kernels.c14
-rw-r--r--bench/bench_late_rerank.c10
-rwxr-xr-xbench/bench_late_rerank.py2
-rwxr-xr-xbench/bench_matrix.py2
-rw-r--r--bench/bench_model.c10
-rw-r--r--bench/bench_tokenizer.c5
-rwxr-xr-xbench/benchstat.py21
8 files changed, 40 insertions, 26 deletions
diff --git a/bench/bench.h b/bench/bench.h
index 8be685c..2c11fa4 100644
--- a/bench/bench.h
+++ b/bench/bench.h
@@ -1,4 +1,4 @@
-/* bench/bench.h - minimal microbenchmark harness.
+/* Minimal microbenchmark harness.
*
* A benchmark function receives a bench_state_t and must run its measured
* loop exactly b->n times. Call bench_begin(b) after setup so allocation
diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c
index 55f859b..d302396 100644
--- a/bench/bench_kernels.c
+++ b/bench/bench_kernels.c
@@ -1,10 +1,12 @@
-/* bench/bench_kernels.c - regression microbenchmarks for the hot kernels.
+/* Regression microbenchmarks for the hot kernels.
*
* Shapes are the real pplx-embed-v1-0.6b dimensions (hidden 1024, 16 query /
* 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts
- * a slowdown in inference. Single-threaded by default for stable numbers;
- * --threads N opts into the pool. Build and record via `make bench`,
- * compare records with bench/benchstat.py.
+ * a slowdown in inference.
+ *
+ * Single-threaded by default for stable numbers; --threads N opts into the pool.
+ *
+ * Build and record via make bench, compare records with bench/benchstat.py.
*/
#include "bench.h"
@@ -133,7 +135,7 @@ static void bm_bf16_widen(bench_state_t *b) {
free(src);
}
-/* ---- attention pieces ---- */
+/* attention pieces */
static void bm_attn_scores(bench_state_t *b) {
/* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */
@@ -171,7 +173,7 @@ static void bm_rope(bench_state_t *b) {
free(x);
}
-/* ---- norms, activations, reductions ---- */
+/* norms, activations, reductions */
static void bm_rms_norm(bench_state_t *b) {
float *x = alloc_f32((size_t)SEQ * HIDDEN, 23);
diff --git a/bench/bench_late_rerank.c b/bench/bench_late_rerank.c
index 987874c..88fc611 100644
--- a/bench/bench_late_rerank.c
+++ b/bench/bench_late_rerank.c
@@ -1,9 +1,13 @@
-/* bench/bench_late_rerank.c - late-interaction MaxSim rerank benchmark.
- * Build via `make bench-late-rerank`; run with:
+/* Late-interaction MaxSim rerank benchmark.
+ *
+ * Build via make bench-late-rerank; run with:
* ./bench_late_rerank MODEL_DIR CANDIDATES DOC_REPEAT RUNS
+ *
* Encodes a query and N candidate documents to token vectors, then times the
* packed MaxSim rerank. check it against bench_late_rerank.py, which builds and
- * runs this binary. */
+ * runs this binary.
+ */
+
#include "internal.h"
#ifdef CHECK_MLX
# include "mlx.h"
diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py
index 99bb817..489ea58 100755
--- a/bench/bench_late_rerank.py
+++ b/bench/bench_late_rerank.py
@@ -2,7 +2,7 @@
"""Benchmark late-interaction MaxSim reranking shapes.
Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it
-against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only.
+against a late model directory (e.g. pplx-embed-v1-late-0.6b).
bench/bench_late_rerank.py --model-dir DIR
"""
diff --git a/bench/bench_matrix.py b/bench/bench_matrix.py
index 7e0d72c..e0d99e2 100755
--- a/bench/bench_matrix.py
+++ b/bench/bench_matrix.py
@@ -15,7 +15,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
-# Each backend is its own binary now: `make <backend>` builds the CLI into
+# Each backend is its own binary now: make <backend> builds the CLI into
# build/release/<dir>/ffwd-cli (and refreshes the ./ffwd-cli root symlink).
BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"}
diff --git a/bench/bench_model.c b/bench/bench_model.c
index 7cb6458..a274285 100644
--- a/bench/bench_model.c
+++ b/bench/bench_model.c
@@ -1,11 +1,11 @@
-/* bench/bench_model.c - end-to-end embedding throughput against real
- * weights, for regression tracking (kernels are covered by bench_kernels.c;
- * this catches slowdowns in the orchestration between them).
+/* End-to-end embedding throughput against real weights, for regression tracking.
*
+ * Kernels are covered by bench_kernels.c; this catches slowdowns in the orchestration
+ * between them:
* make bench-model MODEL_DIR=/path/to/pplx-embed-v1-0.6b
*
- * Uses deterministic synthetic token ids, so only config.json and
- * model.safetensors are needed (no tokenizer). Single-threaded by default;
+ * Uses deterministic synthetic token ids, so only config.json and model.safetensors
+ * are needed (no tokenizer). Single-threaded by default;
* --threads N opts into the pool.
*/
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index a5a16ee..7c06bdd 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -1,5 +1,6 @@
-/* bench/bench_tokenizer.c - tokenizer encode-path benchmark.
- * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */
+/* Tokenizer encode-path benchmark.
+ * Build via make bench-tokenizer; accepts a model dir or vocab.json.
+ */
#include "bpe.h"
#include "sentencepiece.h"
diff --git a/bench/benchstat.py b/bench/benchstat.py
index 7f8b695..8036c97 100755
--- a/bench/benchstat.py
+++ b/bench/benchstat.py
@@ -4,13 +4,20 @@
usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0]
-Per benchmark: median ns/op of both sample sets, the delta, and a
-Mann-Whitney U significance test. A delta only counts when it is BOTH
-significant at --alpha AND at least --threshold percent: back-to-back
-runs of an unchanged binary drift a couple of percent with machine state
-(thermals, core scheduling), so smaller shifts are below the
-reproducibility floor even when statistically clean. Everything else
-prints as "~" - treat it as noise, not improvement or regression.
+Per benchmark:
+median ns/op for both sample sets, the delta, and a Mann-Whitney U
+significance test.
+
+A delta counts only when it is both:
+- significant at --alpha
+- at least --threshold percent
+
+Back-to-back runs of an unchanged binary can drift a few percent with machine
+state, such as thermals and core scheduling. Smaller shifts are below the
+reproducibility floor even when statistically clean.
+
+Everything else prints as "~"; treat it as noise, not improvement or regression.
+
Exits 1 if any benchmark regresses past both bars.
"""