aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-22 17:11:26 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-22 17:11:26 +0200
commit74fc389fb382cc79f67b67591569144b3fa393bd (patch)
treee5b4925f4096a463aa6566585f4cd22433380a29 /tests
parentebdf85d4d3c96fd7015192a44b94bb3bcad19735 (diff)
downloadffwd-74fc389fb382cc79f67b67591569144b3fa393bd.tar.gz
ffwd-74fc389fb382cc79f67b67591569144b3fa393bd.zip
cuda: prune attention experiments
Diffstat (limited to 'tests')
-rwxr-xr-xtests/check_batch_parity.py57
-rw-r--r--tests/check_workspace_api.py9
-rw-r--r--tests/test_workspace.c46
3 files changed, 85 insertions, 27 deletions
diff --git a/tests/check_batch_parity.py b/tests/check_batch_parity.py
index 147f21b..3b02a34 100755
--- a/tests/check_batch_parity.py
+++ b/tests/check_batch_parity.py
@@ -41,6 +41,35 @@ LONG_TEXTS = [
]
+def detect_pooling_mode(model_dir: str) -> str:
+ """Return the sentence pooling mode used by ffwd_config_load, if obvious."""
+ root = Path(model_dir)
+ pooling_path = root / "1_Pooling" / "config.json"
+ if pooling_path.exists():
+ try:
+ cfg = json.loads(pooling_path.read_text())
+ except (OSError, json.JSONDecodeError):
+ return "unknown"
+ if cfg.get("pooling_mode_lasttoken"):
+ return "last-token"
+ if cfg.get("pooling_mode_cls_token"):
+ return "cls"
+ if cfg.get("pooling_mode_mean_tokens"):
+ return "mean"
+ return "unknown"
+
+ config_path = root / "config.json"
+ if config_path.exists():
+ try:
+ cfg = json.loads(config_path.read_text())
+ except (OSError, json.JSONDecodeError):
+ return "unknown"
+ if cfg.get("model_type") == "qwen3":
+ return "last-token"
+
+ return "unknown"
+
+
def run_stdin(binary, model_dir, backend, batch_size, texts,
mlx_quantize_bits, mlx_quantize_group_size,
cuda_weight_dtype=None):
@@ -98,6 +127,12 @@ def main():
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--max-diff", type=float, default=5e-5)
ap.add_argument("--min-cosine", type=float, default=0.99999)
+ ap.add_argument("--cuda-last-token-max-diff", type=float, default=0.002,
+ help=("effective max-abs gate for CUDA last-token pooling; "
+ "keeps cosine strict while allowing batched-GEMM "
+ "float reordering"))
+ ap.add_argument("--cuda-last-token-min-cosine", type=float, default=0.9999,
+ help="effective cosine gate for CUDA last-token pooling")
ap.add_argument("--long", action="store_true",
help="use long inputs that exercise tiled CPU attention")
ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
@@ -117,8 +152,20 @@ def main():
raise SystemExit("--mlx-quant-bits requires --backend mlx")
if args.mlx_quantize_group_size <= 0:
raise SystemExit("--mlx-quant-group-size must be > 0")
+ if args.cuda_last_token_max_diff <= 0:
+ raise SystemExit("--cuda-last-token-max-diff must be > 0")
+ if not 0.0 < args.cuda_last_token_min_cosine <= 1.0:
+ raise SystemExit("--cuda-last-token-min-cosine must be in (0, 1]")
ensure_cli(Path(args.binary))
+ pooling_mode = detect_pooling_mode(args.model_dir)
+ effective_max_diff = args.max_diff
+ effective_min_cosine = args.min_cosine
+ calibrated_cuda_last_token = args.backend == "cuda" and pooling_mode == "last-token"
+ if calibrated_cuda_last_token and effective_max_diff < args.cuda_last_token_max_diff:
+ effective_max_diff = args.cuda_last_token_max_diff
+ if calibrated_cuda_last_token and effective_min_cosine > args.cuda_last_token_min_cosine:
+ effective_min_cosine = args.cuda_last_token_min_cosine
seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts,
args.mlx_quantize_bits,
@@ -145,11 +192,16 @@ def main():
f"max_abs_diff={max_diff:.8g}"
)
- if worst_diff > args.max_diff or worst_cos < args.min_cosine:
+ if worst_diff > effective_max_diff or worst_cos < effective_min_cosine:
raise RuntimeError(
- f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}"
+ f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}, "
+ f"max_diff_gate={effective_max_diff:g}, min_cosine_gate={effective_min_cosine:g}"
)
+ calibration_note = ""
+ if calibrated_cuda_last_token:
+ calibration_note = (f", pooling={pooling_mode}, max_diff_gate={effective_max_diff:g}, "
+ f"min_cosine_gate={effective_min_cosine:g}")
seq_ms = sum(float(row["ms"]) for row in seq_rows)
batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size))
print(
@@ -157,6 +209,7 @@ def main():
f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, "
f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, "
f"batch_chunk_ms_sum={batch_ms:.1f}"
+ + calibration_note
+ (
f", mlx_q{args.mlx_quantize_bits}"
if args.mlx_quantize_bits else ""
diff --git a/tests/check_workspace_api.py b/tests/check_workspace_api.py
index 40bac11..05f0e8c 100644
--- a/tests/check_workspace_api.py
+++ b/tests/check_workspace_api.py
@@ -6,11 +6,12 @@ the synthetic fixture. Given a MODEL_DIR it loads real weights from that
directory instead and exercises the workspace and batch APIs. Needs only
python3 and a model directory (with vocab.json), no reference stack.
-The model must be mean-pooled: the test checks that a whole-sequence span pools
-to the embedding, which holds for mean pooling (pplx-embed, BERT/BGE) but not
-for last-token pooling (Qwen3). Use a pplx-embed model.
+For mean-pooled models, the test additionally checks that a whole-sequence span
+pools to the embedding and that half-spans recombine by token-count weighting.
+For last-token/CLS models, those mean-span identities are skipped because the
+sentence embedding and contextual span APIs intentionally pool different rows.
- tests/check_workspace_api.py --model-dir DIR # DIR = a pplx-embed model
+ tests/check_workspace_api.py --model-dir DIR
"""
import argparse
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
index bbbef59..1921465 100644
--- a/tests/test_workspace.c
+++ b/tests/test_workspace.c
@@ -59,8 +59,10 @@ static int check_vector_helpers(void) {
return 0;
}
-/* The allocating and pooling API variants must agree with the *_into
- * results validated in main. reference is the batched row for ids0. */
+/* The allocating and model-pooling API variants must agree with the *_into
+ * results validated in main. reference is the batched row for ids0. Contextual
+ * spans are always mean-pooled ranges, so only mean-pooling models can require
+ * a whole-sequence span to equal the sentence embedding. */
static int check_alloc_and_pooling_variants(const ffwd_model_t *model,
ffwd_workspace_t *ws,
const int *ids0,
@@ -98,33 +100,35 @@ static int check_alloc_and_pooling_variants(const ffwd_model_t *model,
goto done;
}
- /* Mean-pooling the final states reproduces the embedding. */
if (ffwd_pool_batch(cfg, states, &n0, 1, pooled) != 0 ||
max_abs_diff(pooled, reference, dim) > 0.00005f) {
fprintf(stderr, "ffwd_pool_batch disagrees with embedding\n");
goto done;
}
- /* One span covering the whole sequence pools to the embedding, and
- * two halves recombine into it by token-count weighting. */
- ffwd_span_t whole = {0, n0};
- if (ffwd_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 ||
- max_abs_diff(span_out, reference, dim) > 0.00005f) {
- fprintf(stderr, "whole-sequence span disagrees with embedding\n");
- goto done;
- }
- int h = n0 / 2;
- ffwd_span_t halves[2] = {{0, h}, {h, n0 - h}};
- if (ffwd_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) {
- fprintf(stderr, "ffwd_model_encode_spans failed for two spans\n");
- goto done;
- }
- for (int d = 0; d < dim; d++) {
- float combined = ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0;
- if (fabsf(combined - reference[d]) > 0.00005f) {
- fprintf(stderr, "half spans do not recombine into embedding\n");
+ if (cfg->pooling_mode == FFWD_POOL_MEAN) {
+ /* One span covering the whole sequence pools to the embedding, and
+ * two halves recombine into it by token-count weighting. */
+ ffwd_span_t whole = {0, n0};
+ if (ffwd_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 ||
+ max_abs_diff(span_out, reference, dim) > 0.00005f) {
+ fprintf(stderr, "whole-sequence span disagrees with embedding\n");
goto done;
}
+ int h = n0 / 2;
+ ffwd_span_t halves[2] = {{0, h}, {h, n0 - h}};
+ if (ffwd_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) {
+ fprintf(stderr, "ffwd_model_encode_spans failed for two spans\n");
+ goto done;
+ }
+ for (int d = 0; d < dim; d++) {
+ float combined =
+ ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0;
+ if (fabsf(combined - reference[d]) > 0.00005f) {
+ fprintf(stderr, "half spans do not recombine into embedding\n");
+ goto done;
+ }
+ }
}
rc = 0;