diff options
Diffstat (limited to 'tests')
| -rwxr-xr-x | tests/check_batch_parity.py | 57 | ||||
| -rw-r--r-- | tests/check_workspace_api.py | 9 | ||||
| -rw-r--r-- | tests/test_workspace.c | 46 |
3 files changed, 85 insertions, 27 deletions
diff --git a/tests/check_batch_parity.py b/tests/check_batch_parity.py index 147f21b..3b02a34 100755 --- a/tests/check_batch_parity.py +++ b/tests/check_batch_parity.py @@ -41,6 +41,35 @@ LONG_TEXTS = [ ] +def detect_pooling_mode(model_dir: str) -> str: + """Return the sentence pooling mode used by ffwd_config_load, if obvious.""" + root = Path(model_dir) + pooling_path = root / "1_Pooling" / "config.json" + if pooling_path.exists(): + try: + cfg = json.loads(pooling_path.read_text()) + except (OSError, json.JSONDecodeError): + return "unknown" + if cfg.get("pooling_mode_lasttoken"): + return "last-token" + if cfg.get("pooling_mode_cls_token"): + return "cls" + if cfg.get("pooling_mode_mean_tokens"): + return "mean" + return "unknown" + + config_path = root / "config.json" + if config_path.exists(): + try: + cfg = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + return "unknown" + if cfg.get("model_type") == "qwen3": + return "last-token" + + return "unknown" + + def run_stdin(binary, model_dir, backend, batch_size, texts, mlx_quantize_bits, mlx_quantize_group_size, cuda_weight_dtype=None): @@ -98,6 +127,12 @@ def main(): ap.add_argument("--batch-size", type=int, default=4) ap.add_argument("--max-diff", type=float, default=5e-5) ap.add_argument("--min-cosine", type=float, default=0.99999) + ap.add_argument("--cuda-last-token-max-diff", type=float, default=0.002, + help=("effective max-abs gate for CUDA last-token pooling; " + "keeps cosine strict while allowing batched-GEMM " + "float reordering")) + ap.add_argument("--cuda-last-token-min-cosine", type=float, default=0.9999, + help="effective cosine gate for CUDA last-token pooling") ap.add_argument("--long", action="store_true", help="use long inputs that exercise tiled CPU attention") ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits", @@ -117,8 +152,20 @@ def main(): raise SystemExit("--mlx-quant-bits requires --backend mlx") if args.mlx_quantize_group_size <= 0: raise SystemExit("--mlx-quant-group-size must be > 0") + if args.cuda_last_token_max_diff <= 0: + raise SystemExit("--cuda-last-token-max-diff must be > 0") + if not 0.0 < args.cuda_last_token_min_cosine <= 1.0: + raise SystemExit("--cuda-last-token-min-cosine must be in (0, 1]") ensure_cli(Path(args.binary)) + pooling_mode = detect_pooling_mode(args.model_dir) + effective_max_diff = args.max_diff + effective_min_cosine = args.min_cosine + calibrated_cuda_last_token = args.backend == "cuda" and pooling_mode == "last-token" + if calibrated_cuda_last_token and effective_max_diff < args.cuda_last_token_max_diff: + effective_max_diff = args.cuda_last_token_max_diff + if calibrated_cuda_last_token and effective_min_cosine > args.cuda_last_token_min_cosine: + effective_min_cosine = args.cuda_last_token_min_cosine seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts, args.mlx_quantize_bits, @@ -145,11 +192,16 @@ def main(): f"max_abs_diff={max_diff:.8g}" ) - if worst_diff > args.max_diff or worst_cos < args.min_cosine: + if worst_diff > effective_max_diff or worst_cos < effective_min_cosine: raise RuntimeError( - f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}" + f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}, " + f"max_diff_gate={effective_max_diff:g}, min_cosine_gate={effective_min_cosine:g}" ) + calibration_note = "" + if calibrated_cuda_last_token: + calibration_note = (f", pooling={pooling_mode}, max_diff_gate={effective_max_diff:g}, " + f"min_cosine_gate={effective_min_cosine:g}") seq_ms = sum(float(row["ms"]) for row in seq_rows) batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size)) print( @@ -157,6 +209,7 @@ def main(): f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, " f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, " f"batch_chunk_ms_sum={batch_ms:.1f}" + + calibration_note + ( f", mlx_q{args.mlx_quantize_bits}" if args.mlx_quantize_bits else "" diff --git a/tests/check_workspace_api.py b/tests/check_workspace_api.py index 40bac11..05f0e8c 100644 --- a/tests/check_workspace_api.py +++ b/tests/check_workspace_api.py @@ -6,11 +6,12 @@ the synthetic fixture. Given a MODEL_DIR it loads real weights from that directory instead and exercises the workspace and batch APIs. Needs only python3 and a model directory (with vocab.json), no reference stack. -The model must be mean-pooled: the test checks that a whole-sequence span pools -to the embedding, which holds for mean pooling (pplx-embed, BERT/BGE) but not -for last-token pooling (Qwen3). Use a pplx-embed model. +For mean-pooled models, the test additionally checks that a whole-sequence span +pools to the embedding and that half-spans recombine by token-count weighting. +For last-token/CLS models, those mean-span identities are skipped because the +sentence embedding and contextual span APIs intentionally pool different rows. - tests/check_workspace_api.py --model-dir DIR # DIR = a pplx-embed model + tests/check_workspace_api.py --model-dir DIR """ import argparse diff --git a/tests/test_workspace.c b/tests/test_workspace.c index bbbef59..1921465 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -59,8 +59,10 @@ static int check_vector_helpers(void) { return 0; } -/* The allocating and pooling API variants must agree with the *_into - * results validated in main. reference is the batched row for ids0. */ +/* The allocating and model-pooling API variants must agree with the *_into + * results validated in main. reference is the batched row for ids0. Contextual + * spans are always mean-pooled ranges, so only mean-pooling models can require + * a whole-sequence span to equal the sentence embedding. */ static int check_alloc_and_pooling_variants(const ffwd_model_t *model, ffwd_workspace_t *ws, const int *ids0, @@ -98,33 +100,35 @@ static int check_alloc_and_pooling_variants(const ffwd_model_t *model, goto done; } - /* Mean-pooling the final states reproduces the embedding. */ if (ffwd_pool_batch(cfg, states, &n0, 1, pooled) != 0 || max_abs_diff(pooled, reference, dim) > 0.00005f) { fprintf(stderr, "ffwd_pool_batch disagrees with embedding\n"); goto done; } - /* One span covering the whole sequence pools to the embedding, and - * two halves recombine into it by token-count weighting. */ - ffwd_span_t whole = {0, n0}; - if (ffwd_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 || - max_abs_diff(span_out, reference, dim) > 0.00005f) { - fprintf(stderr, "whole-sequence span disagrees with embedding\n"); - goto done; - } - int h = n0 / 2; - ffwd_span_t halves[2] = {{0, h}, {h, n0 - h}}; - if (ffwd_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) { - fprintf(stderr, "ffwd_model_encode_spans failed for two spans\n"); - goto done; - } - for (int d = 0; d < dim; d++) { - float combined = ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0; - if (fabsf(combined - reference[d]) > 0.00005f) { - fprintf(stderr, "half spans do not recombine into embedding\n"); + if (cfg->pooling_mode == FFWD_POOL_MEAN) { + /* One span covering the whole sequence pools to the embedding, and + * two halves recombine into it by token-count weighting. */ + ffwd_span_t whole = {0, n0}; + if (ffwd_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 || + max_abs_diff(span_out, reference, dim) > 0.00005f) { + fprintf(stderr, "whole-sequence span disagrees with embedding\n"); goto done; } + int h = n0 / 2; + ffwd_span_t halves[2] = {{0, h}, {h, n0 - h}}; + if (ffwd_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) { + fprintf(stderr, "ffwd_model_encode_spans failed for two spans\n"); + goto done; + } + for (int d = 0; d < dim; d++) { + float combined = + ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0; + if (fabsf(combined - reference[d]) > 0.00005f) { + fprintf(stderr, "half spans do not recombine into embedding\n"); + goto done; + } + } } rc = 0; |
