aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-21 12:00:45 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 12:00:45 +0200
commit674fe912b70e256790c6ac046b8f7c47af87d1e3 (patch)
treeca2f77336865458b8f29d858542786fb6e6d0bf4 /tests
parent843a1bb16f05a509392a811e725ea97c85e7a143 (diff)
downloadffwd-674fe912b70e256790c6ac046b8f7c47af87d1e3.tar.gz
ffwd-674fe912b70e256790c6ac046b8f7c47af87d1e3.zip
tests: make qwen3 parity token-count check best-effort
Diffstat (limited to 'tests')
-rw-r--r--tests/check_qwen3_parity.py18
1 files changed, 12 insertions, 6 deletions
diff --git a/tests/check_qwen3_parity.py b/tests/check_qwen3_parity.py
index 2f59fe9..83f936d 100644
--- a/tests/check_qwen3_parity.py
+++ b/tests/check_qwen3_parity.py
@@ -81,10 +81,15 @@ def local_embeddings(
emb_lines = nonempty[-len(texts):]
rows = [[float(x) for x in ln.split()] for ln in emb_lines]
+ if len(rows) != len(texts):
+ raise RuntimeError(f"local output did not contain {len(texts)} embedding rows")
+
+ # Per-text "tokens (N):" is only printed with -v on the single/stream path;
+ # the multi-arg batched path used here does not emit it, so token counts are
+ # best-effort. The C-vs-HF tokenizer is covered by check_tokenizer_parity.
token_counts = [int(x) for x in re.findall(r"tokens \((\d+)\):", proc.stderr)]
- if len(rows) != len(texts) or len(token_counts) != len(texts):
- raise RuntimeError(
- f"expected {len(texts)} rows/tokens, got {len(rows)}/{len(token_counts)}")
+ if len(token_counts) != len(texts):
+ token_counts = []
return np.asarray(rows, dtype=np.float32), token_counts
@@ -115,14 +120,15 @@ def main() -> int:
for i, (got, want) in enumerate(zip(actual, expected)):
diff = float(np.max(np.abs(got - want)))
score = cosine(got, want)
- tokens_ok = token_counts[i] == len(hf_tokens[i])
+ have_tokens = i < len(token_counts)
+ tokens_ok = (not have_tokens) or token_counts[i] == len(hf_tokens[i])
passed = tokens_ok and diff <= args.max_diff and score >= args.min_cosine
ok = ok and passed
worst_diff = max(worst_diff, diff)
worst_cosine = min(worst_cosine, score)
+ tok_str = f"{token_counts[i]}/{len(hf_tokens[i])}" if have_tokens else f"?/{len(hf_tokens[i])}"
print(
- f"{'ok' if passed else 'FAIL'}: row={i} "
- f"tokens={token_counts[i]}/{len(hf_tokens[i])} "
+ f"{'ok' if passed else 'FAIL'}: row={i} tokens={tok_str} "
f"max_abs_diff={diff:.8g} cosine={score:.8f}")
print(