aboutsummaryrefslogtreecommitdiff
path: root/tools/verify.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
commit6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (patch)
tree427e45b901aec902ceba72428fe5de9689d3001b /tools/verify.py
parentcb1a04331b27cc19445de63cedcd4fd30b8a9016 (diff)
downloadlangdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.tar.gz
langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.zip
Add CPU reference comparison and repeatable API tests
Diffstat (limited to 'tools/verify.py')
-rw-r--r--tools/verify.py201
1 files changed, 201 insertions, 0 deletions
diff --git a/tools/verify.py b/tools/verify.py
new file mode 100644
index 0000000..197f68f
--- /dev/null
+++ b/tools/verify.py
@@ -0,0 +1,201 @@
+# /// script
+# requires-python = ">=3.10,<3.14"
+# dependencies = [
+# "torch==2.8.0",
+# "transformers==4.56.2",
+# "tokenizers==0.22.0",
+# ]
+# ///
+"""Compare native tokenization and F32 language scores with Transformers on CPU."""
+
+import argparse
+import json
+import math
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+
+import tokenizers
+import torch
+import transformers
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+ROOT = Path(__file__).resolve().parent.parent
+CLI = ROOT / "langdetect"
+TOKEN_IDS = ROOT / "build" / "token_ids"
+MAX_TOKENS = 512
+SCORE_ATOL = 1e-4
+CASES = [
+ ("en", "This is an English sentence about language detection."),
+ ("tr", "Bugün hava çok güzel ve arkadaşlarımla dışarıda yürümek istiyorum."),
+ ("fr", "Bonjour, ceci est une phrase écrite en français."),
+ ("ja", "これは日本語で書かれた文章です。今日は友達と公園に行きます。"),
+ ("nl", "Dit is een Nederlandse zin. Vandaag gaan we met vrienden naar het park."),
+ ("ar", "هذه جملة باللغة العربية. أريد أن أتعلم المزيد عن تاريخ هذه المدينة."),
+ ("pl", "To jest zdanie napisane po polsku. Dzisiaj idziemy z przyjaciółmi do parku."),
+ ("de", "Dies ist ein deutscher Satz. Heute gehen wir mit unseren Freunden in den Park."),
+ ("it", "Questa è una frase scritta in italiano. Oggi andiamo al parco con gli amici."),
+ ("pt", "Esta é uma frase escrita em português. Hoje vamos ao parque com os nossos amigos."),
+ ("es", "Esta es una frase escrita en español. Hoy vamos al parque con nuestros amigos."),
+ ("hi", "यह हिंदी में लिखा गया एक वाक्य है। आज हम अपने दोस्तों के साथ पार्क जा रहे हैं।"),
+ ("el", "Αυτή είναι μια πρόταση γραμμένη στα ελληνικά. Σήμερα πηγαίνουμε στο πάρκο με φίλους."),
+ ("ur", "یہ اردو میں لکھا ہوا ایک جملہ ہے۔ آج ہم اپنے دوستوں کے ساتھ پارک جا رہے ہیں۔"),
+ ("bg", "Това е изречение, написано на български език. Днес отиваме в парка с приятели."),
+ ("zh", "这是一个用中文写的句子。今天天气很好,我们要和朋友一起去公园。"),
+ ("ru", "Это предложение написано на русском языке. Сегодня мы идём в парк с друзьями."),
+ ("th", "วันนี้อากาศดีมาก เราจะไปเดินเล่นที่สวนสาธารณะกับเพื่อนของเรา"),
+ ("sw", "Leo hali ya hewa ni nzuri sana na tunakwenda kutembea bustanini pamoja na marafiki zetu."),
+ ("vi", "Đây là một câu được viết bằng tiếng Việt. Hôm nay chúng tôi đi công viên cùng bạn bè."),
+ ("empty", ""),
+ ("whitespace", " \t\n "),
+ ("normalization", "café cafe\u0301 FULLWIDTH 😀"),
+ ("lines", "Hello\nworld.\tHow are you?"),
+ ("long", "This is an English sentence about language detection. " * 100),
+]
+
+
+def require(condition, message):
+ if not condition:
+ raise AssertionError(message)
+
+
+def json_lines(command, **kwargs):
+ result = subprocess.run(
+ [str(arg) for arg in command], cwd=ROOT, capture_output=True,
+ text=True, timeout=120, check=True, **kwargs,
+ )
+ return [json.loads(line) for line in result.stdout.splitlines()]
+
+
+def check_failures(model_dir):
+ for args in (["-t", "0"], ["-t", "17"], ["-t", "bad"]):
+ result = subprocess.run(
+ [str(CLI), "-d", str(model_dir), *args, "hello"],
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and result.stderr and not result.stdout,
+ f"Invalid arguments did not return a clean error: {args}")
+ result = subprocess.run(
+ [str(CLI), "-d", str(model_dir)], input=b"hello\x00world\n",
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and b"NUL" in result.stderr and not result.stdout,
+ "NUL input was not rejected cleanly")
+
+ with tempfile.TemporaryDirectory(prefix="failed-model-", dir=ROOT / "build") as temp:
+ base = Path(temp)
+ empty = base / "empty"
+ empty.mkdir()
+ invalid_config = base / "invalid-config"
+ invalid_config.mkdir()
+ (invalid_config / "config.json").write_text("{", encoding="utf-8")
+ missing_weights = base / "missing-weights"
+ missing_weights.mkdir()
+ shutil.copyfile(model_dir / "config.json", missing_weights / "config.json")
+ corrupt_weights = base / "corrupt-weights"
+ corrupt_weights.mkdir()
+ shutil.copyfile(model_dir / "config.json", corrupt_weights / "config.json")
+ (corrupt_weights / "model.safetensors").write_bytes(b"invalid weights")
+ missing_tokenizer = base / "missing-tokenizer"
+ missing_tokenizer.mkdir()
+ shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
+ (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
+ for directory in (base / "absent", empty, invalid_config, missing_weights,
+ corrupt_weights, missing_tokenizer):
+ result = subprocess.run(
+ [str(CLI), "-d", str(directory), "hello"],
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and result.stderr and not result.stdout,
+ f"Failed model load did not return a clean error: {directory.name}")
+ print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True)
+
+
+def check_reference(model_dir):
+ torch.set_num_threads(1)
+ tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
+ require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
+ cases = list(CASES)
+ # Adjacent sequence lengths exercise both sides of the 512-token limit.
+ unit = "test "
+ require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
+ "Boundary fixture must encode to one token")
+ cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513))
+ texts = [text for _, text in cases]
+ native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
+ require(len(native_ids) == len(cases), "Missing native tokenization results")
+ reference_ids = []
+ for (name, text), ids in zip(cases, native_ids):
+ expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
+ if name.startswith("length-"):
+ require(len(expected) == int(name.split("-")[1]), f"{name}: invalid boundary fixture")
+ if name == "long":
+ require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
+ require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
+ reference_ids.append(expected)
+ print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)
+
+ model = AutoModelForSequenceClassification.from_pretrained(
+ model_dir, local_files_only=True, dtype=torch.float32,
+ attn_implementation="eager",
+ ).to("cpu").eval()
+ labels = [model.config.id2label[i] for i in range(model.config.num_labels)]
+ require(len(labels) == 20, "Expected 20 language labels")
+ expected_scores = []
+ encoded_ids = []
+ with torch.inference_mode():
+ for text in texts:
+ inputs = tokenizer(text, truncation=True, max_length=MAX_TOKENS, return_tensors="pt")
+ encoded_ids.append(inputs["input_ids"][0].tolist())
+ expected_scores.append(torch.softmax(model(**inputs).logits[0], dim=-1).tolist())
+ del model
+
+ worst_error = 0.0
+ for threads in (1, 4):
+ native = json_lines([CLI, "-d", model_dir, "-t", threads, "--all", "--", *texts])
+ require(len(native) == len(cases), "Missing native predictions")
+ for index, ((name, _), row, expected) in enumerate(zip(cases, native, expected_scores)):
+ ids = native_ids[index]
+ kept = ids if len(ids) <= MAX_TOKENS else ids[:MAX_TOKENS - 1] + [ids[-1]]
+ require(kept == encoded_ids[index], f"{name}: truncated token IDs differ")
+ require(row["tokens"] == len(kept), f"{name}: wrong token count")
+ require(row["truncated"] == (len(ids) > MAX_TOKENS), f"{name}: wrong truncation flag")
+ require(set(row["scores"]) == set(labels), f"{name}: wrong label mapping")
+ scores = [row["scores"][label] for label in labels]
+ require(all(math.isfinite(p) and 0 <= p <= 1 for p in scores), f"{name}: invalid scores")
+ require(abs(sum(scores) - 1) <= 1e-5, f"{name}: probabilities do not sum to one")
+ error = max(abs(actual - ref) for actual, ref in zip(scores, expected))
+ worst_error = max(worst_error, error)
+ require(error <= SCORE_ATOL,
+ f"{name}, threads={threads}: score error {error:.8g} exceeds {SCORE_ATOL}")
+ best = max(range(len(labels)), key=lambda i: expected[i])
+ require(row["label"] == labels[best], f"{name}: predicted language differs")
+ require(row["score"] == row["scores"][row["label"]], f"{name}: inconsistent best score")
+ print(f"PASS: reference scores and labels for {len(cases)} inputs, threads={threads}.", flush=True)
+
+ stream_texts = [CASES[0][1], CASES[1][1], "", " "]
+ from_args = json_lines([CLI, "-d", model_dir, "--all", "--", *stream_texts])
+ from_stdin = json_lines([CLI, "-d", model_dir, "--all"], input="\n".join(stream_texts) + "\n")
+ for row in from_args + from_stdin:
+ row.pop("ms")
+ require(from_args == from_stdin, "Argument and stdin predictions differ")
+ print("PASS: argument and stdin predictions match.", flush=True)
+ print(f"Maximum absolute probability difference: {worst_error:.8g} (limit {SCORE_ATOL}).", flush=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-dir", type=Path, default=ROOT / "model")
+ args = parser.parse_args()
+ model_dir = args.model_dir.resolve()
+ require((model_dir / "model.safetensors").is_file(), "Download the checkpoint before testing")
+ print(f"Reference: torch {torch.__version__}, transformers {transformers.__version__}, "
+ f"tokenizers {tokenizers.__version__}; CPU F32, eager attention.", flush=True)
+ check_failures(model_dir)
+ check_reference(model_dir)
+
+
+if __name__ == "__main__":
+ main()