# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "torch==2.8.0", # "transformers==4.56.2", # "tokenizers==0.22.0", # ] # [tool.uv.sources] # torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] # [[tool.uv.index]] # name = "pytorch-cpu" # url = "https://download.pytorch.org/whl/cpu" # explicit = true # /// """Compare native tokenization and F32 language scores with Transformers on CPU.""" import argparse import json import math from pathlib import Path import shutil import subprocess import tempfile import tokenizers import torch import transformers from transformers import AutoModelForSequenceClassification, AutoTokenizer ROOT = Path(__file__).resolve().parent.parent CLI = ROOT / "langdetect" TOKEN_IDS = ROOT / "build" / "token_ids" MAX_TOKENS = 512 SCORE_ATOL = 1e-4 CASES = [ ("en", "This is an English sentence about language detection."), ("tr", "Bugün hava çok güzel ve arkadaşlarımla dışarıda yürümek istiyorum."), ("fr", "Bonjour, ceci est une phrase écrite en français."), ("ja", "これは日本語で書かれた文章です。今日は友達と公園に行きます。"), ("nl", "Dit is een Nederlandse zin. Vandaag gaan we met vrienden naar het park."), ("ar", "هذه جملة باللغة العربية. أريد أن أتعلم المزيد عن تاريخ هذه المدينة."), ("pl", "To jest zdanie napisane po polsku. Dzisiaj idziemy z przyjaciółmi do parku."), ("de", "Dies ist ein deutscher Satz. Heute gehen wir mit unseren Freunden in den Park."), ("it", "Questa è una frase scritta in italiano. Oggi andiamo al parco con gli amici."), ("pt", "Esta é uma frase escrita em português. Hoje vamos ao parque com os nossos amigos."), ("es", "Esta es una frase escrita en español. Hoy vamos al parque con nuestros amigos."), ("hi", "यह हिंदी में लिखा गया एक वाक्य है। आज हम अपने दोस्तों के साथ पार्क जा रहे हैं।"), ("el", "Αυτή είναι μια πρόταση γραμμένη στα ελληνικά. Σήμερα πηγαίνουμε στο πάρκο με φίλους."), ("ur", "یہ اردو میں لکھا ہوا ایک جملہ ہے۔ آج ہم اپنے دوستوں کے ساتھ پارک جا رہے ہیں۔"), ("bg", "Това е изречение, написано на български език. Днес отиваме в парка с приятели."), ("zh", "这是一个用中文写的句子。今天天气很好,我们要和朋友一起去公园。"), ("ru", "Это предложение написано на русском языке. Сегодня мы идём в парк с друзьями."), ("th", "วันนี้อากาศดีมาก เราจะไปเดินเล่นที่สวนสาธารณะกับเพื่อนของเรา"), ("sw", "Leo hali ya hewa ni nzuri sana na tunakwenda kutembea bustanini pamoja na marafiki zetu."), ("vi", "Đây là một câu được viết bằng tiếng Việt. Hôm nay chúng tôi đi công viên cùng bạn bè."), ("empty", ""), ("whitespace", " \t\n "), ("normalization", "café cafe\u0301 FULLWIDTH 😀"), ("lines", "Hello\nworld.\tHow are you?"), ("long", "This is an English sentence about language detection. " * 100), ] def require(condition, message): if not condition: raise AssertionError(message) def json_lines(command, **kwargs): result = subprocess.run( [str(arg) for arg in command], cwd=ROOT, capture_output=True, text=True, timeout=120, check=True, **kwargs, ) return [json.loads(line) for line in result.stdout.splitlines()] def check_failures(model_dir): for args in (["-t", "0"], ["-t", "17"], ["-t", "bad"]): result = subprocess.run( [str(CLI), "-d", str(model_dir), *args, "hello"], capture_output=True, timeout=30, ) require(result.returncode == 1 and result.stderr and not result.stdout, f"Invalid arguments did not return a clean error: {args}") result = subprocess.run( [str(CLI), "-d", str(model_dir)], input=b"hello\x00world\n", capture_output=True, timeout=30, ) require(result.returncode == 1 and b"NUL" in result.stderr and not result.stdout, "NUL input was not rejected cleanly") with tempfile.TemporaryDirectory(prefix="failed-model-", dir=ROOT / "build") as temp: base = Path(temp) empty = base / "empty" empty.mkdir() invalid_config = base / "invalid-config" invalid_config.mkdir() (invalid_config / "config.json").write_text("{", encoding="utf-8") missing_weights = base / "missing-weights" missing_weights.mkdir() shutil.copyfile(model_dir / "config.json", missing_weights / "config.json") corrupt_weights = base / "corrupt-weights" corrupt_weights.mkdir() shutil.copyfile(model_dir / "config.json", corrupt_weights / "config.json") (corrupt_weights / "model.safetensors").write_bytes(b"invalid weights") missing_tokenizer = base / "missing-tokenizer" missing_tokenizer.mkdir() shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json") (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors") directories = [base / "absent", empty, invalid_config, missing_weights, corrupt_weights, missing_tokenizer] for name, weights, tokenizer in ( ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"), ("alternate-tokenizer", "model.safetensors", "tokenizer.model"), ("alternate-spiece", "model.safetensors", "spiece.model"), ("bert", "model.safetensors", "sentencepiece.bpe.model"), ): directory = base / name directory.mkdir() config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) if name == "bert": config["model_type"] = "bert" config["architectures"] = ["BertForSequenceClassification"] (directory / "config.json").write_text(json.dumps(config), encoding="utf-8") (directory / weights).symlink_to(model_dir / "model.safetensors") (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model") directories.append(directory) for directory in directories: result = subprocess.run( [str(CLI), "-d", str(directory), "hello"], capture_output=True, timeout=30, ) require(result.returncode == 1 and result.stderr and not result.stdout, f"Failed model load did not return a clean error: {directory.name}") print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.", flush=True) def check_reference(model_dir): torch.set_num_threads(1) tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True) require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer") cases = list(CASES) # Boundary cases straddle the BLAS attention threshold and the token limit. unit = "test " require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1, "Boundary fixture must encode to one token") cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513)) texts = [text for _, text in cases] native_ids = json_lines([TOKEN_IDS, model_dir, *texts]) require(len(native_ids) == len(cases), "Missing native tokenization results") for (name, text), ids in zip(cases, native_ids): expected = tokenizer(text, truncation=False, verbose=False)["input_ids"] if name.startswith("length-"): require(len(expected) == int(name.split("-")[1]), f"{name}: invalid boundary fixture") if name == "long": require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit") require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}") print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True) model = AutoModelForSequenceClassification.from_pretrained( model_dir, local_files_only=True, dtype=torch.float32, attn_implementation="eager", ).to("cpu").eval() labels = [model.config.id2label[i] for i in range(model.config.num_labels)] require(len(labels) == 20, "Expected 20 language labels") expected_scores = [] encoded_ids = [] with torch.inference_mode(): for text in texts: inputs = tokenizer(text, truncation=True, max_length=MAX_TOKENS, return_tensors="pt") encoded_ids.append(inputs["input_ids"][0].tolist()) expected_scores.append(torch.softmax(model(**inputs).logits[0], dim=-1).tolist()) del model worst_error = 0.0 for threads in (1, 4): native = json_lines([CLI, "-d", model_dir, "-t", threads, "--all", "--", *texts]) require(len(native) == len(cases), "Missing native predictions") for index, ((name, _), row, expected) in enumerate(zip(cases, native, expected_scores)): ids = native_ids[index] kept = ids if len(ids) <= MAX_TOKENS else ids[:MAX_TOKENS - 1] + [ids[-1]] require(kept == encoded_ids[index], f"{name}: truncated token IDs differ") require(row["tokens"] == len(kept), f"{name}: wrong token count") require(row["truncated"] == (len(ids) > MAX_TOKENS), f"{name}: wrong truncation flag") require(set(row["scores"]) == set(labels), f"{name}: wrong label mapping") scores = [row["scores"][label] for label in labels] require(all(math.isfinite(p) and 0 <= p <= 1 for p in scores), f"{name}: invalid scores") require(abs(sum(scores) - 1) <= 1e-5, f"{name}: probabilities do not sum to one") error = max(abs(actual - ref) for actual, ref in zip(scores, expected)) worst_error = max(worst_error, error) require(error <= SCORE_ATOL, f"{name}, threads={threads}: score error {error:.8g} exceeds {SCORE_ATOL}") best = max(range(len(labels)), key=lambda i: expected[i]) require(row["label"] == labels[best], f"{name}: predicted language differs") require(row["score"] == row["scores"][row["label"]], f"{name}: inconsistent best score") print(f"PASS: reference scores and labels for {len(cases)} inputs, threads={threads}.", flush=True) stream_texts = [CASES[0][1], CASES[1][1], "", " "] from_args = json_lines([CLI, "-d", model_dir, "--all", "--", *stream_texts]) from_stdin = json_lines([CLI, "-d", model_dir, "--all"], input="\n".join(stream_texts) + "\n") for row in from_args + from_stdin: row.pop("ms") require(from_args == from_stdin, "Argument and stdin predictions differ") print("PASS: argument and stdin predictions match.", flush=True) print(f"Maximum absolute probability difference: {worst_error:.8g} (limit {SCORE_ATOL}).", flush=True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model-dir", type=Path, default=ROOT / "model") args = parser.parse_args() model_dir = args.model_dir.resolve() require((model_dir / "model.safetensors").is_file(), "Download the checkpoint before testing") print(f"Reference: torch {torch.__version__}, transformers {transformers.__version__}, " f"tokenizers {tokenizers.__version__}; CPU F32, eager attention.", flush=True) check_failures(model_dir) check_reference(model_dir) if __name__ == "__main__": main()