aboutsummaryrefslogtreecommitdiff
path: root/tools/verify.py
blob: 8339ac919f2f0d3c9a5389ee0e6b77a0f24f87a7 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
# /// script
# requires-python = ">=3.10,<3.14"
# dependencies = [
#   "torch==2.8.0",
#   "transformers==4.56.2",
#   "tokenizers==0.22.0",
# ]
# [tool.uv.sources]
# torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
# [[tool.uv.index]]
# name = "pytorch-cpu"
# url = "https://download.pytorch.org/whl/cpu"
# explicit = true
# ///
"""Compare native tokenization and F32 language scores with Transformers on CPU."""

import argparse
import json
import math
from pathlib import Path
import shutil
import subprocess
import tempfile

import tokenizers
import torch
import transformers
from transformers import AutoModelForSequenceClassification, AutoTokenizer


ROOT = Path(__file__).resolve().parent.parent
CLI = ROOT / "langdetect"
TOKEN_IDS = ROOT / "build" / "token_ids"
MAX_TOKENS = 512
SCORE_ATOL = 1e-4
CASES = [
    ("en", "This is an English sentence about language detection."),
    ("tr", "Bugün hava çok güzel ve arkadaşlarımla dışarıda yürümek istiyorum."),
    ("fr", "Bonjour, ceci est une phrase écrite en français."),
    ("ja", "これは日本語で書かれた文章です。今日は友達と公園に行きます。"),
    ("nl", "Dit is een Nederlandse zin. Vandaag gaan we met vrienden naar het park."),
    ("ar", "هذه جملة باللغة العربية. أريد أن أتعلم المزيد عن تاريخ هذه المدينة."),
    ("pl", "To jest zdanie napisane po polsku. Dzisiaj idziemy z przyjaciółmi do parku."),
    ("de", "Dies ist ein deutscher Satz. Heute gehen wir mit unseren Freunden in den Park."),
    ("it", "Questa è una frase scritta in italiano. Oggi andiamo al parco con gli amici."),
    ("pt", "Esta é uma frase escrita em português. Hoje vamos ao parque com os nossos amigos."),
    ("es", "Esta es una frase escrita en español. Hoy vamos al parque con nuestros amigos."),
    ("hi", "यह हिंदी में लिखा गया एक वाक्य है। आज हम अपने दोस्तों के साथ पार्क जा रहे हैं।"),
    ("el", "Αυτή είναι μια πρόταση γραμμένη στα ελληνικά. Σήμερα πηγαίνουμε στο πάρκο με φίλους."),
    ("ur", "یہ اردو میں لکھا ہوا ایک جملہ ہے۔ آج ہم اپنے دوستوں کے ساتھ پارک جا رہے ہیں۔"),
    ("bg", "Това е изречение, написано на български език. Днес отиваме в парка с приятели."),
    ("zh", "这是一个用中文写的句子。今天天气很好,我们要和朋友一起去公园。"),
    ("ru", "Это предложение написано на русском языке. Сегодня мы идём в парк с друзьями."),
    ("th", "วันนี้อากาศดีมาก เราจะไปเดินเล่นที่สวนสาธารณะกับเพื่อนของเรา"),
    ("sw", "Leo hali ya hewa ni nzuri sana na tunakwenda kutembea bustanini pamoja na marafiki zetu."),
    ("vi", "Đây là một câu được viết bằng tiếng Việt. Hôm nay chúng tôi đi công viên cùng bạn bè."),
    ("empty", ""),
    ("whitespace", " \t\n "),
    ("normalization", "café cafe\u0301 FULLWIDTH 😀"),
    ("lines", "Hello\nworld.\tHow are you?"),
    ("long", "This is an English sentence about language detection. " * 100),
]


def require(condition, message):
    if not condition:
        raise AssertionError(message)


def json_lines(command, **kwargs):
    result = subprocess.run(
        [str(arg) for arg in command], cwd=ROOT, capture_output=True,
        text=True, timeout=120, check=True, **kwargs,
    )
    return [json.loads(line) for line in result.stdout.splitlines()]


def check_failures(model_dir):
    for args in (["-t", "0"], ["-t", "17"], ["-t", "bad"]):
        result = subprocess.run(
            [str(CLI), "-d", str(model_dir), *args, "hello"],
            capture_output=True, timeout=30,
        )
        require(result.returncode == 1 and result.stderr and not result.stdout,
                f"Invalid arguments did not return a clean error: {args}")
    result = subprocess.run(
        [str(CLI), "-d", str(model_dir)], input=b"hello\x00world\n",
        capture_output=True, timeout=30,
    )
    require(result.returncode == 1 and b"NUL" in result.stderr and not result.stdout,
            "NUL input was not rejected cleanly")

    with tempfile.TemporaryDirectory(prefix="failed-model-", dir=ROOT / "build") as temp:
        base = Path(temp)
        empty = base / "empty"
        empty.mkdir()
        invalid_config = base / "invalid-config"
        invalid_config.mkdir()
        (invalid_config / "config.json").write_text("{", encoding="utf-8")
        missing_weights = base / "missing-weights"
        missing_weights.mkdir()
        shutil.copyfile(model_dir / "config.json", missing_weights / "config.json")
        corrupt_weights = base / "corrupt-weights"
        corrupt_weights.mkdir()
        shutil.copyfile(model_dir / "config.json", corrupt_weights / "config.json")
        (corrupt_weights / "model.safetensors").write_bytes(b"invalid weights")
        missing_tokenizer = base / "missing-tokenizer"
        missing_tokenizer.mkdir()
        shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
        (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
        directories = [base / "absent", empty, invalid_config, missing_weights,
                       corrupt_weights, missing_tokenizer]
        for name, weights, tokenizer in (
            ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"),
            ("alternate-tokenizer", "model.safetensors", "tokenizer.model"),
            ("alternate-spiece", "model.safetensors", "spiece.model"),
            ("bert", "model.safetensors", "sentencepiece.bpe.model"),
        ):
            directory = base / name
            directory.mkdir()
            config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
            if name == "bert":
                config["model_type"] = "bert"
                config["architectures"] = ["BertForSequenceClassification"]
            (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
            (directory / weights).symlink_to(model_dir / "model.safetensors")
            (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model")
            directories.append(directory)
        for directory in directories:
            result = subprocess.run(
                [str(CLI), "-d", str(directory), "hello"],
                capture_output=True, timeout=30,
            )
            require(result.returncode == 1 and result.stderr and not result.stdout,
                    f"Failed model load did not return a clean error: {directory.name}")
    print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.",
          flush=True)


def check_reference(model_dir):
    torch.set_num_threads(1)
    tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
    require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
    cases = list(CASES)
    # Boundary cases straddle the BLAS attention threshold and the token limit.
    unit = "test "
    require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
            "Boundary fixture must encode to one token")
    cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513))
    texts = [text for _, text in cases]
    native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
    require(len(native_ids) == len(cases), "Missing native tokenization results")
    for (name, text), ids in zip(cases, native_ids):
        expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
        if name.startswith("length-"):
            require(len(expected) == int(name.split("-")[1]), f"{name}: invalid boundary fixture")
        if name == "long":
            require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
        require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
    print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)

    model = AutoModelForSequenceClassification.from_pretrained(
        model_dir, local_files_only=True, dtype=torch.float32,
        attn_implementation="eager",
    ).to("cpu").eval()
    labels = [model.config.id2label[i] for i in range(model.config.num_labels)]
    require(len(labels) == 20, "Expected 20 language labels")
    expected_scores = []
    encoded_ids = []
    with torch.inference_mode():
        for text in texts:
            inputs = tokenizer(text, truncation=True, max_length=MAX_TOKENS, return_tensors="pt")
            encoded_ids.append(inputs["input_ids"][0].tolist())
            expected_scores.append(torch.softmax(model(**inputs).logits[0], dim=-1).tolist())
    del model

    worst_error = 0.0
    for threads in (1, 4):
        native = json_lines([CLI, "-d", model_dir, "-t", threads, "--all", "--", *texts])
        require(len(native) == len(cases), "Missing native predictions")
        for index, ((name, _), row, expected) in enumerate(zip(cases, native, expected_scores)):
            ids = native_ids[index]
            kept = ids if len(ids) <= MAX_TOKENS else ids[:MAX_TOKENS - 1] + [ids[-1]]
            require(kept == encoded_ids[index], f"{name}: truncated token IDs differ")
            require(row["tokens"] == len(kept), f"{name}: wrong token count")
            require(row["truncated"] == (len(ids) > MAX_TOKENS), f"{name}: wrong truncation flag")
            require(set(row["scores"]) == set(labels), f"{name}: wrong label mapping")
            scores = [row["scores"][label] for label in labels]
            require(all(math.isfinite(p) and 0 <= p <= 1 for p in scores), f"{name}: invalid scores")
            require(abs(sum(scores) - 1) <= 1e-5, f"{name}: probabilities do not sum to one")
            error = max(abs(actual - ref) for actual, ref in zip(scores, expected))
            worst_error = max(worst_error, error)
            require(error <= SCORE_ATOL,
                    f"{name}, threads={threads}: score error {error:.8g} exceeds {SCORE_ATOL}")
            best = max(range(len(labels)), key=lambda i: expected[i])
            require(row["label"] == labels[best], f"{name}: predicted language differs")
            require(row["score"] == row["scores"][row["label"]], f"{name}: inconsistent best score")
        print(f"PASS: reference scores and labels for {len(cases)} inputs, threads={threads}.", flush=True)

    stream_texts = [CASES[0][1], CASES[1][1], "", "   "]
    from_args = json_lines([CLI, "-d", model_dir, "--all", "--", *stream_texts])
    from_stdin = json_lines([CLI, "-d", model_dir, "--all"], input="\n".join(stream_texts) + "\n")
    for row in from_args + from_stdin:
        row.pop("ms")
    require(from_args == from_stdin, "Argument and stdin predictions differ")
    print("PASS: argument and stdin predictions match.", flush=True)
    print(f"Maximum absolute probability difference: {worst_error:.8g} (limit {SCORE_ATOL}).", flush=True)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--model-dir", type=Path, default=ROOT / "model")
    args = parser.parse_args()
    model_dir = args.model_dir.resolve()
    require((model_dir / "model.safetensors").is_file(), "Download the checkpoint before testing")
    print(f"Reference: torch {torch.__version__}, transformers {transformers.__version__}, "
          f"tokenizers {tokenizers.__version__}; CPU F32, eager attention.", flush=True)
    check_failures(model_dir)
    check_reference(model_dir)


if __name__ == "__main__":
    main()