diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:04:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:04:04 +0200 |
| commit | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (patch) | |
| tree | 427e45b901aec902ceba72428fe5de9689d3001b /tools | |
| parent | cb1a04331b27cc19445de63cedcd4fd30b8a9016 (diff) | |
| download | langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.tar.gz langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.zip | |
Add CPU reference comparison and repeatable API tests
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/test_api.c | 91 | ||||
| -rw-r--r-- | tools/token_ids.c | 33 | ||||
| -rw-r--r-- | tools/verify.py | 201 |
3 files changed, 325 insertions, 0 deletions
diff --git a/tools/test_api.c b/tools/test_api.c new file mode 100644 index 0000000..6923102 --- /dev/null +++ b/tools/test_api.c @@ -0,0 +1,91 @@ +#include "langdet.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ + exit(1); \ + } \ +} while (0) + +static void check_scores(const langdet_result_t *result) { + CHECK(result->label); + float sum = 0; + int best = 0; + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + CHECK(result->labels[i]); + CHECK(isfinite(result->scores[i])); + CHECK(result->scores[i] >= 0 && result->scores[i] <= 1); + sum += result->scores[i]; + if (result->scores[i] > result->scores[best]) + best = i; + } + CHECK(fabsf(sum - 1) < 1e-5f); + CHECK(strcmp(result->label, result->labels[best]) == 0); + CHECK(result->score == result->scores[best]); +} + +int main(int argc, char **argv) { + CHECK(argc == 2); + const char *model_dir = argv[1]; + char error[256]; + CHECK(!langdet_open(NULL, 0, error, sizeof(error)) && error[0]); + CHECK(!langdet_open(model_dir, -1, error, sizeof(error)) && error[0]); + CHECK(!langdet_open(model_dir, 17, error, sizeof(error)) && error[0]); + langdet_free(NULL); + + langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error)); + CHECK(first && !error[0]); + langdet_result_t result; + const char *english = "This is an English sentence."; + CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); + CHECK(!error[0] && strcmp(result.label, "en") == 0); + CHECK(!result.truncated); + check_scores(&result); + langdet_result_t saved = result; + + for (int i = 0; i < 3; i++) { + CHECK(langdet_predict(first, english, &result, NULL, 0) == 0); + CHECK(result.score == saved.score); + CHECK(result.tokens == saved.tokens); + CHECK(memcmp(result.scores, saved.scores, sizeof(result.scores)) == 0); + } + CHECK(langdet_predict(first, "Bugün hava çok güzel.", &result, error, sizeof(error)) == 0); + CHECK(strcmp(result.label, "tr") == 0 && strcmp(saved.label, "en") == 0); + check_scores(&saved); + + CHECK(langdet_predict(first, "", &result, error, sizeof(error)) == 0); + CHECK(result.tokens == 2 && !result.truncated); + check_scores(&result); + char long_text[3601]; + for (int i = 0; i < 600; i++) + memcpy(long_text + i * 6, "hello ", 6); + long_text[3600] = '\0'; + CHECK(langdet_predict(first, long_text, &result, error, sizeof(error)) == 0); + CHECK(result.tokens == LANGDET_MAX_TOKENS && result.truncated); + check_scores(&result); + + CHECK(langdet_predict(first, NULL, &result, error, sizeof(error)) == -1 && error[0]); + CHECK(!result.label && result.score == 0 && result.tokens == 0 && !result.truncated); + for (int i = 0; i < LANGDET_NUM_LABELS; i++) + CHECK(!result.labels[i] && result.scores[i] == 0); + CHECK(langdet_predict(first, english, NULL, error, sizeof(error)) == -1 && error[0]); + char tiny[2] = {'x', 'y'}; + CHECK(langdet_predict(NULL, english, &result, tiny, 1) == -1); + CHECK(tiny[0] == '\0' && tiny[1] == 'y'); + + langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error)); + CHECK(second && !error[0]); + CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); + CHECK(result.score == saved.score); + langdet_free(first); + CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0); + CHECK(result.score == saved.score); + langdet_free(second); + puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle."); + return 0; +} diff --git a/tools/token_ids.c b/tools/token_ids.c new file mode 100644 index 0000000..a8d9948 --- /dev/null +++ b/tools/token_ids.c @@ -0,0 +1,33 @@ +#include "encoder.h" + +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); + return 1; + } + char error[256] = {0}; + langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error)); + if (!tokenizer) { + fprintf(stderr, "%s\n", error); + return 1; + } + for (int i = 2; i < argc; i++) { + int count = 0; + int *ids = langdet_tokenize(tokenizer, argv[i], &count); + if (!ids) { + fprintf(stderr, "Tokenization failed for argument %d\n", i); + langdet_tok_free(tokenizer); + return 1; + } + putchar('['); + for (int j = 0; j < count; j++) + printf("%s%d", j ? "," : "", ids[j]); + puts("]"); + free(ids); + } + langdet_tok_free(tokenizer); + return fflush(stdout) == 0 ? 0 : 1; +} diff --git a/tools/verify.py b/tools/verify.py new file mode 100644 index 0000000..197f68f --- /dev/null +++ b/tools/verify.py @@ -0,0 +1,201 @@ +# /// script +# requires-python = ">=3.10,<3.14" +# dependencies = [ +# "torch==2.8.0", +# "transformers==4.56.2", +# "tokenizers==0.22.0", +# ] +# /// +"""Compare native tokenization and F32 language scores with Transformers on CPU.""" + +import argparse +import json +import math +from pathlib import Path +import shutil +import subprocess +import tempfile + +import tokenizers +import torch +import transformers +from transformers import AutoModelForSequenceClassification, AutoTokenizer + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "langdetect" +TOKEN_IDS = ROOT / "build" / "token_ids" +MAX_TOKENS = 512 +SCORE_ATOL = 1e-4 +CASES = [ + ("en", "This is an English sentence about language detection."), + ("tr", "Bugün hava çok güzel ve arkadaşlarımla dışarıda yürümek istiyorum."), + ("fr", "Bonjour, ceci est une phrase écrite en français."), + ("ja", "これは日本語で書かれた文章です。今日は友達と公園に行きます。"), + ("nl", "Dit is een Nederlandse zin. Vandaag gaan we met vrienden naar het park."), + ("ar", "هذه جملة باللغة العربية. أريد أن أتعلم المزيد عن تاريخ هذه المدينة."), + ("pl", "To jest zdanie napisane po polsku. Dzisiaj idziemy z przyjaciółmi do parku."), + ("de", "Dies ist ein deutscher Satz. Heute gehen wir mit unseren Freunden in den Park."), + ("it", "Questa è una frase scritta in italiano. Oggi andiamo al parco con gli amici."), + ("pt", "Esta é uma frase escrita em português. Hoje vamos ao parque com os nossos amigos."), + ("es", "Esta es una frase escrita en español. Hoy vamos al parque con nuestros amigos."), + ("hi", "यह हिंदी में लिखा गया एक वाक्य है। आज हम अपने दोस्तों के साथ पार्क जा रहे हैं।"), + ("el", "Αυτή είναι μια πρόταση γραμμένη στα ελληνικά. Σήμερα πηγαίνουμε στο πάρκο με φίλους."), + ("ur", "یہ اردو میں لکھا ہوا ایک جملہ ہے۔ آج ہم اپنے دوستوں کے ساتھ پارک جا رہے ہیں۔"), + ("bg", "Това е изречение, написано на български език. Днес отиваме в парка с приятели."), + ("zh", "这是一个用中文写的句子。今天天气很好,我们要和朋友一起去公园。"), + ("ru", "Это предложение написано на русском языке. Сегодня мы идём в парк с друзьями."), + ("th", "วันนี้อากาศดีมาก เราจะไปเดินเล่นที่สวนสาธารณะกับเพื่อนของเรา"), + ("sw", "Leo hali ya hewa ni nzuri sana na tunakwenda kutembea bustanini pamoja na marafiki zetu."), + ("vi", "Đây là một câu được viết bằng tiếng Việt. Hôm nay chúng tôi đi công viên cùng bạn bè."), + ("empty", ""), + ("whitespace", " \t\n "), + ("normalization", "café cafe\u0301 FULLWIDTH 😀"), + ("lines", "Hello\nworld.\tHow are you?"), + ("long", "This is an English sentence about language detection. " * 100), +] + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def json_lines(command, **kwargs): + result = subprocess.run( + [str(arg) for arg in command], cwd=ROOT, capture_output=True, + text=True, timeout=120, check=True, **kwargs, + ) + return [json.loads(line) for line in result.stdout.splitlines()] + + +def check_failures(model_dir): + for args in (["-t", "0"], ["-t", "17"], ["-t", "bad"]): + result = subprocess.run( + [str(CLI), "-d", str(model_dir), *args, "hello"], + capture_output=True, timeout=30, + ) + require(result.returncode == 1 and result.stderr and not result.stdout, + f"Invalid arguments did not return a clean error: {args}") + result = subprocess.run( + [str(CLI), "-d", str(model_dir)], input=b"hello\x00world\n", + capture_output=True, timeout=30, + ) + require(result.returncode == 1 and b"NUL" in result.stderr and not result.stdout, + "NUL input was not rejected cleanly") + + with tempfile.TemporaryDirectory(prefix="failed-model-", dir=ROOT / "build") as temp: + base = Path(temp) + empty = base / "empty" + empty.mkdir() + invalid_config = base / "invalid-config" + invalid_config.mkdir() + (invalid_config / "config.json").write_text("{", encoding="utf-8") + missing_weights = base / "missing-weights" + missing_weights.mkdir() + shutil.copyfile(model_dir / "config.json", missing_weights / "config.json") + corrupt_weights = base / "corrupt-weights" + corrupt_weights.mkdir() + shutil.copyfile(model_dir / "config.json", corrupt_weights / "config.json") + (corrupt_weights / "model.safetensors").write_bytes(b"invalid weights") + missing_tokenizer = base / "missing-tokenizer" + missing_tokenizer.mkdir() + shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json") + (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors") + for directory in (base / "absent", empty, invalid_config, missing_weights, + corrupt_weights, missing_tokenizer): + result = subprocess.run( + [str(CLI), "-d", str(directory), "hello"], + capture_output=True, timeout=30, + ) + require(result.returncode == 1 and result.stderr and not result.stdout, + f"Failed model load did not return a clean error: {directory.name}") + print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True) + + +def check_reference(model_dir): + torch.set_num_threads(1) + tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True) + require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer") + cases = list(CASES) + # Adjacent sequence lengths exercise both sides of the 512-token limit. + unit = "test " + require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1, + "Boundary fixture must encode to one token") + cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513)) + texts = [text for _, text in cases] + native_ids = json_lines([TOKEN_IDS, model_dir, *texts]) + require(len(native_ids) == len(cases), "Missing native tokenization results") + reference_ids = [] + for (name, text), ids in zip(cases, native_ids): + expected = tokenizer(text, truncation=False, verbose=False)["input_ids"] + if name.startswith("length-"): + require(len(expected) == int(name.split("-")[1]), f"{name}: invalid boundary fixture") + if name == "long": + require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit") + require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}") + reference_ids.append(expected) + print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True) + + model = AutoModelForSequenceClassification.from_pretrained( + model_dir, local_files_only=True, dtype=torch.float32, + attn_implementation="eager", + ).to("cpu").eval() + labels = [model.config.id2label[i] for i in range(model.config.num_labels)] + require(len(labels) == 20, "Expected 20 language labels") + expected_scores = [] + encoded_ids = [] + with torch.inference_mode(): + for text in texts: + inputs = tokenizer(text, truncation=True, max_length=MAX_TOKENS, return_tensors="pt") + encoded_ids.append(inputs["input_ids"][0].tolist()) + expected_scores.append(torch.softmax(model(**inputs).logits[0], dim=-1).tolist()) + del model + + worst_error = 0.0 + for threads in (1, 4): + native = json_lines([CLI, "-d", model_dir, "-t", threads, "--all", "--", *texts]) + require(len(native) == len(cases), "Missing native predictions") + for index, ((name, _), row, expected) in enumerate(zip(cases, native, expected_scores)): + ids = native_ids[index] + kept = ids if len(ids) <= MAX_TOKENS else ids[:MAX_TOKENS - 1] + [ids[-1]] + require(kept == encoded_ids[index], f"{name}: truncated token IDs differ") + require(row["tokens"] == len(kept), f"{name}: wrong token count") + require(row["truncated"] == (len(ids) > MAX_TOKENS), f"{name}: wrong truncation flag") + require(set(row["scores"]) == set(labels), f"{name}: wrong label mapping") + scores = [row["scores"][label] for label in labels] + require(all(math.isfinite(p) and 0 <= p <= 1 for p in scores), f"{name}: invalid scores") + require(abs(sum(scores) - 1) <= 1e-5, f"{name}: probabilities do not sum to one") + error = max(abs(actual - ref) for actual, ref in zip(scores, expected)) + worst_error = max(worst_error, error) + require(error <= SCORE_ATOL, + f"{name}, threads={threads}: score error {error:.8g} exceeds {SCORE_ATOL}") + best = max(range(len(labels)), key=lambda i: expected[i]) + require(row["label"] == labels[best], f"{name}: predicted language differs") + require(row["score"] == row["scores"][row["label"]], f"{name}: inconsistent best score") + print(f"PASS: reference scores and labels for {len(cases)} inputs, threads={threads}.", flush=True) + + stream_texts = [CASES[0][1], CASES[1][1], "", " "] + from_args = json_lines([CLI, "-d", model_dir, "--all", "--", *stream_texts]) + from_stdin = json_lines([CLI, "-d", model_dir, "--all"], input="\n".join(stream_texts) + "\n") + for row in from_args + from_stdin: + row.pop("ms") + require(from_args == from_stdin, "Argument and stdin predictions differ") + print("PASS: argument and stdin predictions match.", flush=True) + print(f"Maximum absolute probability difference: {worst_error:.8g} (limit {SCORE_ATOL}).", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", type=Path, default=ROOT / "model") + args = parser.parse_args() + model_dir = args.model_dir.resolve() + require((model_dir / "model.safetensors").is_file(), "Download the checkpoint before testing") + print(f"Reference: torch {torch.__version__}, transformers {transformers.__version__}, " + f"tokenizers {tokenizers.__version__}; CPU F32, eager attention.", flush=True) + check_failures(model_dir) + check_reference(model_dir) + + +if __name__ == "__main__": + main() |
