aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
commit6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (patch)
tree427e45b901aec902ceba72428fe5de9689d3001b /tools
parentcb1a04331b27cc19445de63cedcd4fd30b8a9016 (diff)
downloadlangdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.tar.gz
langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.zip
Add CPU reference comparison and repeatable API tests
Diffstat (limited to 'tools')
-rw-r--r--tools/test_api.c91
-rw-r--r--tools/token_ids.c33
-rw-r--r--tools/verify.py201
3 files changed, 325 insertions, 0 deletions
diff --git a/tools/test_api.c b/tools/test_api.c
new file mode 100644
index 0000000..6923102
--- /dev/null
+++ b/tools/test_api.c
@@ -0,0 +1,91 @@
+#include "langdet.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
+ exit(1); \
+ } \
+} while (0)
+
+static void check_scores(const langdet_result_t *result) {
+ CHECK(result->label);
+ float sum = 0;
+ int best = 0;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ CHECK(result->labels[i]);
+ CHECK(isfinite(result->scores[i]));
+ CHECK(result->scores[i] >= 0 && result->scores[i] <= 1);
+ sum += result->scores[i];
+ if (result->scores[i] > result->scores[best])
+ best = i;
+ }
+ CHECK(fabsf(sum - 1) < 1e-5f);
+ CHECK(strcmp(result->label, result->labels[best]) == 0);
+ CHECK(result->score == result->scores[best]);
+}
+
+int main(int argc, char **argv) {
+ CHECK(argc == 2);
+ const char *model_dir = argv[1];
+ char error[256];
+ CHECK(!langdet_open(NULL, 0, error, sizeof(error)) && error[0]);
+ CHECK(!langdet_open(model_dir, -1, error, sizeof(error)) && error[0]);
+ CHECK(!langdet_open(model_dir, 17, error, sizeof(error)) && error[0]);
+ langdet_free(NULL);
+
+ langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error));
+ CHECK(first && !error[0]);
+ langdet_result_t result;
+ const char *english = "This is an English sentence.";
+ CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
+ CHECK(!error[0] && strcmp(result.label, "en") == 0);
+ CHECK(!result.truncated);
+ check_scores(&result);
+ langdet_result_t saved = result;
+
+ for (int i = 0; i < 3; i++) {
+ CHECK(langdet_predict(first, english, &result, NULL, 0) == 0);
+ CHECK(result.score == saved.score);
+ CHECK(result.tokens == saved.tokens);
+ CHECK(memcmp(result.scores, saved.scores, sizeof(result.scores)) == 0);
+ }
+ CHECK(langdet_predict(first, "Bugün hava çok güzel.", &result, error, sizeof(error)) == 0);
+ CHECK(strcmp(result.label, "tr") == 0 && strcmp(saved.label, "en") == 0);
+ check_scores(&saved);
+
+ CHECK(langdet_predict(first, "", &result, error, sizeof(error)) == 0);
+ CHECK(result.tokens == 2 && !result.truncated);
+ check_scores(&result);
+ char long_text[3601];
+ for (int i = 0; i < 600; i++)
+ memcpy(long_text + i * 6, "hello ", 6);
+ long_text[3600] = '\0';
+ CHECK(langdet_predict(first, long_text, &result, error, sizeof(error)) == 0);
+ CHECK(result.tokens == LANGDET_MAX_TOKENS && result.truncated);
+ check_scores(&result);
+
+ CHECK(langdet_predict(first, NULL, &result, error, sizeof(error)) == -1 && error[0]);
+ CHECK(!result.label && result.score == 0 && result.tokens == 0 && !result.truncated);
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++)
+ CHECK(!result.labels[i] && result.scores[i] == 0);
+ CHECK(langdet_predict(first, english, NULL, error, sizeof(error)) == -1 && error[0]);
+ char tiny[2] = {'x', 'y'};
+ CHECK(langdet_predict(NULL, english, &result, tiny, 1) == -1);
+ CHECK(tiny[0] == '\0' && tiny[1] == 'y');
+
+ langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error));
+ CHECK(second && !error[0]);
+ CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
+ CHECK(result.score == saved.score);
+ langdet_free(first);
+ CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0);
+ CHECK(result.score == saved.score);
+ langdet_free(second);
+ puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle.");
+ return 0;
+}
diff --git a/tools/token_ids.c b/tools/token_ids.c
new file mode 100644
index 0000000..a8d9948
--- /dev/null
+++ b/tools/token_ids.c
@@ -0,0 +1,33 @@
+#include "encoder.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+ if (argc < 3) {
+ fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
+ return 1;
+ }
+ char error[256] = {0};
+ langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error));
+ if (!tokenizer) {
+ fprintf(stderr, "%s\n", error);
+ return 1;
+ }
+ for (int i = 2; i < argc; i++) {
+ int count = 0;
+ int *ids = langdet_tokenize(tokenizer, argv[i], &count);
+ if (!ids) {
+ fprintf(stderr, "Tokenization failed for argument %d\n", i);
+ langdet_tok_free(tokenizer);
+ return 1;
+ }
+ putchar('[');
+ for (int j = 0; j < count; j++)
+ printf("%s%d", j ? "," : "", ids[j]);
+ puts("]");
+ free(ids);
+ }
+ langdet_tok_free(tokenizer);
+ return fflush(stdout) == 0 ? 0 : 1;
+}
diff --git a/tools/verify.py b/tools/verify.py
new file mode 100644
index 0000000..197f68f
--- /dev/null
+++ b/tools/verify.py
@@ -0,0 +1,201 @@
+# /// script
+# requires-python = ">=3.10,<3.14"
+# dependencies = [
+# "torch==2.8.0",
+# "transformers==4.56.2",
+# "tokenizers==0.22.0",
+# ]
+# ///
+"""Compare native tokenization and F32 language scores with Transformers on CPU."""
+
+import argparse
+import json
+import math
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+
+import tokenizers
+import torch
+import transformers
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+ROOT = Path(__file__).resolve().parent.parent
+CLI = ROOT / "langdetect"
+TOKEN_IDS = ROOT / "build" / "token_ids"
+MAX_TOKENS = 512
+SCORE_ATOL = 1e-4
+CASES = [
+ ("en", "This is an English sentence about language detection."),
+ ("tr", "Bugün hava çok güzel ve arkadaşlarımla dışarıda yürümek istiyorum."),
+ ("fr", "Bonjour, ceci est une phrase écrite en français."),
+ ("ja", "これは日本語で書かれた文章です。今日は友達と公園に行きます。"),
+ ("nl", "Dit is een Nederlandse zin. Vandaag gaan we met vrienden naar het park."),
+ ("ar", "هذه جملة باللغة العربية. أريد أن أتعلم المزيد عن تاريخ هذه المدينة."),
+ ("pl", "To jest zdanie napisane po polsku. Dzisiaj idziemy z przyjaciółmi do parku."),
+ ("de", "Dies ist ein deutscher Satz. Heute gehen wir mit unseren Freunden in den Park."),
+ ("it", "Questa è una frase scritta in italiano. Oggi andiamo al parco con gli amici."),
+ ("pt", "Esta é uma frase escrita em português. Hoje vamos ao parque com os nossos amigos."),
+ ("es", "Esta es una frase escrita en español. Hoy vamos al parque con nuestros amigos."),
+ ("hi", "यह हिंदी में लिखा गया एक वाक्य है। आज हम अपने दोस्तों के साथ पार्क जा रहे हैं।"),
+ ("el", "Αυτή είναι μια πρόταση γραμμένη στα ελληνικά. Σήμερα πηγαίνουμε στο πάρκο με φίλους."),
+ ("ur", "یہ اردو میں لکھا ہوا ایک جملہ ہے۔ آج ہم اپنے دوستوں کے ساتھ پارک جا رہے ہیں۔"),
+ ("bg", "Това е изречение, написано на български език. Днес отиваме в парка с приятели."),
+ ("zh", "这是一个用中文写的句子。今天天气很好,我们要和朋友一起去公园。"),
+ ("ru", "Это предложение написано на русском языке. Сегодня мы идём в парк с друзьями."),
+ ("th", "วันนี้อากาศดีมาก เราจะไปเดินเล่นที่สวนสาธารณะกับเพื่อนของเรา"),
+ ("sw", "Leo hali ya hewa ni nzuri sana na tunakwenda kutembea bustanini pamoja na marafiki zetu."),
+ ("vi", "Đây là một câu được viết bằng tiếng Việt. Hôm nay chúng tôi đi công viên cùng bạn bè."),
+ ("empty", ""),
+ ("whitespace", " \t\n "),
+ ("normalization", "café cafe\u0301 FULLWIDTH 😀"),
+ ("lines", "Hello\nworld.\tHow are you?"),
+ ("long", "This is an English sentence about language detection. " * 100),
+]
+
+
+def require(condition, message):
+ if not condition:
+ raise AssertionError(message)
+
+
+def json_lines(command, **kwargs):
+ result = subprocess.run(
+ [str(arg) for arg in command], cwd=ROOT, capture_output=True,
+ text=True, timeout=120, check=True, **kwargs,
+ )
+ return [json.loads(line) for line in result.stdout.splitlines()]
+
+
+def check_failures(model_dir):
+ for args in (["-t", "0"], ["-t", "17"], ["-t", "bad"]):
+ result = subprocess.run(
+ [str(CLI), "-d", str(model_dir), *args, "hello"],
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and result.stderr and not result.stdout,
+ f"Invalid arguments did not return a clean error: {args}")
+ result = subprocess.run(
+ [str(CLI), "-d", str(model_dir)], input=b"hello\x00world\n",
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and b"NUL" in result.stderr and not result.stdout,
+ "NUL input was not rejected cleanly")
+
+ with tempfile.TemporaryDirectory(prefix="failed-model-", dir=ROOT / "build") as temp:
+ base = Path(temp)
+ empty = base / "empty"
+ empty.mkdir()
+ invalid_config = base / "invalid-config"
+ invalid_config.mkdir()
+ (invalid_config / "config.json").write_text("{", encoding="utf-8")
+ missing_weights = base / "missing-weights"
+ missing_weights.mkdir()
+ shutil.copyfile(model_dir / "config.json", missing_weights / "config.json")
+ corrupt_weights = base / "corrupt-weights"
+ corrupt_weights.mkdir()
+ shutil.copyfile(model_dir / "config.json", corrupt_weights / "config.json")
+ (corrupt_weights / "model.safetensors").write_bytes(b"invalid weights")
+ missing_tokenizer = base / "missing-tokenizer"
+ missing_tokenizer.mkdir()
+ shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
+ (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
+ for directory in (base / "absent", empty, invalid_config, missing_weights,
+ corrupt_weights, missing_tokenizer):
+ result = subprocess.run(
+ [str(CLI), "-d", str(directory), "hello"],
+ capture_output=True, timeout=30,
+ )
+ require(result.returncode == 1 and result.stderr and not result.stdout,
+ f"Failed model load did not return a clean error: {directory.name}")
+ print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True)
+
+
+def check_reference(model_dir):
+ torch.set_num_threads(1)
+ tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
+ require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
+ cases = list(CASES)
+ # Adjacent sequence lengths exercise both sides of the 512-token limit.
+ unit = "test "
+ require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
+ "Boundary fixture must encode to one token")
+ cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513))
+ texts = [text for _, text in cases]
+ native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
+ require(len(native_ids) == len(cases), "Missing native tokenization results")
+ reference_ids = []
+ for (name, text), ids in zip(cases, native_ids):
+ expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
+ if name.startswith("length-"):
+ require(len(expected) == int(name.split("-")[1]), f"{name}: invalid boundary fixture")
+ if name == "long":
+ require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
+ require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
+ reference_ids.append(expected)
+ print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)
+
+ model = AutoModelForSequenceClassification.from_pretrained(
+ model_dir, local_files_only=True, dtype=torch.float32,
+ attn_implementation="eager",
+ ).to("cpu").eval()
+ labels = [model.config.id2label[i] for i in range(model.config.num_labels)]
+ require(len(labels) == 20, "Expected 20 language labels")
+ expected_scores = []
+ encoded_ids = []
+ with torch.inference_mode():
+ for text in texts:
+ inputs = tokenizer(text, truncation=True, max_length=MAX_TOKENS, return_tensors="pt")
+ encoded_ids.append(inputs["input_ids"][0].tolist())
+ expected_scores.append(torch.softmax(model(**inputs).logits[0], dim=-1).tolist())
+ del model
+
+ worst_error = 0.0
+ for threads in (1, 4):
+ native = json_lines([CLI, "-d", model_dir, "-t", threads, "--all", "--", *texts])
+ require(len(native) == len(cases), "Missing native predictions")
+ for index, ((name, _), row, expected) in enumerate(zip(cases, native, expected_scores)):
+ ids = native_ids[index]
+ kept = ids if len(ids) <= MAX_TOKENS else ids[:MAX_TOKENS - 1] + [ids[-1]]
+ require(kept == encoded_ids[index], f"{name}: truncated token IDs differ")
+ require(row["tokens"] == len(kept), f"{name}: wrong token count")
+ require(row["truncated"] == (len(ids) > MAX_TOKENS), f"{name}: wrong truncation flag")
+ require(set(row["scores"]) == set(labels), f"{name}: wrong label mapping")
+ scores = [row["scores"][label] for label in labels]
+ require(all(math.isfinite(p) and 0 <= p <= 1 for p in scores), f"{name}: invalid scores")
+ require(abs(sum(scores) - 1) <= 1e-5, f"{name}: probabilities do not sum to one")
+ error = max(abs(actual - ref) for actual, ref in zip(scores, expected))
+ worst_error = max(worst_error, error)
+ require(error <= SCORE_ATOL,
+ f"{name}, threads={threads}: score error {error:.8g} exceeds {SCORE_ATOL}")
+ best = max(range(len(labels)), key=lambda i: expected[i])
+ require(row["label"] == labels[best], f"{name}: predicted language differs")
+ require(row["score"] == row["scores"][row["label"]], f"{name}: inconsistent best score")
+ print(f"PASS: reference scores and labels for {len(cases)} inputs, threads={threads}.", flush=True)
+
+ stream_texts = [CASES[0][1], CASES[1][1], "", " "]
+ from_args = json_lines([CLI, "-d", model_dir, "--all", "--", *stream_texts])
+ from_stdin = json_lines([CLI, "-d", model_dir, "--all"], input="\n".join(stream_texts) + "\n")
+ for row in from_args + from_stdin:
+ row.pop("ms")
+ require(from_args == from_stdin, "Argument and stdin predictions differ")
+ print("PASS: argument and stdin predictions match.", flush=True)
+ print(f"Maximum absolute probability difference: {worst_error:.8g} (limit {SCORE_ATOL}).", flush=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-dir", type=Path, default=ROOT / "model")
+ args = parser.parse_args()
+ model_dir = args.model_dir.resolve()
+ require((model_dir / "model.safetensors").is_file(), "Download the checkpoint before testing")
+ print(f"Reference: torch {torch.__version__}, transformers {transformers.__version__}, "
+ f"tokenizers {tokenizers.__version__}; CPU F32, eager attention.", flush=True)
+ check_failures(model_dir)
+ check_reference(model_dir)
+
+
+if __name__ == "__main__":
+ main()