diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 15:41:41 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 16:06:03 +0200 |
| commit | 939b9a8deae07c2f06aa16c08f3ddad70f97b261 (patch) | |
| tree | 00245f8e747349ddcca171be878e5ac19e8110a4 | |
| parent | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (diff) | |
| download | langdet-master.tar.gz langdet-master.zip | |
Diffstat (limited to '')
| -rw-r--r-- | Makefile | 18 | ||||
| -rw-r--r-- | README.md | 168 | ||||
| -rw-r--r-- | langdetect.c | 2 | ||||
| -rw-r--r-- | liblangdet/langdet.c | 3 | ||||
| -rw-r--r-- | liblangdet/langdet.h | 2 | ||||
| -rw-r--r-- | liblangdet/threadpool.c | 3 | ||||
| -rw-r--r-- | tools/linux.Dockerfile | 19 | ||||
| -rwxr-xr-x | tools/test-linux.sh | 30 | ||||
| -rw-r--r-- | tools/test_api.c | 16 | ||||
| -rw-r--r-- | tools/verify.py | 6 |
10 files changed, 178 insertions, 89 deletions
@@ -3,10 +3,24 @@ AR ?= ar UV ?= uv PYTHON ?= python3 MODEL ?= model -CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK +CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS + +SYSTEM := $(shell uname -s) +ifeq ($(SYSTEM),Darwin) +CPPFLAGS += -DACCELERATE_NEW_LAPACK CFLAGS ?= -O3 -march=native +LDLIBS += -framework Accelerate +else ifeq ($(SYSTEM),Linux) +PKG_CONFIG ?= pkg-config +CPPFLAGS += -D_POSIX_C_SOURCE=200809L -DUSE_OPENBLAS $(shell $(PKG_CONFIG) --cflags openblas) +CFLAGS ?= -O3 +LDLIBS += $(shell $(PKG_CONFIG) --libs openblas) +else +$(error Unsupported operating system: $(SYSTEM)) +endif + CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread -LDLIBS += -framework Accelerate -lm -pthread +LDLIBS += -lm -pthread CPU_SOURCES = langdet.c model.c model_blas.c forward.c alloc.c config.c \ tokenize.c sentencepiece.c safetensors.c \ @@ -1,15 +1,31 @@ -# XLM-R language detection on macOS CPU +# XLM-R language detection -A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the -liblangdet prediction API, ARM NEON kernels, and Apple Accelerate BLAS. It returns language -predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python -runtime is required for inference. +A native C library and CLI for `papluca/xlm-roberta-base-language-detection`. +It uses F32 CPU inference with Apple Accelerate on macOS and OpenBLAS on Linux, +and returns predictions for the checkpoint's 20 languages. ## Build and run -Use an Apple Silicon Mac with Xcode Command Line Tools installed -(`xcode-select --install` if needed). Python 3 is used for downloading the model -and running the optional reference checks, not for native inference. +### macOS (Apple Silicon) + +Install Xcode Command Line Tools (`xcode-select --install` if needed). +Apple Accelerate is included with macOS. + +### Linux (ARM64 or x86-64) + +Install a C compiler, Make, `pkg-config`, and pthread OpenBLAS development files. +On Ubuntu or Debian: + +```sh +sudo apt-get update +sudo apt-get install build-essential pkg-config libopenblas-pthread-dev python3 ca-certificates +``` + +The Makefile selects OpenBLAS on Linux using `pkg-config`. ARM64 uses NEON. +The default Linux build does not require AVX2; OpenBLAS selects its CPU kernels +at runtime. Optional compiler flags can enable AVX2/FMA in the attention code. + +### Download and run ```sh python3 download_model.py @@ -21,9 +37,6 @@ printf '%s\n' 'Bonjour tout le monde.' 'これは日本語です。' | ./langdet Run these commands from this directory. The default model path is `./model`; use `-d DIRECTORY` for a different path. -The Makefile currently targets macOS ARM64. The source includes Linux/OpenBLAS -and x86 paths, but Linux still needs build configuration and testing. - The model is loaded once per process. Each text argument, or each stdin line when there are no text arguments, produces one JSON line in input order: @@ -34,32 +47,19 @@ when there are no text arguments, produces one JSON line in input order: - `ms` measures tokenization and inference, excluding model loading and output. `--all` includes a `scores` object with probabilities for every language. -`-t N` selects 1 to 16 CPU kernel threads; the default is the detected CPU count, -capped at 16. Accelerate manages its own BLAS threads. `-h` prints help. -Use `--` before text that begins with a hyphen. Diagnostics go to stderr. - -## Inference - -`liblangdet/langdet.c` loads and validates the four F32 classifier tensors and the -`id2label` mapping. `langdetect.c` handles only command-line input and JSON output. The encoder accepts F32 XLM-R weights with exact GeLU activation; -other model families and reduced-precision weights are not supported. -The encoder uses first-token pooling without L2 normalization. -The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer, -and stable softmax. Dropout is disabled during inference. - -The native loader requires `config.json`, one `model.safetensors` file, and -`sentencepiece.bpe.model`. Sharded weights and alternate tokenizer files are not -supported. The safetensors and configuration readers both use `yyjson`. +`-t N` selects 1 to 16 CPU threads; the default is the detected CPU count, +capped at 16. On macOS, it sets the internal kernel worker count while Accelerate +manages its own BLAS threads. On Linux, it sets the OpenBLAS thread count and +keeps the auxiliary kernel pool serial to avoid nested parallelism. +`-h` prints help. Use `--` before text that begins with a hyphen. Diagnostics go +to stderr. -The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in -`deps/yyjson/`. Their license files are included. The build uses only local source -files and the macOS SDK, with no dependency on another repository. +## Input limits -Inputs are processed individually, not as a parallel batch. Long inputs are -truncated to 512 tokens while preserving the final `</s>`. Empty inputs are -accepted, but their predictions are not useful. The model always chooses among -its 20 labels; it has no unknown-language result, and scores are not a guarantee -of accuracy on ambiguous or unsupported text. +Inputs longer than 512 tokens are truncated while preserving the final `</s>`. +Empty inputs are accepted, but their predictions are not useful. The model +always chooses among its 20 labels; it has no unknown-language result, and +scores are not a guarantee of accuracy on ambiguous or unsupported text. ## CPU library @@ -82,16 +82,24 @@ Use one API call at a time per process, including when multiple detectors are loaded. Internal CPU workers may cooperate on a prediction, but simultaneous calls from application threads are not supported. -`examples/detect.c` is a complete application using only the public header: +`examples/detect.c` is a complete application using only the public header. +Link it on macOS with: ```sh cc -Iliblangdet examples/detect.c liblangdet.a -framework Accelerate -lm -pthread -o build/detect-example -./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.' ``` -The archive includes its JSON parser. No separate parser library or private -inference headers are needed. The encoder, tokenizer, classifier, and thread -configuration are managed by the detector. +On Linux, link OpenBLAS instead: + +```sh +cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example +``` + +Then run it on either platform: + +```sh +./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.' +``` ## Download @@ -104,16 +112,13 @@ python3 download_model.py python3 download_model.py ./models/language-detection ``` -The default destination is `model/` beside the script. The download uses revision -`9865598389ca9d95637462f743f683b51d75b87b` and includes safetensors weights, -configuration, tokenizer files, and the model card. It excludes duplicate PyTorch -and TensorFlow weights. Allow about 1.13 GB of disk space for a fresh download. +The default destination is `model/` beside the script. The checkpoint revision is +`9865598389ca9d95637462f743f683b51d75b87b`. Allow about 1.13 GB of disk space. +The library needs `config.json`, `model.safetensors`, and `sentencepiece.bpe.model` +in the model directory. -The script checks each file's size and hash against Hugging Face metadata. Large -LFS files use SHA-256; ordinary files use Git blob SHA-1. Verified files are reused. -Downloads go to `.part` files and become final files only after verification. -Interrupted files restart on the next run; they do not resume from their previous -byte offset. +The downloader verifies file hashes and reuses verified files. Interrupted +downloads restart on the next run. ## Tests @@ -125,43 +130,38 @@ make test ``` Use `make test MODEL=/path/to/model PYTHON=python3.12` to select a different -checkpoint directory or Python executable. The first run needs internet access -for the test-only Python packages. `uv` manages the pinned dependencies declared -in `tools/verify.py`; its cache stays in `.uv-cache/` inside this project. -The Transformers reference uses only the local model files and runs offline on -CPU in F32, with dropout disabled and eager attention. +checkpoint directory or Python executable. Tests check the public API and +compare token IDs and language scores with Transformers on CPU. +The first run needs internet access for Python dependencies, which `uv` caches +in `.uv-cache/` inside this project. -The target builds and runs: +### Linux tests through Docker -- `tools/test_api.c`: repeated predictions, score normalization, result ownership, - empty input, truncation, invalid arguments, error buffers, and detector lifecycle. -- `tools/test_safetensors.c`: single-file tensor loading, metadata, lookup, and - rejection of malformed shapes, offsets, and JSON headers. -- `tools/token_ids.c`: a test-only adapter that returns the native token IDs. -- `tools/verify.py`: exact token-ID comparison and all 20 language scores against - Transformers on 31 inputs. These cover the 20 languages, empty and whitespace - input, Unicode normalization, multiple lines, and lengths below, at, and above - the 128-token BLAS threshold and the 512-token limit. Scores must differ by no more than `1e-4` in absolute value, - and the predicted labels must match, with both one and four CPU kernel threads. +With Docker running and `model/` downloaded, run: + +```sh +./tools/test-linux.sh arm64 +./tools/test-linux.sh amd64 +``` + +Without an architecture argument, the script uses the Docker engine's native +architecture. On an ARM64 Mac, `amd64` uses Docker Desktop's x86-64 emulation. +Docker Desktop must have emulation enabled for that command. + +The script uses Ubuntu 24.04 and mounts the source and `model/` read-only. +Build products stay inside the container. The first image build needs internet +access; tests run with networking disabled. + +Each run performs `make test`, then builds and runs the public API example as +both C and C++. Additional arguments are passed to Make. To exercise AVX2/FMA +on an x86-64 CPU or emulator that supports those instructions: + +```sh +./tools/test-linux.sh amd64 CFLAGS='-O3 -march=x86-64-v3' +``` -The Python checks also verify stdin behavior, invalid CLI arguments, NUL rejection, -and failures for missing or invalid configuration, weights, and tokenizer files. -They also check rejection of sharded weights, alternate tokenizer filenames, and -BERT model configuration. -No Python package is linked into or required by the native library. +## License -## Verification status +MIT License. -- The executable builds and runs as native macOS ARM64, linked to Accelerate and - the system C library, with no GPU runtime. -- CLI smoke checks pass for one sample in each of the 20 supported languages, - normalized scores, stdin and argument input, empty and whitespace-only input, - single-thread inference, and 512-token truncation. -- Invalid thread counts, a missing model directory, and NUL bytes in stdin - produce errors. -- Standalone C and C++ applications link using only `langdet.h` and the static - archive. Prediction results, error returns, and detector lifecycle checks pass. -- The downloaded checkpoint files match Hugging Face's sizes and hashes. -- The reference comparisons pass on the 31 test inputs: token IDs match exactly, - labels agree, and probability differences stay below the `1e-4` tolerance. - These regression checks are not an accuracy benchmark or exhaustive tokenizer test. +The model weights are `papluca/xlm-roberta-base-language-detection` which is based on `FacebookAI/xlm-roberta-base`; see the [model card](https://huggingface.co/papluca/xlm-roberta-base-language-detection). diff --git a/langdetect.c b/langdetect.c index 76d74ab..7cccbf3 100644 --- a/langdetect.c +++ b/langdetect.c @@ -61,7 +61,7 @@ static void usage(FILE *out, const char *program) { "Without TEXT arguments, read one UTF-8 text per line from stdin.\n" "Write one JSON result per text, in input order.\n" " -d DIR Model directory (default: ./model)\n" - " -t N CPU kernel threads, 1-16 (default: detected CPU count, capped at 16)\n" + " -t N CPU threads, 1-16 (default: detected CPU count, capped at 16)\n" " --all Include scores for all 20 languages\n" " -h Show help\n" "Inputs longer than 512 tokens are truncated, retaining the final </s>.\n", diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c index 0fe5433..13a6359 100644 --- a/liblangdet/langdet.c +++ b/liblangdet/langdet.c @@ -143,8 +143,7 @@ int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result, if (!d || !text || !result) return set_error(err, errlen, "Detector, text, and result must not be NULL"); - if (tp_num_threads() != d->n_threads) - tp_set_threads(d->n_threads); + tp_set_threads(d->n_threads); int count = 0; int *ids = langdet_tokenize(d->tokenizer, text, &count); if (!ids || count < 2) { diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h index 22e0b64..d5cb9b3 100644 --- a/liblangdet/langdet.h +++ b/liblangdet/langdet.h @@ -28,6 +28,8 @@ typedef struct { /* Load the model, tokenizer, classifier, and labels from model_dir. * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16. + * On macOS it sets kernel workers; Accelerate manages its own threads. + * On Linux it sets OpenBLAS threads; the auxiliary kernel pool stays serial. * Return NULL on failure. An optional err[errlen] receives a NUL-terminated * message on failure and is cleared on success. Model-file readers may also * write diagnostics to stderr. */ diff --git a/liblangdet/threadpool.c b/liblangdet/threadpool.c index 6cb4252..e0ffbc6 100644 --- a/liblangdet/threadpool.c +++ b/liblangdet/threadpool.c @@ -93,6 +93,9 @@ void tp_set_threads(int n) { n = 1; #endif + if (n == tp.n_threads) + return; + /* Shutdown existing workers */ if (tp.n_threads > 1) { pthread_mutex_lock(&tp.mutex); diff --git a/tools/linux.Dockerfile b/tools/linux.Dockerfile new file mode 100644 index 0000000..2d1b93d --- /dev/null +++ b/tools/linux.Dockerfile @@ -0,0 +1,19 @@ +FROM ghcr.io/astral-sh/uv:0.11.19 AS uv +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential pkg-config libopenblas-pthread-dev python3 python3-venv ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=uv /uv /usr/local/bin/uv + +WORKDIR /work +ENV UV_CACHE_DIR=/work/.uv-cache \ + UV_PYTHON_DOWNLOADS=never \ + HF_HOME=/work/.hf-cache \ + HF_HUB_OFFLINE=1 \ + TRANSFORMERS_OFFLINE=1 \ + TOKENIZERS_PARALLELISM=false +COPY verify.py tools/verify.py +RUN uv --no-config run --no-project --python python3 --script tools/verify.py --help + +CMD ["make", "-j4", "test"] diff --git a/tools/test-linux.sh b/tools/test-linux.sh new file mode 100755 index 0000000..9431c1f --- /dev/null +++ b/tools/test-linux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +arch=${1:-$(docker info --format '{{.Architecture}}')} +case "$arch" in + arm64|aarch64) arch=arm64 ;; + amd64|x86_64) arch=amd64 ;; + *) printf 'Usage: %s [arm64|amd64] [MAKE_ARGUMENT ...]\n' "$0" >&2; exit 2 ;; +esac +if [ "$#" -gt 0 ]; then shift; fi + +if [ ! -f "$root/model/model.safetensors" ]; then + printf 'Download the model first: python3 %s/download_model.py\n' "$root" >&2 + exit 1 +fi + +image="langdet-linux-test:$arch" +docker build --platform "linux/$arch" -f "$root/tools/linux.Dockerfile" -t "$image" "$root/tools" +args=(run --rm --network none --platform "linux/$arch" --workdir /work) +for path in Makefile langdetect.c liblangdet deps tools examples model; do + args+=(--mount "type=bind,src=$root/$path,dst=/work/$path,readonly") +done +docker "${args[@]}" "$image" sh -eu -c ' + make -j4 test "$@" + cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example + c++ -Iliblangdet -x c++ examples/detect.c -x none liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example-cpp + ./build/detect-example model "This is an English sentence." "Bugün hava çok güzel." + ./build/detect-example-cpp model "This is an English sentence." "Bugün hava çok güzel." +' linux-tests "$@" diff --git a/tools/test_api.c b/tools/test_api.c index 6923102..18f68b6 100644 --- a/tools/test_api.c +++ b/tools/test_api.c @@ -5,6 +5,10 @@ #include <stdlib.h> #include <string.h> +#ifdef USE_OPENBLAS +#include <cblas.h> +#endif + #define CHECK(condition) do { \ if (!(condition)) { \ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ @@ -40,6 +44,9 @@ int main(int argc, char **argv) { langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error)); CHECK(first && !error[0]); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 1); +#endif langdet_result_t result; const char *english = "This is an English sentence."; CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); @@ -80,10 +87,19 @@ int main(int argc, char **argv) { langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error)); CHECK(second && !error[0]); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 4); +#endif CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 1); +#endif CHECK(result.score == saved.score); langdet_free(first); CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 4); +#endif CHECK(result.score == saved.score); langdet_free(second); puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle."); diff --git a/tools/verify.py b/tools/verify.py index d8bccf4..8339ac9 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -5,6 +5,12 @@ # "transformers==4.56.2", # "tokenizers==0.22.0", # ] +# [tool.uv.sources] +# torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true # /// """Compare native tokenization and F32 language scores with Transformers on CPU.""" |
