aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--Makefile18
-rw-r--r--README.md168
-rw-r--r--langdetect.c2
-rw-r--r--liblangdet/langdet.c3
-rw-r--r--liblangdet/langdet.h2
-rw-r--r--liblangdet/threadpool.c3
-rw-r--r--tools/linux.Dockerfile19
-rwxr-xr-xtools/test-linux.sh30
-rw-r--r--tools/test_api.c16
-rw-r--r--tools/verify.py6
10 files changed, 178 insertions, 89 deletions
diff --git a/Makefile b/Makefile
index 7ea80f8..03f76db 100644
--- a/Makefile
+++ b/Makefile
@@ -3,10 +3,24 @@ AR ?= ar
UV ?= uv
PYTHON ?= python3
MODEL ?= model
-CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK
+CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS
+
+SYSTEM := $(shell uname -s)
+ifeq ($(SYSTEM),Darwin)
+CPPFLAGS += -DACCELERATE_NEW_LAPACK
CFLAGS ?= -O3 -march=native
+LDLIBS += -framework Accelerate
+else ifeq ($(SYSTEM),Linux)
+PKG_CONFIG ?= pkg-config
+CPPFLAGS += -D_POSIX_C_SOURCE=200809L -DUSE_OPENBLAS $(shell $(PKG_CONFIG) --cflags openblas)
+CFLAGS ?= -O3
+LDLIBS += $(shell $(PKG_CONFIG) --libs openblas)
+else
+$(error Unsupported operating system: $(SYSTEM))
+endif
+
CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread
-LDLIBS += -framework Accelerate -lm -pthread
+LDLIBS += -lm -pthread
CPU_SOURCES = langdet.c model.c model_blas.c forward.c alloc.c config.c \
tokenize.c sentencepiece.c safetensors.c \
diff --git a/README.md b/README.md
index 0735571..cf952c4 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,31 @@
-# XLM-R language detection on macOS CPU
+# XLM-R language detection
-A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the
-liblangdet prediction API, ARM NEON kernels, and Apple Accelerate BLAS. It returns language
-predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python
-runtime is required for inference.
+A native C library and CLI for `papluca/xlm-roberta-base-language-detection`.
+It uses F32 CPU inference with Apple Accelerate on macOS and OpenBLAS on Linux,
+and returns predictions for the checkpoint's 20 languages.
## Build and run
-Use an Apple Silicon Mac with Xcode Command Line Tools installed
-(`xcode-select --install` if needed). Python 3 is used for downloading the model
-and running the optional reference checks, not for native inference.
+### macOS (Apple Silicon)
+
+Install Xcode Command Line Tools (`xcode-select --install` if needed).
+Apple Accelerate is included with macOS.
+
+### Linux (ARM64 or x86-64)
+
+Install a C compiler, Make, `pkg-config`, and pthread OpenBLAS development files.
+On Ubuntu or Debian:
+
+```sh
+sudo apt-get update
+sudo apt-get install build-essential pkg-config libopenblas-pthread-dev python3 ca-certificates
+```
+
+The Makefile selects OpenBLAS on Linux using `pkg-config`. ARM64 uses NEON.
+The default Linux build does not require AVX2; OpenBLAS selects its CPU kernels
+at runtime. Optional compiler flags can enable AVX2/FMA in the attention code.
+
+### Download and run
```sh
python3 download_model.py
@@ -21,9 +37,6 @@ printf '%s\n' 'Bonjour tout le monde.' 'これは日本語です。' | ./langdet
Run these commands from this directory. The default model path is `./model`;
use `-d DIRECTORY` for a different path.
-The Makefile currently targets macOS ARM64. The source includes Linux/OpenBLAS
-and x86 paths, but Linux still needs build configuration and testing.
-
The model is loaded once per process. Each text argument, or each stdin line
when there are no text arguments, produces one JSON line in input order:
@@ -34,32 +47,19 @@ when there are no text arguments, produces one JSON line in input order:
- `ms` measures tokenization and inference, excluding model loading and output.
`--all` includes a `scores` object with probabilities for every language.
-`-t N` selects 1 to 16 CPU kernel threads; the default is the detected CPU count,
-capped at 16. Accelerate manages its own BLAS threads. `-h` prints help.
-Use `--` before text that begins with a hyphen. Diagnostics go to stderr.
-
-## Inference
-
-`liblangdet/langdet.c` loads and validates the four F32 classifier tensors and the
-`id2label` mapping. `langdetect.c` handles only command-line input and JSON output. The encoder accepts F32 XLM-R weights with exact GeLU activation;
-other model families and reduced-precision weights are not supported.
-The encoder uses first-token pooling without L2 normalization.
-The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer,
-and stable softmax. Dropout is disabled during inference.
-
-The native loader requires `config.json`, one `model.safetensors` file, and
-`sentencepiece.bpe.model`. Sharded weights and alternate tokenizer files are not
-supported. The safetensors and configuration readers both use `yyjson`.
+`-t N` selects 1 to 16 CPU threads; the default is the detected CPU count,
+capped at 16. On macOS, it sets the internal kernel worker count while Accelerate
+manages its own BLAS threads. On Linux, it sets the OpenBLAS thread count and
+keeps the auxiliary kernel pool serial to avoid nested parallelism.
+`-h` prints help. Use `--` before text that begins with a hyphen. Diagnostics go
+to stderr.
-The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in
-`deps/yyjson/`. Their license files are included. The build uses only local source
-files and the macOS SDK, with no dependency on another repository.
+## Input limits
-Inputs are processed individually, not as a parallel batch. Long inputs are
-truncated to 512 tokens while preserving the final `</s>`. Empty inputs are
-accepted, but their predictions are not useful. The model always chooses among
-its 20 labels; it has no unknown-language result, and scores are not a guarantee
-of accuracy on ambiguous or unsupported text.
+Inputs longer than 512 tokens are truncated while preserving the final `</s>`.
+Empty inputs are accepted, but their predictions are not useful. The model
+always chooses among its 20 labels; it has no unknown-language result, and
+scores are not a guarantee of accuracy on ambiguous or unsupported text.
## CPU library
@@ -82,16 +82,24 @@ Use one API call at a time per process, including when multiple detectors are
loaded. Internal CPU workers may cooperate on a prediction, but simultaneous calls
from application threads are not supported.
-`examples/detect.c` is a complete application using only the public header:
+`examples/detect.c` is a complete application using only the public header.
+Link it on macOS with:
```sh
cc -Iliblangdet examples/detect.c liblangdet.a -framework Accelerate -lm -pthread -o build/detect-example
-./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.'
```
-The archive includes its JSON parser. No separate parser library or private
-inference headers are needed. The encoder, tokenizer, classifier, and thread
-configuration are managed by the detector.
+On Linux, link OpenBLAS instead:
+
+```sh
+cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example
+```
+
+Then run it on either platform:
+
+```sh
+./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.'
+```
## Download
@@ -104,16 +112,13 @@ python3 download_model.py
python3 download_model.py ./models/language-detection
```
-The default destination is `model/` beside the script. The download uses revision
-`9865598389ca9d95637462f743f683b51d75b87b` and includes safetensors weights,
-configuration, tokenizer files, and the model card. It excludes duplicate PyTorch
-and TensorFlow weights. Allow about 1.13 GB of disk space for a fresh download.
+The default destination is `model/` beside the script. The checkpoint revision is
+`9865598389ca9d95637462f743f683b51d75b87b`. Allow about 1.13 GB of disk space.
+The library needs `config.json`, `model.safetensors`, and `sentencepiece.bpe.model`
+in the model directory.
-The script checks each file's size and hash against Hugging Face metadata. Large
-LFS files use SHA-256; ordinary files use Git blob SHA-1. Verified files are reused.
-Downloads go to `.part` files and become final files only after verification.
-Interrupted files restart on the next run; they do not resume from their previous
-byte offset.
+The downloader verifies file hashes and reuses verified files. Interrupted
+downloads restart on the next run.
## Tests
@@ -125,43 +130,38 @@ make test
```
Use `make test MODEL=/path/to/model PYTHON=python3.12` to select a different
-checkpoint directory or Python executable. The first run needs internet access
-for the test-only Python packages. `uv` manages the pinned dependencies declared
-in `tools/verify.py`; its cache stays in `.uv-cache/` inside this project.
-The Transformers reference uses only the local model files and runs offline on
-CPU in F32, with dropout disabled and eager attention.
+checkpoint directory or Python executable. Tests check the public API and
+compare token IDs and language scores with Transformers on CPU.
+The first run needs internet access for Python dependencies, which `uv` caches
+in `.uv-cache/` inside this project.
-The target builds and runs:
+### Linux tests through Docker
-- `tools/test_api.c`: repeated predictions, score normalization, result ownership,
- empty input, truncation, invalid arguments, error buffers, and detector lifecycle.
-- `tools/test_safetensors.c`: single-file tensor loading, metadata, lookup, and
- rejection of malformed shapes, offsets, and JSON headers.
-- `tools/token_ids.c`: a test-only adapter that returns the native token IDs.
-- `tools/verify.py`: exact token-ID comparison and all 20 language scores against
- Transformers on 31 inputs. These cover the 20 languages, empty and whitespace
- input, Unicode normalization, multiple lines, and lengths below, at, and above
- the 128-token BLAS threshold and the 512-token limit. Scores must differ by no more than `1e-4` in absolute value,
- and the predicted labels must match, with both one and four CPU kernel threads.
+With Docker running and `model/` downloaded, run:
+
+```sh
+./tools/test-linux.sh arm64
+./tools/test-linux.sh amd64
+```
+
+Without an architecture argument, the script uses the Docker engine's native
+architecture. On an ARM64 Mac, `amd64` uses Docker Desktop's x86-64 emulation.
+Docker Desktop must have emulation enabled for that command.
+
+The script uses Ubuntu 24.04 and mounts the source and `model/` read-only.
+Build products stay inside the container. The first image build needs internet
+access; tests run with networking disabled.
+
+Each run performs `make test`, then builds and runs the public API example as
+both C and C++. Additional arguments are passed to Make. To exercise AVX2/FMA
+on an x86-64 CPU or emulator that supports those instructions:
+
+```sh
+./tools/test-linux.sh amd64 CFLAGS='-O3 -march=x86-64-v3'
+```
-The Python checks also verify stdin behavior, invalid CLI arguments, NUL rejection,
-and failures for missing or invalid configuration, weights, and tokenizer files.
-They also check rejection of sharded weights, alternate tokenizer filenames, and
-BERT model configuration.
-No Python package is linked into or required by the native library.
+## License
-## Verification status
+MIT License.
-- The executable builds and runs as native macOS ARM64, linked to Accelerate and
- the system C library, with no GPU runtime.
-- CLI smoke checks pass for one sample in each of the 20 supported languages,
- normalized scores, stdin and argument input, empty and whitespace-only input,
- single-thread inference, and 512-token truncation.
-- Invalid thread counts, a missing model directory, and NUL bytes in stdin
- produce errors.
-- Standalone C and C++ applications link using only `langdet.h` and the static
- archive. Prediction results, error returns, and detector lifecycle checks pass.
-- The downloaded checkpoint files match Hugging Face's sizes and hashes.
-- The reference comparisons pass on the 31 test inputs: token IDs match exactly,
- labels agree, and probability differences stay below the `1e-4` tolerance.
- These regression checks are not an accuracy benchmark or exhaustive tokenizer test.
+The model weights are `papluca/xlm-roberta-base-language-detection` which is based on `FacebookAI/xlm-roberta-base`; see the [model card](https://huggingface.co/papluca/xlm-roberta-base-language-detection).
diff --git a/langdetect.c b/langdetect.c
index 76d74ab..7cccbf3 100644
--- a/langdetect.c
+++ b/langdetect.c
@@ -61,7 +61,7 @@ static void usage(FILE *out, const char *program) {
"Without TEXT arguments, read one UTF-8 text per line from stdin.\n"
"Write one JSON result per text, in input order.\n"
" -d DIR Model directory (default: ./model)\n"
- " -t N CPU kernel threads, 1-16 (default: detected CPU count, capped at 16)\n"
+ " -t N CPU threads, 1-16 (default: detected CPU count, capped at 16)\n"
" --all Include scores for all 20 languages\n"
" -h Show help\n"
"Inputs longer than 512 tokens are truncated, retaining the final </s>.\n",
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
index 0fe5433..13a6359 100644
--- a/liblangdet/langdet.c
+++ b/liblangdet/langdet.c
@@ -143,8 +143,7 @@ int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result,
if (!d || !text || !result)
return set_error(err, errlen, "Detector, text, and result must not be NULL");
- if (tp_num_threads() != d->n_threads)
- tp_set_threads(d->n_threads);
+ tp_set_threads(d->n_threads);
int count = 0;
int *ids = langdet_tokenize(d->tokenizer, text, &count);
if (!ids || count < 2) {
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h
index 22e0b64..d5cb9b3 100644
--- a/liblangdet/langdet.h
+++ b/liblangdet/langdet.h
@@ -28,6 +28,8 @@ typedef struct {
/* Load the model, tokenizer, classifier, and labels from model_dir.
* n_threads is 1..16, or 0 to detect the CPU count and cap it at 16.
+ * On macOS it sets kernel workers; Accelerate manages its own threads.
+ * On Linux it sets OpenBLAS threads; the auxiliary kernel pool stays serial.
* Return NULL on failure. An optional err[errlen] receives a NUL-terminated
* message on failure and is cleared on success. Model-file readers may also
* write diagnostics to stderr. */
diff --git a/liblangdet/threadpool.c b/liblangdet/threadpool.c
index 6cb4252..e0ffbc6 100644
--- a/liblangdet/threadpool.c
+++ b/liblangdet/threadpool.c
@@ -93,6 +93,9 @@ void tp_set_threads(int n) {
n = 1;
#endif
+ if (n == tp.n_threads)
+ return;
+
/* Shutdown existing workers */
if (tp.n_threads > 1) {
pthread_mutex_lock(&tp.mutex);
diff --git a/tools/linux.Dockerfile b/tools/linux.Dockerfile
new file mode 100644
index 0000000..2d1b93d
--- /dev/null
+++ b/tools/linux.Dockerfile
@@ -0,0 +1,19 @@
+FROM ghcr.io/astral-sh/uv:0.11.19 AS uv
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential pkg-config libopenblas-pthread-dev python3 python3-venv ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=uv /uv /usr/local/bin/uv
+
+WORKDIR /work
+ENV UV_CACHE_DIR=/work/.uv-cache \
+ UV_PYTHON_DOWNLOADS=never \
+ HF_HOME=/work/.hf-cache \
+ HF_HUB_OFFLINE=1 \
+ TRANSFORMERS_OFFLINE=1 \
+ TOKENIZERS_PARALLELISM=false
+COPY verify.py tools/verify.py
+RUN uv --no-config run --no-project --python python3 --script tools/verify.py --help
+
+CMD ["make", "-j4", "test"]
diff --git a/tools/test-linux.sh b/tools/test-linux.sh
new file mode 100755
index 0000000..9431c1f
--- /dev/null
+++ b/tools/test-linux.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
+arch=${1:-$(docker info --format '{{.Architecture}}')}
+case "$arch" in
+ arm64|aarch64) arch=arm64 ;;
+ amd64|x86_64) arch=amd64 ;;
+ *) printf 'Usage: %s [arm64|amd64] [MAKE_ARGUMENT ...]\n' "$0" >&2; exit 2 ;;
+esac
+if [ "$#" -gt 0 ]; then shift; fi
+
+if [ ! -f "$root/model/model.safetensors" ]; then
+ printf 'Download the model first: python3 %s/download_model.py\n' "$root" >&2
+ exit 1
+fi
+
+image="langdet-linux-test:$arch"
+docker build --platform "linux/$arch" -f "$root/tools/linux.Dockerfile" -t "$image" "$root/tools"
+args=(run --rm --network none --platform "linux/$arch" --workdir /work)
+for path in Makefile langdetect.c liblangdet deps tools examples model; do
+ args+=(--mount "type=bind,src=$root/$path,dst=/work/$path,readonly")
+done
+docker "${args[@]}" "$image" sh -eu -c '
+ make -j4 test "$@"
+ cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example
+ c++ -Iliblangdet -x c++ examples/detect.c -x none liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example-cpp
+ ./build/detect-example model "This is an English sentence." "Bugün hava çok güzel."
+ ./build/detect-example-cpp model "This is an English sentence." "Bugün hava çok güzel."
+' linux-tests "$@"
diff --git a/tools/test_api.c b/tools/test_api.c
index 6923102..18f68b6 100644
--- a/tools/test_api.c
+++ b/tools/test_api.c
@@ -5,6 +5,10 @@
#include <stdlib.h>
#include <string.h>
+#ifdef USE_OPENBLAS
+#include <cblas.h>
+#endif
+
#define CHECK(condition) do { \
if (!(condition)) { \
fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
@@ -40,6 +44,9 @@ int main(int argc, char **argv) {
langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error));
CHECK(first && !error[0]);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 1);
+#endif
langdet_result_t result;
const char *english = "This is an English sentence.";
CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
@@ -80,10 +87,19 @@ int main(int argc, char **argv) {
langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error));
CHECK(second && !error[0]);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 4);
+#endif
CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 1);
+#endif
CHECK(result.score == saved.score);
langdet_free(first);
CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 4);
+#endif
CHECK(result.score == saved.score);
langdet_free(second);
puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle.");
diff --git a/tools/verify.py b/tools/verify.py
index d8bccf4..8339ac9 100644
--- a/tools/verify.py
+++ b/tools/verify.py
@@ -5,6 +5,12 @@
# "transformers==4.56.2",
# "tokenizers==0.22.0",
# ]
+# [tool.uv.sources]
+# torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
+# [[tool.uv.index]]
+# name = "pytorch-cpu"
+# url = "https://download.pytorch.org/whl/cpu"
+# explicit = true
# ///
"""Compare native tokenization and F32 language scores with Transformers on CPU."""