aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 15:41:41 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 16:06:03 +0200
commit939b9a8deae07c2f06aa16c08f3ddad70f97b261 (patch)
tree00245f8e747349ddcca171be878e5ac19e8110a4 /tools
parent91c6aa07597462705a7f30ee5b97e2c7943b5616 (diff)
downloadlangdet-master.tar.gz
langdet-master.zip
Add Linux OpenBLAS builds and Docker validationHEADmaster
Diffstat (limited to 'tools')
-rw-r--r--tools/linux.Dockerfile19
-rwxr-xr-xtools/test-linux.sh30
-rw-r--r--tools/test_api.c16
-rw-r--r--tools/verify.py6
4 files changed, 71 insertions, 0 deletions
diff --git a/tools/linux.Dockerfile b/tools/linux.Dockerfile
new file mode 100644
index 0000000..2d1b93d
--- /dev/null
+++ b/tools/linux.Dockerfile
@@ -0,0 +1,19 @@
+FROM ghcr.io/astral-sh/uv:0.11.19 AS uv
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential pkg-config libopenblas-pthread-dev python3 python3-venv ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=uv /uv /usr/local/bin/uv
+
+WORKDIR /work
+ENV UV_CACHE_DIR=/work/.uv-cache \
+ UV_PYTHON_DOWNLOADS=never \
+ HF_HOME=/work/.hf-cache \
+ HF_HUB_OFFLINE=1 \
+ TRANSFORMERS_OFFLINE=1 \
+ TOKENIZERS_PARALLELISM=false
+COPY verify.py tools/verify.py
+RUN uv --no-config run --no-project --python python3 --script tools/verify.py --help
+
+CMD ["make", "-j4", "test"]
diff --git a/tools/test-linux.sh b/tools/test-linux.sh
new file mode 100755
index 0000000..9431c1f
--- /dev/null
+++ b/tools/test-linux.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
+arch=${1:-$(docker info --format '{{.Architecture}}')}
+case "$arch" in
+ arm64|aarch64) arch=arm64 ;;
+ amd64|x86_64) arch=amd64 ;;
+ *) printf 'Usage: %s [arm64|amd64] [MAKE_ARGUMENT ...]\n' "$0" >&2; exit 2 ;;
+esac
+if [ "$#" -gt 0 ]; then shift; fi
+
+if [ ! -f "$root/model/model.safetensors" ]; then
+ printf 'Download the model first: python3 %s/download_model.py\n' "$root" >&2
+ exit 1
+fi
+
+image="langdet-linux-test:$arch"
+docker build --platform "linux/$arch" -f "$root/tools/linux.Dockerfile" -t "$image" "$root/tools"
+args=(run --rm --network none --platform "linux/$arch" --workdir /work)
+for path in Makefile langdetect.c liblangdet deps tools examples model; do
+ args+=(--mount "type=bind,src=$root/$path,dst=/work/$path,readonly")
+done
+docker "${args[@]}" "$image" sh -eu -c '
+ make -j4 test "$@"
+ cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example
+ c++ -Iliblangdet -x c++ examples/detect.c -x none liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example-cpp
+ ./build/detect-example model "This is an English sentence." "Bugün hava çok güzel."
+ ./build/detect-example-cpp model "This is an English sentence." "Bugün hava çok güzel."
+' linux-tests "$@"
diff --git a/tools/test_api.c b/tools/test_api.c
index 6923102..18f68b6 100644
--- a/tools/test_api.c
+++ b/tools/test_api.c
@@ -5,6 +5,10 @@
#include <stdlib.h>
#include <string.h>
+#ifdef USE_OPENBLAS
+#include <cblas.h>
+#endif
+
#define CHECK(condition) do { \
if (!(condition)) { \
fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
@@ -40,6 +44,9 @@ int main(int argc, char **argv) {
langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error));
CHECK(first && !error[0]);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 1);
+#endif
langdet_result_t result;
const char *english = "This is an English sentence.";
CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
@@ -80,10 +87,19 @@ int main(int argc, char **argv) {
langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error));
CHECK(second && !error[0]);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 4);
+#endif
CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 1);
+#endif
CHECK(result.score == saved.score);
langdet_free(first);
CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0);
+#ifdef USE_OPENBLAS
+ CHECK(openblas_get_num_threads() == 4);
+#endif
CHECK(result.score == saved.score);
langdet_free(second);
puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle.");
diff --git a/tools/verify.py b/tools/verify.py
index d8bccf4..8339ac9 100644
--- a/tools/verify.py
+++ b/tools/verify.py
@@ -5,6 +5,12 @@
# "transformers==4.56.2",
# "tokenizers==0.22.0",
# ]
+# [tool.uv.sources]
+# torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
+# [[tool.uv.index]]
+# name = "pytorch-cpu"
+# url = "https://download.pytorch.org/whl/cpu"
+# explicit = true
# ///
"""Compare native tokenization and F32 language scores with Transformers on CPU."""