diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/linux.Dockerfile | 19 | ||||
| -rwxr-xr-x | tools/test-linux.sh | 30 | ||||
| -rw-r--r-- | tools/test_api.c | 16 | ||||
| -rw-r--r-- | tools/verify.py | 6 |
4 files changed, 71 insertions, 0 deletions
diff --git a/tools/linux.Dockerfile b/tools/linux.Dockerfile new file mode 100644 index 0000000..2d1b93d --- /dev/null +++ b/tools/linux.Dockerfile @@ -0,0 +1,19 @@ +FROM ghcr.io/astral-sh/uv:0.11.19 AS uv +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential pkg-config libopenblas-pthread-dev python3 python3-venv ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=uv /uv /usr/local/bin/uv + +WORKDIR /work +ENV UV_CACHE_DIR=/work/.uv-cache \ + UV_PYTHON_DOWNLOADS=never \ + HF_HOME=/work/.hf-cache \ + HF_HUB_OFFLINE=1 \ + TRANSFORMERS_OFFLINE=1 \ + TOKENIZERS_PARALLELISM=false +COPY verify.py tools/verify.py +RUN uv --no-config run --no-project --python python3 --script tools/verify.py --help + +CMD ["make", "-j4", "test"] diff --git a/tools/test-linux.sh b/tools/test-linux.sh new file mode 100755 index 0000000..9431c1f --- /dev/null +++ b/tools/test-linux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +arch=${1:-$(docker info --format '{{.Architecture}}')} +case "$arch" in + arm64|aarch64) arch=arm64 ;; + amd64|x86_64) arch=amd64 ;; + *) printf 'Usage: %s [arm64|amd64] [MAKE_ARGUMENT ...]\n' "$0" >&2; exit 2 ;; +esac +if [ "$#" -gt 0 ]; then shift; fi + +if [ ! -f "$root/model/model.safetensors" ]; then + printf 'Download the model first: python3 %s/download_model.py\n' "$root" >&2 + exit 1 +fi + +image="langdet-linux-test:$arch" +docker build --platform "linux/$arch" -f "$root/tools/linux.Dockerfile" -t "$image" "$root/tools" +args=(run --rm --network none --platform "linux/$arch" --workdir /work) +for path in Makefile langdetect.c liblangdet deps tools examples model; do + args+=(--mount "type=bind,src=$root/$path,dst=/work/$path,readonly") +done +docker "${args[@]}" "$image" sh -eu -c ' + make -j4 test "$@" + cc -Iliblangdet examples/detect.c liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example + c++ -Iliblangdet -x c++ examples/detect.c -x none liblangdet.a $(pkg-config --libs openblas) -lm -pthread -o build/detect-example-cpp + ./build/detect-example model "This is an English sentence." "Bugün hava çok güzel." + ./build/detect-example-cpp model "This is an English sentence." "Bugün hava çok güzel." +' linux-tests "$@" diff --git a/tools/test_api.c b/tools/test_api.c index 6923102..18f68b6 100644 --- a/tools/test_api.c +++ b/tools/test_api.c @@ -5,6 +5,10 @@ #include <stdlib.h> #include <string.h> +#ifdef USE_OPENBLAS +#include <cblas.h> +#endif + #define CHECK(condition) do { \ if (!(condition)) { \ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ @@ -40,6 +44,9 @@ int main(int argc, char **argv) { langdet_t *first = langdet_open(model_dir, 1, error, sizeof(error)); CHECK(first && !error[0]); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 1); +#endif langdet_result_t result; const char *english = "This is an English sentence."; CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); @@ -80,10 +87,19 @@ int main(int argc, char **argv) { langdet_t *second = langdet_open(model_dir, 4, error, sizeof(error)); CHECK(second && !error[0]); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 4); +#endif CHECK(langdet_predict(first, english, &result, error, sizeof(error)) == 0); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 1); +#endif CHECK(result.score == saved.score); langdet_free(first); CHECK(langdet_predict(second, english, &result, error, sizeof(error)) == 0); +#ifdef USE_OPENBLAS + CHECK(openblas_get_num_threads() == 4); +#endif CHECK(result.score == saved.score); langdet_free(second); puts("PASS: public API, repeated predictions, result ownership, truncation, errors, and lifecycle."); diff --git a/tools/verify.py b/tools/verify.py index d8bccf4..8339ac9 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -5,6 +5,12 @@ # "transformers==4.56.2", # "tokenizers==0.22.0", # ] +# [tool.uv.sources] +# torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true # /// """Compare native tokenization and F32 language scores with Transformers on CPU.""" |
