From 37cbaef9ceb828c667da8f398027ba5415ee7475 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 29 Sep 2026 13:11:00 +0200 Subject: Rename CPU library API and sources to langdet --- .gitignore | 1 + Makefile | 22 +- README.md | 25 +- langdetect.c | 24 +- libffwd/LICENSE | 20 - libffwd/act.c | 29 -- libffwd/alloc.c | 65 ---- libffwd/alloc.h | 22 -- libffwd/attn.c | 242 ------------ libffwd/config.c | 67 ---- libffwd/config.h | 8 - libffwd/ffwd.h | 49 --- libffwd/forward.c | 96 ----- libffwd/gemm.c | 49 --- libffwd/generic.c | 27 -- libffwd/impl.h | 254 ------------- libffwd/kernels.h | 42 -- libffwd/model.c | 159 -------- libffwd/model_blas.c | 68 ---- libffwd/model_internal.h | 17 - libffwd/model_types.h | 56 --- libffwd/norm.c | 33 -- libffwd/safetensors.c | 483 ----------------------- libffwd/safetensors.h | 64 ---- libffwd/sentencepiece.c | 905 -------------------------------------------- libffwd/sentencepiece.h | 71 ---- libffwd/threadpool.c | 218 ----------- libffwd/threadpool.h | 30 -- libffwd/tokenize.c | 85 ----- liblangdet/LICENSE | 20 + liblangdet/act.c | 29 ++ liblangdet/alloc.c | 65 ++++ liblangdet/alloc.h | 22 ++ liblangdet/attn.c | 242 ++++++++++++ liblangdet/config.c | 67 ++++ liblangdet/config.h | 8 + liblangdet/forward.c | 96 +++++ liblangdet/gemm.c | 49 +++ liblangdet/generic.c | 27 ++ liblangdet/impl.h | 254 +++++++++++++ liblangdet/kernels.h | 42 ++ liblangdet/langdet.h | 49 +++ liblangdet/model.c | 159 ++++++++ liblangdet/model_blas.c | 68 ++++ liblangdet/model_internal.h | 17 + liblangdet/model_types.h | 56 +++ liblangdet/norm.c | 33 ++ liblangdet/safetensors.c | 483 +++++++++++++++++++++++ liblangdet/safetensors.h | 64 ++++ liblangdet/sentencepiece.c | 905 ++++++++++++++++++++++++++++++++++++++++++++ liblangdet/sentencepiece.h | 71 ++++ liblangdet/threadpool.c | 218 +++++++++++ liblangdet/threadpool.h | 30 ++ liblangdet/tokenize.c | 85 +++++ 54 files changed, 3207 insertions(+), 3183 deletions(-) delete mode 100644 libffwd/LICENSE delete mode 100644 libffwd/act.c delete mode 100644 libffwd/alloc.c delete mode 100644 libffwd/alloc.h delete mode 100644 libffwd/attn.c delete mode 100644 libffwd/config.c delete mode 100644 libffwd/config.h delete mode 100644 libffwd/ffwd.h delete mode 100644 libffwd/forward.c delete mode 100644 libffwd/gemm.c delete mode 100644 libffwd/generic.c delete mode 100644 libffwd/impl.h delete mode 100644 libffwd/kernels.h delete mode 100644 libffwd/model.c delete mode 100644 libffwd/model_blas.c delete mode 100644 libffwd/model_internal.h delete mode 100644 libffwd/model_types.h delete mode 100644 libffwd/norm.c delete mode 100644 libffwd/safetensors.c delete mode 100644 libffwd/safetensors.h delete mode 100644 libffwd/sentencepiece.c delete mode 100644 libffwd/sentencepiece.h delete mode 100644 libffwd/threadpool.c delete mode 100644 libffwd/threadpool.h delete mode 100644 libffwd/tokenize.c create mode 100644 liblangdet/LICENSE create mode 100644 liblangdet/act.c create mode 100644 liblangdet/alloc.c create mode 100644 liblangdet/alloc.h create mode 100644 liblangdet/attn.c create mode 100644 liblangdet/config.c create mode 100644 liblangdet/config.h create mode 100644 liblangdet/forward.c create mode 100644 liblangdet/gemm.c create mode 100644 liblangdet/generic.c create mode 100644 liblangdet/impl.h create mode 100644 liblangdet/kernels.h create mode 100644 liblangdet/langdet.h create mode 100644 liblangdet/model.c create mode 100644 liblangdet/model_blas.c create mode 100644 liblangdet/model_internal.h create mode 100644 liblangdet/model_types.h create mode 100644 liblangdet/norm.c create mode 100644 liblangdet/safetensors.c create mode 100644 liblangdet/safetensors.h create mode 100644 liblangdet/sentencepiece.c create mode 100644 liblangdet/sentencepiece.h create mode 100644 liblangdet/threadpool.c create mode 100644 liblangdet/threadpool.h create mode 100644 liblangdet/tokenize.c diff --git a/.gitignore b/.gitignore index 9d40c3e..e23dc2a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /__pycache__/ /build/ /langdetect +/liblangdet.a diff --git a/Makefile b/Makefile index 56e17a0..4ac4306 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ CC = cc -CPPFLAGS += -Ilibffwd -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK +AR ?= ar +CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK CFLAGS ?= -O3 -march=native CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread LDLIBS += -framework Accelerate -lm -pthread @@ -7,20 +8,25 @@ LDLIBS += -framework Accelerate -lm -pthread CPU_SOURCES = model.c model_blas.c forward.c alloc.c config.c \ tokenize.c sentencepiece.c safetensors.c \ act.c gemm.c norm.c attn.c threadpool.c generic.c -SOURCES = langdetect.c $(addprefix libffwd/,$(CPU_SOURCES)) deps/yyjson/yyjson.c -OBJECTS = $(SOURCES:%.c=build/%.o) +LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c +LIB_OBJECTS = $(LIB_SOURCES:%.c=build/%.o) +OBJECTS = build/langdetect.o $(LIB_OBJECTS) .PHONY: all clean -all: langdetect +all: langdetect liblangdet.a -langdetect: $(OBJECTS) - $(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LDLIBS) +langdetect: build/langdetect.o liblangdet.a + $(CC) $(LDFLAGS) -o $@ build/langdetect.o liblangdet.a $(LDLIBS) -build/%.o: %.c +liblangdet.a: $(LIB_OBJECTS) Makefile + rm -f $@ + $(AR) rcs $@ $(LIB_OBJECTS) + +build/%.o: %.c Makefile @mkdir -p $(@D) $(CC) $(CPPFLAGS) $(CFLAGS) -MMD -MP -c $< -o $@ clean: - rm -rf build langdetect + rm -rf build langdetect liblangdet.a -include $(OBJECTS:.o=.d) diff --git a/README.md b/README.md index a7f5aae..e74eab9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # XLM-R language detection on macOS CPU -A native C CLI for `papluca/xlm-roberta-base-language-detection`, using ffwd's -CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language +A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the +liblangdet CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python runtime is required for inference. @@ -31,7 +31,7 @@ when there are no text arguments, produces one JSON line in input order: - `ms` measures tokenization and inference, excluding model loading and output. `--all` includes a `scores` object with probabilities for every language. -`-t N` selects 1 to 16 ffwd kernel threads; the default is the detected CPU count, +`-t N` selects 1 to 16 CPU kernel threads; the default is the detected CPU count, capped at 16. Accelerate manages its own BLAS threads. `-h` prints help. Use `--` before text that begins with a hyphen. Diagnostics go to stderr. @@ -44,7 +44,7 @@ The encoder uses first-token pooling without L2 normalization. The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer, and stable softmax. Dropout is disabled during inference. -The CPU encoder and tokenizer are in `libffwd/`; the JSON parser is in +The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in `deps/yyjson/`. Their license files are included. The build uses only local source files and the macOS SDK, with no dependency on another repository. @@ -54,6 +54,23 @@ accepted, but their predictions are not useful. The model always chooses among its 20 labels; it has no unknown-language result, and scores are not a guarantee of accuracy on ambiguous or unsupported text. +## CPU library + +`make` builds both `langdetect` and the static archive `liblangdet.a`. The public +encoder and tokenizer API is declared in `liblangdet/langdet.h`, with the +`langdet_` prefix. Each encoder call returns the first token's final hidden state; +the language-classification head and JSON output remain in `langdetect.c`. + +Link a C application with: + +```sh +cc -Iliblangdet application.c liblangdet.a -framework Accelerate -lm -pthread -o application +``` + +The archive includes its JSON parser. No separate parser library is needed. +Models own their memory-mapped weights; free their workspaces before freeing the +model. A tokenizer or workspace must not be used concurrently by multiple calls. + ## Download Python 3 and internet access are the only requirements. No Hugging Face token or diff --git a/langdetect.c b/langdetect.c index 003692d..6418d2d 100644 --- a/langdetect.c +++ b/langdetect.c @@ -17,9 +17,9 @@ #define MAX_TOKENS 512 typedef struct { - ffwd_model_t *model; - ffwd_workspace_t *workspace; - ffwd_tok_t *tokenizer; + langdet_model_t *model; + langdet_workspace_t *workspace; + langdet_tok_t *tokenizer; yyjson_doc *config; const char *labels[NUM_LABELS]; const float *dense_weight, *dense_bias; @@ -46,9 +46,9 @@ static const float *load_tensor(detector_t *d, const char *name, const int64_t * } static void detector_free(detector_t *d) { - ffwd_tok_free(d->tokenizer); - ffwd_workspace_free(d->workspace); - ffwd_model_free(d->model); + langdet_tok_free(d->tokenizer); + langdet_workspace_free(d->workspace); + langdet_model_free(d->model); yyjson_doc_free(d->config); } @@ -87,10 +87,10 @@ static int detector_load(detector_t *d, const char *directory) { } } - d->model = ffwd_model_load(directory); + d->model = langdet_model_load(directory); if (!d->model) return -1; - const ffwd_config_t *cfg = ffwd_model_config(d->model); + const langdet_config_t *cfg = langdet_model_config(d->model); if (cfg->hidden_size != HIDDEN_SIZE || cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); @@ -106,12 +106,12 @@ static int detector_load(detector_t *d, const char *directory) { return -1; char message[256] = {0}; - d->tokenizer = ffwd_tok_open(directory, message, sizeof(message)); + d->tokenizer = langdet_tok_open(directory, message, sizeof(message)); if (!d->tokenizer) { fprintf(stderr, "Tokenizer: %s\n", message); return -1; } - d->workspace = ffwd_workspace_new(d->model); + d->workspace = langdet_workspace_new(d->model); if (!d->workspace) { fprintf(stderr, "Cannot allocate inference workspace\n"); return -1; @@ -122,7 +122,7 @@ static int detector_load(detector_t *d, const char *directory) { static int classify(detector_t *d, const char *text, int all_scores) { double start = now_ms(); int count = 0; - int *ids = ffwd_tokenize(d->tokenizer, text, &count); + int *ids = langdet_tokenize(d->tokenizer, text, &count); if (!ids || count < 2) { fprintf(stderr, "Tokenization failed\n"); free(ids); @@ -134,7 +134,7 @@ static int classify(detector_t *d, const char *text, int all_scores) { count = MAX_TOKENS; } float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS]; - int result = ffwd_model_encode_into(d->model, d->workspace, ids, count, hidden); + int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); free(ids); if (result != 0) { fprintf(stderr, "Encoder inference failed\n"); diff --git a/libffwd/LICENSE b/libffwd/LICENSE deleted file mode 100644 index a00caa7..0000000 --- a/libffwd/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -MIT License - -Copyright (c) 2026 The ffwd authors - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/libffwd/act.c b/libffwd/act.c deleted file mode 100644 index 3912f2f..0000000 --- a/libffwd/act.c +++ /dev/null @@ -1,29 +0,0 @@ -#include "kernels.h" - -#include - -void gelu_inplace(float *x, int n) { - /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ - for (int i = 0; i < n; i++) - x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); -} - -void softmax(float *x, int rows, int cols) { - for (int r = 0; r < rows; r++) { - float *row = x + r * cols; - float max_val = row[0]; - for (int c = 1; c < cols; c++) { - if (row[c] > max_val) - max_val = row[c]; - } - float sum = 0.0f; - for (int c = 0; c < cols; c++) { - row[c] = expf(row[c] - max_val); - sum += row[c]; - } - float inv_sum = 1.0f / sum; - for (int c = 0; c < cols; c++) { - row[c] *= inv_sum; - } - } -} diff --git a/libffwd/alloc.c b/libffwd/alloc.c deleted file mode 100644 index ba21fbb..0000000 --- a/libffwd/alloc.c +++ /dev/null @@ -1,65 +0,0 @@ -#include "alloc.h" - -#include -#include -#include - -/* Sequence-capacity growth policy for the forward-pass workspaces: - * For new growth: - * - never allocate less than the minimum required size. - * - round up to the granularity so small length changes do not trigger a realloc on every call. - * - keep existing capacity if it is already larger. */ -#define FFWD_MIN_WORKSPACE_SEQ_CAP 16 -#define FFWD_WORKSPACE_SEQ_GRANULARITY 16 - -int mul_size(size_t a, size_t b, size_t *out) { - if (a != 0 && b > SIZE_MAX / a) - return -1; - *out = a * b; - return 0; -} - -int grow_cap(int current, int needed, int *out) { - if (current < 0 || needed <= 0) - return -1; - - int cap = needed; - if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP) - cap = FFWD_MIN_WORKSPACE_SEQ_CAP; - - int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY; - if (rem != 0) { - int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem; - if (cap > INT_MAX - pad) - return -1; - cap += pad; - } - - if (cap < current) - cap = current; - - *out = cap; - return 0; -} - -int realloc_floats(float **ptr, size_t count) { - size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) - return -1; - - void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) - return -1; - *ptr = (float *)p; - return 0; -} - -int realloc_floats_2d(float **ptr, int rows, int cols) { - if (rows < 0 || cols < 0) - return -1; - - size_t count; - if (mul_size((size_t)rows, (size_t)cols, &count) != 0) - return -1; - return realloc_floats(ptr, count); -} diff --git a/libffwd/alloc.h b/libffwd/alloc.h deleted file mode 100644 index d3ef81a..0000000 --- a/libffwd/alloc.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef FFWD_ALLOC_H -#define FFWD_ALLOC_H - -#include - -/* Checked size arithmetic and typed allocation helpers. - * Return -1 or NULL on overflow/allocation failure. - * realloc_* leave the original pointer unchanged on failure. */ - -/* out = a * b, or -1 if the product overflows size_t. */ -int mul_size(size_t a, size_t b, size_t *out); - -/* Grow sequence capacity without shrinking. */ -int grow_cap(int current, int needed, int *out); - -/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ -int realloc_floats(float **ptr, size_t count); - -/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ -int realloc_floats_2d(float **ptr, int rows, int cols); - -#endif /* FFWD_ALLOC_H */ diff --git a/libffwd/attn.c b/libffwd/attn.c deleted file mode 100644 index ac0f4d1..0000000 --- a/libffwd/attn.c +++ /dev/null @@ -1,242 +0,0 @@ -#include "kernels.h" -#include "impl.h" -#include "threadpool.h" - -#include - -#ifdef USE_BLAS -# ifdef __APPLE__ -# include -# else -# include -# endif -#endif - -static inline float dot_f32_fast(const float *a, const float *b, int n) { - return dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void vec_scale_inplace(float *dst, float scale, int n) { - vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { - vec_scale_add_impl(dst, src, correction, n); -} - -#define PACKED_ATTN_QUERY_TILE 32 -#define PACKED_ATTN_BLAS_MIN_SEQ 128 - -static void bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - - float score = dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - vec_scale_inplace(o_row, inv_sum, head_dim); - } - } -} - -#ifdef USE_BLAS -static void bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, - kv_hidden, 0.0f, scores, keys); - softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; - int head_dim; - float scale; - float *scores; - int max_seq; -} packed_gqa_attn_task_t; - -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { - bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { -#ifdef USE_BLAS - if (!offsets || batch <= 0) - return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) - return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) - return 0; - return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); -#else - (void)offsets; - (void)batch; - return 0; -#endif -} - -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); - } else { - packed_gqa_attn_worker(0, 1, &task); - } -} diff --git a/libffwd/config.c b/libffwd/config.c deleted file mode 100644 index 6343113..0000000 --- a/libffwd/config.c +++ /dev/null @@ -1,67 +0,0 @@ -#include "config.h" - -#include - -#include -#include -#include - -static int json_get_int(yyjson_val *obj, const char *key, int fallback) { - yyjson_val *v = yyjson_obj_get(obj, key); - if (!v) - return fallback; - if (!yyjson_is_int(v)) - return -1; - int64_t n = yyjson_get_sint(v); - return n >= 0 && n <= INT_MAX ? (int)n : -1; -} - -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); - if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "ffwd_config: model path too long\n"); - return -1; - } - - yyjson_read_err error; - yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); - if (!doc) { - fprintf(stderr, "ffwd_config: cannot read %s: %s\n", path, error.msg); - return -1; - } - yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *causal = yyjson_obj_get(root, "is_causal"); - if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || - !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || - (causal && !yyjson_is_false(causal))) { - fprintf(stderr, "ffwd_config: expected bidirectional XLM-R with GeLU activation\n"); - yyjson_doc_free(doc); - return -1; - } - - cfg->hidden_size = json_get_int(root, "hidden_size", 0); - cfg->n_layers = json_get_int(root, "num_hidden_layers", 0); - cfg->n_heads = json_get_int(root, "num_attention_heads", 0); - cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; - cfg->intermediate_size = json_get_int(root, "intermediate_size", 0); - cfg->vocab_size = json_get_int(root, "vocab_size", 0); - cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0); - cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1); - yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps"); - cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f; - int pad_id = json_get_int(root, "pad_token_id", 1); - cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; - yyjson_doc_free(doc); - - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > FFWD_MAX_LAYERS || - cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || - cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || - cfg->position_id_offset < 0 || - cfg->max_position_embeddings <= cfg->position_id_offset || - !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { - fprintf(stderr, "ffwd_config: invalid encoder dimensions or normalization in %s\n", path); - return -1; - } - return 0; -} diff --git a/libffwd/config.h b/libffwd/config.h deleted file mode 100644 index b3e2f67..0000000 --- a/libffwd/config.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef FFWD_CONFIG_H -#define FFWD_CONFIG_H - -#include "ffwd.h" - -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir); - -#endif /* FFWD_CONFIG_H */ diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h deleted file mode 100644 index 527e92f..0000000 --- a/libffwd/ffwd.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef FFWD_H -#define FFWD_H - -#include - -#define FFWD_MAX_LAYERS 64 - -typedef struct { - int hidden_size; - int n_layers; - int n_heads; - int head_dim; - int intermediate_size; - int vocab_size; - int max_position_embeddings; - int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ - int type_vocab_size; - float layer_norm_eps; -} ffwd_config_t; - -typedef struct ffwd_model ffwd_model_t; -typedef struct ffwd_workspace ffwd_workspace_t; -typedef struct ffwd_tok ffwd_tok_t; - -/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ -ffwd_model_t *ffwd_model_load(const char *model_dir); -void ffwd_model_free(ffwd_model_t *model); -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model); - -/* A workspace belongs to one model and may serve only one inference call at a time. */ -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model); -void ffwd_workspace_free(ffwd_workspace_t *ws); - -/* Write the first token's final hidden state to out_embedding[hidden_size]. - * Token IDs must include and , without padding. Returns 0 or -1 on error. */ -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen); -void ffwd_tok_free(ffwd_tok_t *t); - -/* Return malloc-owned IDs wrapped with and , or NULL on error. - * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out); - -#endif /* FFWD_H */ diff --git a/libffwd/forward.c b/libffwd/forward.c deleted file mode 100644 index b314642..0000000 --- a/libffwd/forward.c +++ /dev/null @@ -1,96 +0,0 @@ -#include "model_internal.h" -#include "kernels.h" - -#include -#include -#include -#include - -static void add_bias_rows(float *x, const float *bias, int rows, int dim) { - for (int row = 0; row < rows; row++) { - float *dst = x + (size_t)row * dim; - for (int d = 0; d < dim; d++) - dst[d] += bias[d]; - } -} - -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { - if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) - return -1; - - const ffwd_config_t *cfg = &model->config; - const ffwd_weights_t *w = &model->weights; - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int head_dim = cfg->head_dim; - int inter = cfg->intermediate_size; - float eps = cfg->layer_norm_eps; - - if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { - fprintf(stderr, - "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings " - "%d\n", - n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); - return -1; - } - if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || - ensure_buffers(ws, cfg, n_tokens) != 0) - return -1; - int offsets[2] = {0, n_tokens}; - ensure_attention_scores(ws, offsets, 1); - - float *x = ws->x; - float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; - float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; - - for (int i = 0; i < n_tokens; i++) { - int id = token_ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i); - return -1; - } - float *row = x + (size_t)i * hidden; - memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float)); - int pos = cfg->position_id_offset + i; - add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); - add_inplace(row, w->token_type_embeddings, hidden); - } - layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps); - - float scale = 1.0f / sqrtf((float)head_dim); - - for (int layer = 0; layer < cfg->n_layers; layer++) { - const ffwd_layer_t *l = &w->layers[layer]; - - linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); - linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); - linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden); - add_bias_rows(q_buf, l->q_bias, n_tokens, hidden); - add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); - add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); - - bidirectional_gqa_attention_packed_with_scratch( - attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, - ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); - add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); - add_inplace(x, proj_out, n_tokens * hidden); - layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps); - - linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter); - add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter); - gelu_inplace(ffn_up, n_tokens * inter); - linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden); - add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden); - add_inplace(x, proj_out, n_tokens * hidden); - layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps); - } - /* The classifier consumes after the encoder's final LayerNorm, without L2 normalization. */ - memcpy(out_embedding, x, (size_t)hidden * sizeof(float)); - return 0; -} diff --git a/libffwd/gemm.c b/libffwd/gemm.c deleted file mode 100644 index c750be1..0000000 --- a/libffwd/gemm.c +++ /dev/null @@ -1,49 +0,0 @@ -#include "kernels.h" - -#ifdef USE_BLAS -# ifdef __APPLE__ -# include -# else -# include -# endif -#endif - -void add_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) - a[i] += b[i]; -} - -void linear( - float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) { -#ifdef USE_BLAS - if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1); - } else { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, - in_dim, W, in_dim, 0.0f, y, out_dim); - } - if (b != NULL) { - for (int s = 0; s < seq_len; s++) { - for (int o = 0; o < out_dim; o++) { - y[s * out_dim + o] += b[o]; - } - } - } -#else - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * in_dim; - float *y_row = y + s * out_dim; - for (int o = 0; o < out_dim; o++) { - const float *w_row = W + o * in_dim; - float sum = (b != NULL) ? b[o] : 0.0f; - for (int i = 0; i < in_dim; i++) - sum += x_row[i] * w_row[i]; - y_row[o] = sum; - } - } -#endif -} - -void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { - linear(y, x, W, NULL, seq_len, in_dim, out_dim); -} diff --git a/libffwd/generic.c b/libffwd/generic.c deleted file mode 100644 index 729c610..0000000 --- a/libffwd/generic.c +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Architecture-generic hot kernels / non-SIMD fallback. - */ - -#include "impl.h" - -float dot_f32_generic(const float *a, const float *b, int n) { - float sum = 0.0f; - for (int i = 0; i < n; i++) - sum += a[i] * b[i]; - return sum; -} - -void vec_scale_inplace_generic(float *dst, float scale, int n) { - for (int i = 0; i < n; i++) - dst[i] *= scale; -} - -void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { - for (int i = 0; i < n; i++) - dst[i] += alpha * src[i]; -} - -void vec_scale_add_generic(float *dst, const float *src, float correction, int n) { - for (int i = 0; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -} diff --git a/libffwd/impl.h b/libffwd/impl.h deleted file mode 100644 index f216518..0000000 --- a/libffwd/impl.h +++ /dev/null @@ -1,254 +0,0 @@ -/* Inline vector operations keep attention rows in registers across reductions. */ - -#ifndef KERNELS_IMPL_H -#define KERNELS_IMPL_H - -#include - -float dot_f32_generic(const float *a, const float *b, int n); -void vec_scale_inplace_generic(float *dst, float scale, int n); -void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); -void vec_scale_add_generic(float *dst, const float *src, float correction, int n); - -#ifdef __ARM_NEON - -# include - -static inline float dot_f32_impl(const float *a, const float *b, int n) { - int i = 0; - float32x4_t acc0 = vdupq_n_f32(0.0f); - float32x4_t acc1 = vdupq_n_f32(0.0f); - for (; i + 8 <= n; i += 8) { - float32x4_t a0 = vld1q_f32(a + i); - float32x4_t b0 = vld1q_f32(b + i); - float32x4_t a1 = vld1q_f32(a + i + 4); - float32x4_t b1 = vld1q_f32(b + i + 4); - acc0 = vfmaq_f32(acc0, a0, b0); - acc1 = vfmaq_f32(acc1, a1, b1); - } - float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -} - -static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { - int i = 0; - float32x4_t s = vdupq_n_f32(scale); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); - vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); - } - for (; i < n; i++) - dst[i] *= scale; -} - -static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { - int i = 0; - float32x4_t a = vdupq_n_f32(alpha); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); - vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -} - -static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { - int i = 0; - float32x4_t c = vdupq_n_f32(correction); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); - vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -} - -#elif defined(__AVX2__) && defined(__FMA__) - -# include - -static inline float dot_f32_impl(const float *a, const float *b, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 acc0 = _mm512_setzero_ps(); - __m512 acc1 = _mm512_setzero_ps(); - __m512 acc2 = _mm512_setzero_ps(); - __m512 acc3 = _mm512_setzero_ps(); - for (; i + 64 <= n; i += 64) { - acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); - acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); - acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); - acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); - } - __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); - for (; i + 16 <= n; i += 16) { - acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -# else - int i = 0; - __m256 acc0 = _mm256_setzero_ps(); - __m256 acc1 = _mm256_setzero_ps(); - __m256 acc2 = _mm256_setzero_ps(); - __m256 acc3 = _mm256_setzero_ps(); - for (; i + 32 <= n; i += 32) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); - acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); - acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); - } - acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); - for (; i + 8 <= n; i += 8) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - } - __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); - r = _mm_hadd_ps(r, r); - r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -# endif -} - -static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 s = _mm512_set1_ps(scale); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); - _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); - _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -# else - int i = 0; - __m256 s = _mm256_set1_ps(scale); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); - _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); - _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -# endif -} - -static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 a = _mm512_set1_ps(alpha); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, - _mm512_loadu_ps(dst + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, - _mm512_loadu_ps(dst + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, - _mm512_loadu_ps(dst + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -# else - int i = 0; - __m256 a = _mm256_set1_ps(alpha); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, - _mm256_loadu_ps(dst + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, - _mm256_loadu_ps(dst + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, - _mm256_loadu_ps(dst + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -# endif -} - -static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 c = _mm512_set1_ps(correction); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, - _mm512_loadu_ps(src + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, - _mm512_loadu_ps(src + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, - _mm512_loadu_ps(src + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -# else - int i = 0; - __m256 c = _mm256_set1_ps(correction); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, - _mm256_loadu_ps(src + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, - _mm256_loadu_ps(src + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, - _mm256_loadu_ps(src + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -# endif -} - -#else /* no SIMD path: dispatch to the scalar references */ - -# define dot_f32_impl dot_f32_generic -# define vec_scale_inplace_impl vec_scale_inplace_generic -# define vec_axpy_inplace_impl vec_axpy_inplace_generic -# define vec_scale_add_impl vec_scale_add_generic - -#endif - -#endif /* KERNELS_IMPL_H */ diff --git a/libffwd/kernels.h b/libffwd/kernels.h deleted file mode 100644 index 1025602..0000000 --- a/libffwd/kernels.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef KERNELS_H -#define KERNELS_H - -#include -#include - -void add_inplace(float *a, const float *b, int n); - -/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */ -void linear( - float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); - -/* Mean and population variance are reduced over the hidden axis per row. */ -void layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps); - -void softmax(float *x, int rows, int cols); -void gelu_inplace(float *x, int n); - -/* Attention is block-diagonal: each token attends only within its offsets span. - * Missing scratch uses the online-softmax path instead of tiled BLAS. */ -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); - -#endif /* KERNELS_H */ diff --git a/libffwd/model.c b/libffwd/model.c deleted file mode 100644 index 81073f3..0000000 --- a/libffwd/model.c +++ /dev/null @@ -1,159 +0,0 @@ -#include "model_internal.h" -#include "config.h" - -#include -#include - -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim) { - if (t->dtype != DTYPE_F32) { - fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, - "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "ffwd: shape too large or empty for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - size_t bytes; - if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { - fprintf(stderr, "ffwd: invalid data size for %s\n", name); - return 0; - } - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static const float *load_weight(multi_safetensors_t *ms, const char *name, - const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "ffwd: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) - return NULL; - return safetensors_data(sf, t); -} - -static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { - const ffwd_config_t *cfg = &model->config; - ffwd_weights_t *w = &model->weights; - char name[256]; - int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - - w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", - (int64_t[]){cfg->vocab_size, hidden}, 2); - w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", - (int64_t[]){cfg->max_position_embeddings, hidden}, 2); - w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", - (int64_t[]){cfg->type_vocab_size, hidden}, 2); - w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); - w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); - if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || - !w->ffwd_ln_w || !w->ffwd_ln_b) - return -1; - - w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); - if (!w->layers) - return -1; - - for (int i = 0; i < cfg->n_layers; i++) { - ffwd_layer_t *l = &w->layers[i]; - -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 2); \ - if (!l->field) \ - return -1; \ - } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ - } while (0) - - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); - -#undef BERT_W -#undef BERT_V - } - return 0; -} - -ffwd_model_t *ffwd_model_load(const char *model_dir) { - if (!model_dir) - return NULL; - ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t)); - if (!model) - return NULL; - if (ffwd_config_parse(&model->config, model_dir) != 0) - goto fail; - model->safetensors = multi_safetensors_open(model_dir); - if (!model->safetensors) { - fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir); - goto fail; - } - if (load_bert_weights(model, model->safetensors) != 0) - goto fail; - return model; - -fail: - ffwd_model_free(model); - return NULL; -} - -void ffwd_model_free(ffwd_model_t *model) { - if (!model) - return; - multi_safetensors_close(model->safetensors); - free(model->weights.layers); - free(model); -} - -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) { - return model ? &model->config : NULL; -} diff --git a/libffwd/model_blas.c b/libffwd/model_blas.c deleted file mode 100644 index 409ba66..0000000 --- a/libffwd/model_blas.c +++ /dev/null @@ -1,68 +0,0 @@ -#include "model_internal.h" -#include "kernels.h" - -#include - -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) { - if (!model) - return NULL; - ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t)); - if (!ws) - return NULL; - ws->model = model; - return ws; -} - -void ffwd_workspace_free(ffwd_workspace_t *ws) { - if (!ws) - return; - free(ws->x); - free(ws->q); - free(ws->k); - free(ws->v); - free(ws->attn_out); - free(ws->proj_out); - free(ws->ffn_up); - free(ws->attn_scores); - free(ws); -} - -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) { - if (seq <= ws->buf_seq_cap) - return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) - return -1; - -#define R(ptr, n) \ - do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ - return -1; \ - } while (0) - - R(ws->x, c->hidden_size); - R(ws->q, c->hidden_size); - R(ws->k, c->hidden_size); - R(ws->v, c->hidden_size); - R(ws->attn_out, c->hidden_size); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) - return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) - return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} diff --git a/libffwd/model_internal.h b/libffwd/model_internal.h deleted file mode 100644 index aeea52e..0000000 --- a/libffwd/model_internal.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef FFWD_MODEL_INTERNAL_H -#define FFWD_MODEL_INTERNAL_H - -#include "model_types.h" -#include "alloc.h" - -/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */ -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim); - -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int seq); -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); - -#endif /* FFWD_MODEL_INTERNAL_H */ diff --git a/libffwd/model_types.h b/libffwd/model_types.h deleted file mode 100644 index 447b653..0000000 --- a/libffwd/model_types.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef FFWD_MODEL_TYPES_H -#define FFWD_MODEL_TYPES_H - -#include "ffwd.h" -#include "safetensors.h" - -typedef struct { - const float *wq; - const float *wk; - const float *wv; - const float *wo; - const float *q_bias; - const float *k_bias; - const float *v_bias; - const float *o_bias; - const float *input_norm; - const float *attn_ln_bias; - const float *up_proj; - const float *ffn_inter_bias; - const float *down_proj; - const float *ffn_out_bias; - const float *post_attn_norm; - const float *ffn_ln_bias; -} ffwd_layer_t; - -typedef struct { - const float *embed_tokens; - ffwd_layer_t *layers; - const float *position_embeddings; - const float *token_type_embeddings; - const float *ffwd_ln_w; - const float *ffwd_ln_b; -} ffwd_weights_t; - -struct ffwd_model { - ffwd_config_t config; - ffwd_weights_t weights; - /* All weight pointers borrow this mapping until model teardown. */ - multi_safetensors_t *safetensors; -}; - -struct ffwd_workspace { - const ffwd_model_t *model; - int buf_seq_cap; - float *x; - float *q; - float *k; - float *v; - float *attn_out; - float *proj_out; - float *ffn_up; - float *attn_scores; - size_t attn_scores_bytes; -}; - -#endif /* FFWD_MODEL_TYPES_H */ diff --git a/libffwd/norm.c b/libffwd/norm.c deleted file mode 100644 index 59e8bdb..0000000 --- a/libffwd/norm.c +++ /dev/null @@ -1,33 +0,0 @@ -#include "kernels.h" - -#include - -/* Safe in place: each row's mean and variance are reduced before it is rewritten. */ -void layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - float inv_h = 1.0f / (float)hidden; - for (int s = 0; s < seq_len; s++) { - const float *row = x + (size_t)s * hidden; - float *dst = out + (size_t)s * hidden; - float mean = 0.0f; - for (int d = 0; d < hidden; d++) - mean += row[d]; - mean *= inv_h; - float var = 0.0f; - for (int d = 0; d < hidden; d++) { - float diff = row[d] - mean; - var += diff * diff; - } - var *= inv_h; - float inv_std = 1.0f / sqrtf(var + eps); - for (int d = 0; d < hidden; d++) - dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; - } -} diff --git a/libffwd/safetensors.c b/libffwd/safetensors.c deleted file mode 100644 index cfba76e..0000000 --- a/libffwd/safetensors.c +++ /dev/null @@ -1,483 +0,0 @@ -#include "safetensors.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -/* Minimal JSON parser for safetensors header */ -/* TODO: consider replacing with yyjson */ - -static void skip_whitespace(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') - (*p)++; -} - -static int parse_string(const char **p, char *out, size_t max_len) { - skip_whitespace(p); - if (**p != '"') - return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') - out[i++] = '\n'; - else if (**p == 't') - out[i++] = '\t'; - else if (**p == '"') - out[i++] = '"'; - else if (**p == '\\') - out[i++] = '\\'; - else - out[i++] = **p; - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') - return -1; - (*p)++; - return 0; -} - -static int parse_int(const char **p, int64_t *out) { - skip_whitespace(p); - int64_t val = 0; - int neg = 0; - if (**p == '-') { - neg = 1; - (*p)++; - } - if (**p < '0' || **p > '9') - return -1; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - *out = neg ? -val : val; - return 0; -} - -static safetensor_dtype_t parse_dtype(const char *s) { - if (strcmp(s, "F32") == 0) - return DTYPE_F32; - if (strcmp(s, "F16") == 0) - return DTYPE_F16; - if (strcmp(s, "BF16") == 0) - return DTYPE_BF16; - if (strcmp(s, "I32") == 0) - return DTYPE_I32; - if (strcmp(s, "I64") == 0) - return DTYPE_I64; - if (strcmp(s, "BOOL") == 0) - return DTYPE_BOOL; - return DTYPE_UNKNOWN; -} - -static int parse_tensor_entry(const char **p, safetensor_t *t) { - skip_whitespace(p); - if (**p != '{') - return -1; - (*p)++; - - t->dtype = DTYPE_UNKNOWN; - t->ndim = 0; - t->data_offset = 0; - t->data_size = 0; - - while (**p && **p != '}') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - - char key[64]; - if (parse_string(p, key, sizeof(key)) != 0) - return -1; - skip_whitespace(p); - if (**p != ':') - return -1; - (*p)++; - skip_whitespace(p); - - if (strcmp(key, "dtype") == 0) { - char dtype_str[32]; - if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) - return -1; - t->dtype = parse_dtype(dtype_str); - } else if (strcmp(key, "shape") == 0) { - if (**p != '[') - return -1; - (*p)++; - t->ndim = 0; - while (**p && **p != ']') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - if (t->ndim >= 8) - return -1; - if (parse_int(p, &t->shape[t->ndim]) != 0) - return -1; - t->ndim++; - } - if (**p != ']') - return -1; - (*p)++; - } else if (strcmp(key, "data_offsets") == 0) { - if (**p != '[') - return -1; - (*p)++; - skip_whitespace(p); - int64_t start_i; - if (parse_int(p, &start_i) != 0) - return -1; - skip_whitespace(p); - if (**p != ',') - return -1; - (*p)++; - skip_whitespace(p); - int64_t end_i; - if (parse_int(p, &end_i) != 0) - return -1; - if (start_i < 0 || end_i < start_i) - return -1; - t->data_offset = (size_t)start_i; - t->data_size = (size_t)(end_i - start_i); - skip_whitespace(p); - if (**p != ']') - return -1; - (*p)++; - } else { - /* Skip unknown value */ - if (**p == '"') { - (*p)++; - while (**p && **p != '"') { - if (**p == '\\') - (*p)++; - if (**p) - (*p)++; - } - if (**p == '"') - (*p)++; - } else if (**p == '[') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '[') - depth++; - else if (**p == ']') - depth--; - (*p)++; - } - } else if (**p == '{') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '{') - depth++; - else if (**p == '}') - depth--; - (*p)++; - } - } else { - while (**p && **p != ',' && **p != '}') - (*p)++; - } - } - } - if (**p != '}') - return -1; - (*p)++; - return 0; -} - -static int parse_header(safetensors_file_t *sf) { - const char *p = sf->header_json; - skip_whitespace(&p); - if (*p != '{') - return -1; - p++; - - sf->num_tensors = 0; - - while (*p && *p != '}') { - skip_whitespace(&p); - if (*p == ',') { - p++; - continue; - } - if (*p == '}') - break; - - char name[256]; - if (parse_string(&p, name, sizeof(name)) != 0) - return -1; - skip_whitespace(&p); - if (*p != ':') - return -1; - p++; - - if (strcmp(name, "__metadata__") == 0) { - skip_whitespace(&p); - if (*p == '{') { - int depth = 1; - p++; - while (*p && depth > 0) { - if (*p == '{') - depth++; - else if (*p == '}') - depth--; - p++; - } - } - continue; - } - - if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { - fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", - sf->path ? sf->path : "", SAFETENSORS_MAX_TENSORS); - return -1; - } - - safetensor_t *t = &sf->tensors[sf->num_tensors]; - snprintf(t->name, sizeof(t->name), "%s", name); - if (parse_tensor_entry(&p, t) != 0) - return -1; - sf->num_tensors++; - } - if (*p != '}') - return -1; - return 0; -} - -/* Single file operations */ - -safetensors_file_t *safetensors_open(const char *path) { - int fd = open(path, O_RDONLY); - if (fd < 0) - return NULL; - - struct stat st; - if (fstat(fd, &st) < 0) { - close(fd); - return NULL; - } - - size_t file_size = (size_t)st.st_size; - if (file_size < 8) { - close(fd); - return NULL; - } - - void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - if (data == MAP_FAILED) - return NULL; - - uint64_t header_size = 0; - memcpy(&header_size, data, 8); - if (header_size > file_size - 8) { - munmap(data, file_size); - return NULL; - } - - safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); - if (!sf) { - munmap(data, file_size); - return NULL; - } - - sf->path = strdup(path); - sf->data = data; - sf->file_size = file_size; - sf->header_size = (size_t)header_size; - - sf->header_json = malloc(header_size + 1); - if (!sf->header_json) { - safetensors_close(sf); - return NULL; - } - memcpy(sf->header_json, (char *)data + 8, header_size); - sf->header_json[header_size] = '\0'; - - if (parse_header(sf) != 0) { - safetensors_close(sf); - return NULL; - } - - return sf; -} - -void safetensors_close(safetensors_file_t *sf) { - if (!sf) - return; - if (sf->data) - munmap(sf->data, sf->file_size); - free(sf->path); - free(sf->header_json); - free(sf); -} - -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { - return (const char *)sf->data + 8 + sf->header_size + t->data_offset; -} - -/* Multi-shard operations */ - -static int cmp_shard_names(const void *a, const void *b) { - const char *sa = (const char *)a; - const char *sb = (const char *)b; - return strcmp(sa, sb); -} - -static int parse_shard_name(const char *name, int *idx, int *total) { - int i = 0, n = 0, consumed = 0; - if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) - return -1; - if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) - return -1; - *idx = i; - *total = n; - return 0; -} - -multi_safetensors_t *multi_safetensors_open(const char *model_dir) { - multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); - if (!ms) - return NULL; - - char path[4096]; - - /* Try single file first */ - snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); - safetensors_file_t *sf = safetensors_open(path); - if (sf) { - ms->shards[0] = sf; - ms->num_shards = 1; - return ms; - } - - /* Scan directory for shard files */ - DIR *dir = opendir(model_dir); - if (!dir) { - free(ms); - return NULL; - } - - struct dirent *entry; - char shard_names[SAFETENSORS_MAX_SHARDS][256]; - int seen[SAFETENSORS_MAX_SHARDS] = {0}; - int n_shards = 0; - int expected_total = 0; - - while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "model-", 6) == 0 && - strstr(entry->d_name, ".safetensors") != NULL) { - int shard_idx = 0, shard_total = 0; - if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { - fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (shard_total > SAFETENSORS_MAX_SHARDS) { - fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, - SAFETENSORS_MAX_SHARDS); - closedir(dir); - free(ms); - return NULL; - } - if (expected_total == 0) { - expected_total = shard_total; - } else if (expected_total != shard_total) { - fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", - entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (seen[shard_idx - 1]) { - fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); - closedir(dir); - free(ms); - return NULL; - } - seen[shard_idx - 1] = 1; - - snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); - n_shards++; - } - } - closedir(dir); - - if (n_shards == 0) { - fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); - free(ms); - return NULL; - } - if (expected_total != n_shards) { - fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", - expected_total, n_shards, model_dir); - free(ms); - return NULL; - } - for (int i = 0; i < expected_total; i++) { - if (!seen[i]) { - fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, - expected_total, model_dir); - free(ms); - return NULL; - } - } - - /* Sort shard names to ensure consistent ordering */ - qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); - - /* Open each shard */ - for (int i = 0; i < n_shards; i++) { - snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); - ms->shards[i] = safetensors_open(path); - if (!ms->shards[i]) { - fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); - multi_safetensors_close(ms); - return NULL; - } - } - ms->num_shards = n_shards; - return ms; -} - -void multi_safetensors_close(multi_safetensors_t *ms) { - if (!ms) - return; - for (int i = 0; i < ms->num_shards; i++) { - safetensors_close(ms->shards[i]); - } - free(ms); -} - -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { - for (int s = 0; s < ms->num_shards; s++) { - safetensors_file_t *sf = ms->shards[s]; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) { - if (out_sf) - *out_sf = sf; - return &sf->tensors[i]; - } - } - } - return NULL; -} diff --git a/libffwd/safetensors.h b/libffwd/safetensors.h deleted file mode 100644 index dca68a7..0000000 --- a/libffwd/safetensors.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Safetensors file format reader with multi-shard support. - */ - -#ifndef SAFETENSORS_H -#define SAFETENSORS_H - -#include -#include - -#define SAFETENSORS_MAX_TENSORS 1024 -#define SAFETENSORS_MAX_SHARDS 8 - -typedef enum { - DTYPE_F32 = 0, - DTYPE_F16 = 1, - DTYPE_BF16 = 2, - DTYPE_I32 = 3, - DTYPE_I64 = 4, - DTYPE_BOOL = 5, - DTYPE_UNKNOWN = -1 -} safetensor_dtype_t; - -typedef struct { - char name[256]; - safetensor_dtype_t dtype; - int ndim; - int64_t shape[8]; - size_t data_offset; - size_t data_size; -} safetensor_t; - -typedef struct { - char *path; - void *data; - size_t file_size; - size_t header_size; - char *header_json; - int num_tensors; - safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; -} safetensors_file_t; - -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ -typedef struct { - safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; - int num_shards; -} multi_safetensors_t; - -/* Open a single safetensors file (memory-mapped) */ -safetensors_file_t *safetensors_open(const char *path); -void safetensors_close(safetensors_file_t *sf); - -/* Open model from directory (auto-detects single file or multi-shard) */ -multi_safetensors_t *multi_safetensors_open(const char *model_dir); -void multi_safetensors_close(multi_safetensors_t *ms); - -/* Find a tensor by name across all shards */ -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); - -/* Get raw pointer to tensor data (within mmap'd region) */ -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); - -#endif /* SAFETENSORS_H */ diff --git a/libffwd/sentencepiece.c b/libffwd/sentencepiece.c deleted file mode 100644 index 6c30ba1..0000000 --- a/libffwd/sentencepiece.c +++ /dev/null @@ -1,905 +0,0 @@ -/* - * SentencePiece Unigram tokenizer. - * - * The implementation reads the small subset of ModelProto needed at runtime: - * pieces, scores, piece types, trainer model_type, and normalizer options. The - * normalizer supports SentencePiece's precompiled charsmap format used by - * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program - * over UTF-8 byte offsets. - */ - -#include "sentencepiece.h" - -#include -#include -#include -#include -#include -#include - -enum { - SP_MODEL_UNIGRAM = 1, - SP_PIECE_NORMAL = 1, - SP_PIECE_UNKNOWN = 2, - SP_PIECE_CONTROL = 3, - SP_PIECE_USER_DEFINED = 4, - SP_PIECE_UNUSED = 5, - SP_PIECE_BYTE = 6, -}; - -typedef struct { - const char *key; - uint16_t len; - int val; -} sp_entry_t; - -typedef struct { - int id; - int prev; - double score; -} sp_best_t; - -struct tok_spm_workspace { - char *norm; - size_t norm_len; - size_t norm_cap; - sp_best_t *best; - int best_cap; - int *ids; - int ids_cap; -}; - -static uint64_t fnv1a_bytes(const char *s, size_t n) { - uint64_t h = 1469598103934665603ULL; - for (size_t i = 0; i < n; i++) { - h ^= (unsigned char)s[i]; - h *= 1099511628211ULL; - } - return h; -} - -static int next_pow2(int x) { - int p = 1; - while (p < x && p <= (1 << 29)) - p <<= 1; - return p; -} - -static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) { - if (len <= 0 || len > UINT16_MAX) - return -1; - uint64_t mask = (uint64_t)cap - 1; - int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); - while (map[pos].key) { - if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) { - map[pos].val = val; - return 0; - } - pos = (int)((pos + 1) & (int)mask); - } - map[pos].key = key; - map[pos].len = (uint16_t)len; - map[pos].val = val; - return 0; -} - -static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { - if (!map || len <= 0) - return -1; - uint64_t mask = (uint64_t)cap - 1; - int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); - while (map[pos].key) { - if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) - return map[pos].val; - pos = (int)((pos + 1) & (int)mask); - } - return -1; -} - -static int is_tokenizable_type(int type) { - return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || - type == SP_PIECE_BYTE; -} - -static int reserve_chars(char **buf, size_t *cap, size_t need) { - if (need <= *cap) - return 0; - size_t nc = *cap ? *cap : 128; - while (nc < need) { - if (nc > SIZE_MAX / 2) - return -1; - nc *= 2; - } - char *p = (char *)realloc(*buf, nc); - if (!p) - return -1; - *buf = p; - *cap = nc; - return 0; -} - -static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) { - if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0) - return -1; - memcpy(*buf + *len, s, n); - *len += n; - (*buf)[*len] = '\0'; - return 0; -} - -static int reserve_best(tok_spm_workspace_t *ws, int need) { - if (need <= ws->best_cap) - return 0; - int nc = ws->best_cap ? ws->best_cap : 128; - while (nc < need) { - if (nc > INT_MAX / 2) - return -1; - nc *= 2; - } - sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p)); - if (!p) - return -1; - ws->best = p; - ws->best_cap = nc; - return 0; -} - -static int reserve_ids(tok_spm_workspace_t *ws, int need) { - if (need <= ws->ids_cap) - return 0; - int nc = ws->ids_cap ? ws->ids_cap : 64; - while (nc < need) { - if (nc > INT_MAX / 2) - return -1; - nc *= 2; - } - int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p)); - if (!p) - return -1; - ws->ids = p; - ws->ids_cap = nc; - return 0; -} - -static int utf8_len(unsigned char c) { - if (c < 0x80) - return 1; - if ((c & 0xE0) == 0xC0) - return 2; - if ((c & 0xF0) == 0xE0) - return 3; - if ((c & 0xF8) == 0xF0) - return 4; - return 1; -} - -static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) { - if (off >= n) - return 0; - const unsigned char *p = (const unsigned char *)s + off; - int len = utf8_len(p[0]); - if (off + (size_t)len > n) { - *len_out = 1; - return 0; - } - if (len > 1) { - for (int i = 1; i < len; i++) { - if ((p[i] & 0xC0) != 0x80) { - *len_out = 1; - return 0; - } - } - } - *len_out = (size_t)len; - return 1; -} - -static uint32_t read_le32(const unsigned char *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) { - uint64_t v = 0; - int shift = 0; - while (*p < end && shift <= 63) { - unsigned char c = *(*p)++; - v |= (uint64_t)(c & 0x7f) << shift; - if (!(c & 0x80)) { - *out = v; - return 0; - } - shift += 7; - } - return -1; -} - -static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) { - uint64_t key = 0; - if (pb_read_varint(p, end, &key) != 0) - return -1; - *field = (int)(key >> 3); - *wire = (int)(key & 7); - return *field > 0 ? 0 : -1; -} - -static int pb_read_len(const unsigned char **p, - const unsigned char *end, - const unsigned char **data, - size_t *len) { - uint64_t n = 0; - if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p)) - return -1; - *data = *p; - *len = (size_t)n; - *p += n; - return 0; -} - -static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) { - uint64_t v = 0; - const unsigned char *data = NULL; - size_t len = 0; - switch (wire) { - case 0: - return pb_read_varint(p, end, &v); - case 1: - if ((size_t)(end - *p) < 8) - return -1; - *p += 8; - return 0; - case 2: - return pb_read_len(p, end, &data, &len); - case 5: - if ((size_t)(end - *p) < 4) - return -1; - *p += 4; - return 0; - default: - return -1; - } -} - -static char *memdup0(const unsigned char *data, size_t len) { - char *s = (char *)malloc(len + 1); - if (!s) - return NULL; - memcpy(s, data, len); - s[len] = '\0'; - return s; -} - -static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - char *piece = NULL; - float score = 0.0f; - int type = SP_PIECE_NORMAL; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - goto fail; - if (field == 1 && wire == 2) { - const unsigned char *s = NULL; - size_t n = 0; - if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX) - goto fail; - free(piece); - piece = memdup0(s, n); - if (!piece) - goto fail; - tok->piece_lens[idx] = (uint16_t)n; - } else if (field == 2 && wire == 5) { - if ((size_t)(end - p) < 4) - goto fail; - uint32_t bits = read_le32(p); - memcpy(&score, &bits, sizeof(score)); - p += 4; - } else if (field == 3 && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX) - goto fail; - type = (int)v; - } else if (pb_skip(&p, end, wire) != 0) { - goto fail; - } - } - if (!piece) - goto fail; - tok->id_to_piece[idx] = piece; - tok->scores[idx] = score; - tok->types[idx] = (uint8_t)type; - if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) - tok->max_piece_len = tok->piece_lens[idx]; - return 0; -fail: - free(piece); - return -1; -} - -static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - int model_type = SP_MODEL_UNIGRAM; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (field == 3 && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX) - return -1; - model_type = (int)v; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - if (model_type != SP_MODEL_UNIGRAM) { - fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n", - model_type); - return -1; - } - (void)tok; - return 0; -} - -static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if ((field == 3 || field == 4 || field == 5) && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0) - return -1; - if (field == 3) - tok->add_dummy_prefix = v != 0; - else if (field == 4) - tok->remove_extra_whitespaces = v != 0; - else - tok->escape_whitespaces = v != 0; - } else if (field == 2 && wire == 2) { - const unsigned char *blob = NULL; - size_t n = 0; - if (pb_read_len(&p, end, &blob, &n) != 0) - return -1; - if (n >= 4) { - uint32_t xcda_bytes = read_le32(blob); - if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) { - tok->xcda_count = xcda_bytes / 4; - tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda)); - if (!tok->xcda) - return -1; - for (size_t i = 0; i < tok->xcda_count; i++) - tok->xcda[i] = read_le32(blob + 4 + i * 4); - tok->prefix_replacements_len = n - 4 - xcda_bytes; - tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1); - if (!tok->prefix_replacements) - return -1; - memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes, - tok->prefix_replacements_len); - tok->prefix_replacements[tok->prefix_replacements_len] = '\0'; - } - } - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - return 0; -} - -static int count_pieces(const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - int n = 0; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (field == 1 && wire == 2) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0) - return -1; - n++; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - return n; -} - -static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { - int n_pieces = count_pieces(data, len); - if (n_pieces <= 0) - return -1; - tok->piece_count = n_pieces; - tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); - tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); - tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); - tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); - if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) - return -1; - - const unsigned char *p = data, *end = data + len; - int piece_idx = 0; - int saw_trainer = 0; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (wire == 2 && field == 1) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces || - parse_piece(tok, piece_idx++, msg, msg_len) != 0) - return -1; - } else if (wire == 2 && field == 2) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) - return -1; - saw_trainer = 1; - } else if (wire == 2 && field == 3) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0) - return -1; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - if (piece_idx != n_pieces || !saw_trainer) - return -1; - return 0; -} - -static int raw_piece_id(const tok_spm_t *tok, const char *piece) { - if (!tok || !piece) - return -1; - return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); -} - -static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { - if (!tok || raw_id < 0 || raw_id >= tok->piece_count) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (raw_id == 1) - return 0; /* */ - if (raw_id == 2) - return 2; /* */ - if (raw_id == 0) - return 3; /* */ - return raw_id + 1; - } - return raw_id; -} - -static int file_contains_xlm_roberta(const char *model_dir, const char *name) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); - if (n < 0 || (size_t)n >= sizeof(path)) - return 0; - FILE *f = fopen(path, "rb"); - if (!f) - return 0; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return 0; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return 0; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf) { - fclose(f); - return 0; - } - size_t got = fread(buf, 1, (size_t)sz, f); - fclose(f); - buf[got] = '\0'; - int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || - strstr(buf, "XLMRobertaTokenizerFast") != NULL || - strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; - free(buf); - return yes; -} - -static int detect_xlm_roberta(const char *model_dir) { - return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || - file_contains_xlm_roberta(model_dir, "config.json"); -} - -static int find_model_file(const char *model_dir, char path[1024]) { - static const char *names[] = { - "sentencepiece.bpe.model", - "tokenizer.model", - "spiece.model", - }; - for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { - int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); - if (n < 0 || n >= 1024) - return -1; - FILE *f = fopen(path, "rb"); - if (f) { - fclose(f); - return 0; - } - if (errno != ENOENT) - return -1; - } - return -1; -} - -static int load_file(const char *path, unsigned char **out, size_t *out_len) { - FILE *f = fopen(path, "rb"); - if (!f) - return -1; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - unsigned char *buf = (unsigned char *)malloc((size_t)sz); - if (!buf) { - fclose(f); - return -1; - } - if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); - free(buf); - return -1; - } - fclose(f); - *out = buf; - *out_len = (size_t)sz; - return 0; -} - -static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) { - uint32_t v = tok->xcda[idx]; - return (v >> 10) << ((v & (1u << 9)) >> 6); -} - -static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) { - return tok->xcda[idx] & ((1u << 31) | 0xffu); -} - -static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; } - -static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) { - return tok->xcda[idx] & ((1u << 31) - 1u); -} - -typedef struct { - const char *data; - size_t len; - size_t consumed; -} norm_prefix_t; - -static norm_prefix_t -normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) { - static const char repl[] = "\xef\xbf\xbd"; - if (off >= input_len) - return (norm_prefix_t){input + off, 0, 0}; - - size_t best_len = 0; - size_t best_repl = 0; - if (tok->xcda && tok->xcda_count > 0) { - uint32_t node = xcda_base(tok, 0); - for (size_t p = off; p < input_len; p++) { - unsigned char c = (unsigned char)input[p]; - if (c == 0) - break; - node ^= c; - if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c) - break; - int leaf = xcda_leaf(tok, node); - node ^= xcda_base(tok, node); - if (leaf) { - best_len = p - off + 1; - best_repl = xcda_value(tok, node); - } - } - } - if (best_len > 0 && best_repl < tok->prefix_replacements_len) { - const char *r = tok->prefix_replacements + best_repl; - size_t max = tok->prefix_replacements_len - best_repl; - size_t n = strnlen(r, max); - return (norm_prefix_t){r, n, best_len}; - } - - size_t clen = 1; - if (valid_utf8_one(input, input_len, off, &clen)) - return (norm_prefix_t){input + off, clen, clen}; - return (norm_prefix_t){repl, 3, 1}; -} - -static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) { - ws->norm_len = 0; - if (!text) - return -1; - size_t input_len = strlen(text); - const char space[] = "\xe2\x96\x81"; /* U+2581 */ - const char *space_s = tok->escape_whitespaces ? space : " "; - size_t space_n = tok->escape_whitespaces ? 3 : 1; - int processing_non_ws = 0; - int space_prepended = 0; - - for (size_t off = 0; off < input_len;) { - norm_prefix_t r = normalize_prefix(tok, text, input_len, off); - for (size_t i = 0; i < r.len; i++) { - char c = r.data[i]; - if (c != ' ') { - if (!processing_non_ws) { - processing_non_ws = 1; - if ((tok->add_dummy_prefix && !space_prepended) || - tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != - 0) - return -1; - space_prepended = 1; - } - } - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) - return -1; - } else { - processing_non_ws = 0; - if (!tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) - return -1; - } - } - } - off += r.consumed ? r.consumed : 1; - } - if (ws->norm) - ws->norm[ws->norm_len] = '\0'; - return 0; -} - -static double compute_unknown_score(const tok_spm_t *tok) { - float min_score = FLT_MAX; - for (int i = 0; i < tok->piece_count; i++) { - if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) - min_score = tok->scores[i]; - } - if (min_score == FLT_MAX) - min_score = -10.0f; - return (double)min_score - 10.0; -} - -static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { - size_t n = ws->norm_len; - if (n == 0) - return 0; - if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 || - reserve_ids(ws, (int)n + 1) != 0) - return -1; - for (size_t i = 0; i <= n; i++) - ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; - ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; - double unk_score = tok->unknown_score; - - for (size_t off = 0; off < n;) { - size_t cp_len = 1; - valid_utf8_one(ws->norm, n, off, &cp_len); - sp_best_t cur = ws->best[off]; - if (cur.score <= -DBL_MAX / 2.0) { - off += cp_len; - continue; - } - - int single_cp_found = 0; - for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { - int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, - (int)(end - off)); - if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { - if (end - off == cp_len) - single_cp_found = 1; - double sc = - cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); - if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; - } - if (end == n) - break; - size_t next_len = 1; - valid_utf8_one(ws->norm, n, end, &next_len); - end += next_len; - } - if (!single_cp_found) { - size_t end = off + cp_len; - double sc = cur.score + unk_score; - if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; - } - off += cp_len; - } - - int out_n = 0; - int prev_unk = 0; - for (size_t off = n; off > 0;) { - sp_best_t b = ws->best[off]; - if (b.prev < 0 || b.prev >= (int)off) - return -1; - int is_unk = b.id == tok->unk_id; - if (!(prev_unk && is_unk)) { - if (out_n >= ws->ids_cap) - return -1; - ws->ids[out_n++] = b.id; - } - prev_unk = is_unk; - off = (size_t)b.prev; - } - for (int i = 0; i < out_n / 2; i++) { - int t = ws->ids[i]; - ws->ids[i] = ws->ids[out_n - 1 - i]; - ws->ids[out_n - 1 - i] = t; - } - return out_n; -} - -tok_spm_t *tok_spm_load(const char *model_dir) { - if (!model_dir) - return NULL; - char path[1024]; - if (find_model_file(model_dir, path) != 0) { - fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); - return NULL; - } - unsigned char *data = NULL; - size_t len = 0; - if (load_file(path, &data, &len) != 0) { - fprintf(stderr, "sentencepiece: cannot read %s\n", path); - return NULL; - } - - tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t)); - if (!tok) { - free(data); - return NULL; - } - tok->add_dummy_prefix = 1; - tok->remove_extra_whitespaces = 1; - tok->escape_whitespaces = 1; - tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; - if (parse_model(tok, data, len) != 0) { - fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); - free(data); - tok_spm_free(tok); - return NULL; - } - free(data); - - int cap = next_pow2(tok->piece_count * 2 + 1); - tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); - if (!tok->piece_map) { - tok_spm_free(tok); - return NULL; - } - tok->piece_map_cap = cap; - for (int i = 0; i < tok->piece_count; i++) { - if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], - tok->piece_lens[i], i) != 0) { - tok_spm_free(tok); - return NULL; - } - } - - tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "")); - tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "")); - tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "")); - tok->pad_id = - tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "")); - tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA - ? tok->piece_count + 1 - : raw_to_model_id(tok, raw_piece_id(tok, "")); - tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; - tok->unknown_score = compute_unknown_score(tok); - if (tok->unk_id < 0 || tok->max_piece_len <= 0) { - fprintf(stderr, "sentencepiece: missing or normal pieces in %s\n", path); - tok_spm_free(tok); - return NULL; - } - return tok; -} - -int tok_spm_token_id(const tok_spm_t *tok, const char *token) { - if (!tok || !token) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (!strcmp(token, "")) - return 0; - if (!strcmp(token, "")) - return 1; - if (!strcmp(token, "")) - return 2; - if (!strcmp(token, "")) - return 3; - if (!strcmp(token, "")) - return tok->piece_count + 1; - } - return raw_to_model_id(tok, raw_piece_id(tok, token)); -} - -static int tok_spm_encode_into(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text) - return -1; - if (normalize_into(tok, ws, text) != 0) - return -1; - int n = segment(tok, ws); - if (n < 0) - return -1; - if (out_n_tokens) - *out_n_tokens = n; - if (n == 0) - return 0; - if (!out_ids || out_cap < n) - return -2; - memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); - return 0; -} - -int *tok_spm_encode_with_workspace(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text) - return NULL; - int n = 0; - int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); - if (rc < 0 && rc != -2) - return NULL; - if (out_n_tokens) - *out_n_tokens = n; - if (n == 0) - return NULL; - int *ids = (int *)malloc((size_t)n * sizeof(*ids)); - if (!ids) - return NULL; - memcpy(ids, ws->ids, (size_t)n * sizeof(*ids)); - return ids; -} - -tok_spm_workspace_t *tok_spm_workspace_new(void) { - return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t)); -} - -void tok_spm_workspace_free(tok_spm_workspace_t *ws) { - if (!ws) - return; - free(ws->norm); - free(ws->best); - free(ws->ids); - free(ws); -} - -void tok_spm_free(tok_spm_t *tok) { - if (!tok) - return; - if (tok->id_to_piece) { - for (int i = 0; i < tok->piece_count; i++) - free(tok->id_to_piece[i]); - } - free(tok->id_to_piece); - free(tok->piece_lens); - free(tok->scores); - free(tok->types); - free(tok->piece_map); - free(tok->xcda); - free(tok->prefix_replacements); - free(tok); -} diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h deleted file mode 100644 index 2230538..0000000 --- a/libffwd/sentencepiece.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SentencePiece tokenizer for XLM-R style models. - * - * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, - * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R - * and multilingual E5. It encodes text only; frontends add the model's special - * token layout, e.g. ... for XLM-R. - */ - -#ifndef TOK_SPM_H -#define TOK_SPM_H - -#include -#include - -typedef enum { - TOK_SPM_IDENTITY = 0, - TOK_SPM_XLM_ROBERTA = 1, -} tok_spm_id_map_t; - -typedef struct { - char **id_to_piece; /* [piece_count] raw SentencePiece ids */ - uint16_t *piece_lens; /* byte length for id_to_piece entries */ - float *scores; /* SentencePiece score per raw id */ - uint8_t *types; /* SentencePiece piece type per raw id */ - int piece_count; /* raw SentencePiece vocabulary size */ - int vocab_size; /* model embedding vocab size after id mapping */ - int unk_id; /* model-space unknown-token id */ - int bos_id; /* model-space beginning-token id, or -1 */ - int eos_id; /* model-space end-token id, or -1 */ - int pad_id; /* model-space pad-token id, or -1 */ - int mask_id; /* model-space mask-token id, or -1 */ - int max_piece_len; /* max normal/user/unused piece byte length */ - double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ - int add_dummy_prefix; - int remove_extra_whitespaces; - int escape_whitespaces; - tok_spm_id_map_t id_map; - - void *piece_map; /* piece bytes -> raw SentencePiece id */ - int piece_map_cap; - - uint32_t *xcda; - size_t xcda_count; - char *prefix_replacements; - size_t prefix_replacements_len; -} tok_spm_t; - -typedef struct tok_spm_workspace tok_spm_workspace_t; - -/* Load the first supported SentencePiece model found in model_dir. Returns NULL - * on missing model file, unsupported model type, malformed protobuf, or OOM. */ -tok_spm_t *tok_spm_load(const char *model_dir); - -/* Resolve a named token (e.g. "", "", "", "") to a - * model-space id, or -1 when absent. */ -int tok_spm_token_id(const tok_spm_t *tok, const char *token); - -tok_spm_workspace_t *tok_spm_workspace_new(void); -void tok_spm_workspace_free(tok_spm_workspace_t *ws); - -/* Return malloc-owned model-space token IDs, reusing workspace scratch. - * Empty text returns NULL with a zero count. */ -int *tok_spm_encode_with_workspace(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_n_tokens); - -void tok_spm_free(tok_spm_t *tok); - -#endif /* TOK_SPM_H */ diff --git a/libffwd/threadpool.c b/libffwd/threadpool.c deleted file mode 100644 index 6cb4252..0000000 --- a/libffwd/threadpool.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * CPU thread pool. - */ - -#if defined(__linux__) && !defined(_GNU_SOURCE) -# define _GNU_SOURCE -#endif - -#include "threadpool.h" - -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -# include -#else -# ifdef __linux__ -# include -# endif -# include -#endif - -#if defined(USE_OPENBLAS) -void openblas_set_num_threads(int num_threads); -#endif - -#define TP_MAX_THREADS 16 - -static struct { - pthread_t threads[TP_MAX_THREADS - 1]; - int tids[TP_MAX_THREADS - 1]; - int n_threads; - int shutdown; - - parallel_fn_t fn; - void *arg; - uint64_t generation; - - pthread_mutex_t mutex; - pthread_cond_t cond_work; - pthread_cond_t cond_done; - int n_done; -} tp = { - .n_threads = 1, - .shutdown = 0, - .generation = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER, - .cond_work = PTHREAD_COND_INITIALIZER, - .cond_done = PTHREAD_COND_INITIALIZER, -}; - -static void *worker_loop(void *arg) { - int tid = *(int *)arg; - uint64_t my_gen = 0; - - for (;;) { - pthread_mutex_lock(&tp.mutex); - while (tp.generation == my_gen && !tp.shutdown) - pthread_cond_wait(&tp.cond_work, &tp.mutex); - if (tp.shutdown) { - pthread_mutex_unlock(&tp.mutex); - return NULL; - } - my_gen = tp.generation; - parallel_fn_t fn = tp.fn; - void *a = tp.arg; - int nt = tp.n_threads; - pthread_mutex_unlock(&tp.mutex); - - fn(tid, nt, a); - - pthread_mutex_lock(&tp.mutex); - if (++tp.n_done >= tp.n_threads - 1) - pthread_cond_signal(&tp.cond_done); - pthread_mutex_unlock(&tp.mutex); - } -} - -void tp_set_threads(int n) { - if (n < 1) - n = 1; - if (n > TP_MAX_THREADS) - n = TP_MAX_THREADS; - -#if defined(USE_OPENBLAS) - /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the - * auxiliary pool serial to avoid nested thread contention. */ - int blas_threads = n; - openblas_set_num_threads(blas_threads); - n = 1; -#endif - - /* Shutdown existing workers */ - if (tp.n_threads > 1) { - pthread_mutex_lock(&tp.mutex); - tp.shutdown = 1; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - for (int i = 0; i < tp.n_threads - 1; i++) - pthread_join(tp.threads[i], NULL); - tp.shutdown = 0; - tp.generation = 0; - } - - tp.n_threads = n; - if (n <= 1) - return; - - for (int i = 0; i < n - 1; i++) { - tp.tids[i] = i + 1; - pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); - } -} - -#ifndef __APPLE__ -/* CPU quota available to this process, in whole cores, or 0 if uncapped. - * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host - * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process - * can actually use. - * - * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: - * threads use the period budget early, then get throttled for the rest of the - * period, causing large bursty stalls. Example: a 16-thread default against a - * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. - * - * Floor instead of ceil, so a fully busy pool stays under quota and avoids - * throttling. - */ -static int cgroup_cpu_quota(void) { - /* cgroup v2: "/sys/fs/cgroup/cpu.max" is " ", or - * "max " when unlimited. */ - FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r"); - if (f) { - char quota[32]; - long period = 0; - int got = fscanf(f, "%31s %ld", quota, &period); - fclose(f); - if (got == 2 && period > 0 && strcmp(quota, "max") != 0) { - long q = atol(quota); - if (q > 0) - return q / period < 1 ? 1 : (int)(q / period); - } - } - /* cgroup v1: separate files; quota -1 means unlimited. */ - long q = -1, period = 0; - f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r"); - if (f) { - if (fscanf(f, "%ld", &q) != 1) - q = -1; - fclose(f); - } - f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r"); - if (f) { - if (fscanf(f, "%ld", &period) != 1) - period = 0; - fclose(f); - } - if (q > 0 && period > 0) - return q / period < 1 ? 1 : (int)(q / period); - return 0; -} -#endif - -int tp_get_num_cpus(void) { -#ifdef __APPLE__ - int n = 0; - size_t len = sizeof(n); - sysctlbyname("hw.ncpu", &n, &len, NULL, 0); - return n > 0 ? n : 1; -#else - int online = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (online < 1) - online = 1; -# ifdef __linux__ - cpu_set_t set; - if (sched_getaffinity(0, sizeof(set), &set) == 0) { - int affinity = CPU_COUNT(&set); - if (affinity >= 1 && affinity < online) - online = affinity; - } -# endif - /* Honor a cgroup CPU quota when it is tighter than the visible core count, - * so the default thread pool does not oversubscribe a CPU-limited - * container and get throttled. */ - int quota = cgroup_cpu_quota(); - if (quota >= 1 && quota < online) - return quota; - return online; -#endif -} - -int tp_num_threads(void) { return tp.n_threads; } - -/* Dispatch work to all threads; main thread is tid=0 */ -void parallel_for(parallel_fn_t fn, void *arg) { - if (tp.n_threads <= 1) { - fn(0, 1, arg); - return; - } - - pthread_mutex_lock(&tp.mutex); - tp.fn = fn; - tp.arg = arg; - tp.n_done = 0; - tp.generation++; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - - fn(0, tp.n_threads, arg); - - pthread_mutex_lock(&tp.mutex); - while (tp.n_done < tp.n_threads - 1) - pthread_cond_wait(&tp.cond_done, &tp.mutex); - pthread_mutex_unlock(&tp.mutex); -} diff --git a/libffwd/threadpool.h b/libffwd/threadpool.h deleted file mode 100644 index f9739a1..0000000 --- a/libffwd/threadpool.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef TP_THREADPOOL_H -#define TP_THREADPOOL_H - -/* Persistent worker pool for CPU kernels. - * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. - * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. - * - * Kept separate from the kernels so they depend only on parallel_for and - * tp_num_threads, not pool internals. - * - * kernels.h includes this, so kernel callers need no extra include. */ - -/* Worker callback: (thread index, total threads, user arg). */ -typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); - -/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call - * before inference. */ -void tp_set_threads(int n); - -/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */ -int tp_get_num_cpus(void); - -/* Current pool size (1 == serial). Kernels gate parallelism on this. */ -int tp_num_threads(void); - -/* Run fn across the pool; the calling thread participates as tid 0. Blocks - * until every worker has finished. */ -void parallel_for(parallel_fn_t fn, void *arg); - -#endif /* TP_THREADPOOL_H */ diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c deleted file mode 100644 index 8e1000f..0000000 --- a/libffwd/tokenize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "ffwd.h" -#include "sentencepiece.h" - -#include -#include -#include -#include - -struct ffwd_tok { - tok_spm_t *spm; - tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; -}; - -static void set_err(char *err, size_t errlen, const char *msg) { - if (err && errlen) - snprintf(err, errlen, "%s", msg); -} - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { - if (!model_dir) { - set_err(err, errlen, "model_dir is NULL"); - return NULL; - } - ffwd_tok_t *t = calloc(1, sizeof(*t)); - if (!t) { - set_err(err, errlen, "out of memory"); - return NULL; - } - t->spm = tok_spm_load(model_dir); - if (!t->spm) { - set_err(err, errlen, "failed to load SentencePiece tokenizer"); - goto fail; - } - t->workspace = tok_spm_workspace_new(); - if (!t->workspace) { - set_err(err, errlen, "out of memory"); - goto fail; - } - t->cls_id = tok_spm_token_id(t->spm, ""); - t->sep_id = tok_spm_token_id(t->spm, ""); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing /"); - goto fail; - } - return t; - -fail: - ffwd_tok_free(t); - return NULL; -} - -void ffwd_tok_free(ffwd_tok_t *t) { - if (!t) - return; - tok_spm_workspace_free(t->workspace); - tok_spm_free(t->spm); - free(t); -} - -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { - if (n_out) - *n_out = 0; - if (!t || !text) - return NULL; - int n = 0; - int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); - if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { - free(core); - return NULL; - } - /* Empty text still produces the two special tokens. */ - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return NULL; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; - if (n_out) - *n_out = n + 2; - return ids; -} diff --git a/liblangdet/LICENSE b/liblangdet/LICENSE new file mode 100644 index 0000000..a00caa7 --- /dev/null +++ b/liblangdet/LICENSE @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) 2026 The ffwd authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/liblangdet/act.c b/liblangdet/act.c new file mode 100644 index 0000000..3912f2f --- /dev/null +++ b/liblangdet/act.c @@ -0,0 +1,29 @@ +#include "kernels.h" + +#include + +void gelu_inplace(float *x, int n) { + /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ + for (int i = 0; i < n; i++) + x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); +} + +void softmax(float *x, int rows, int cols) { + for (int r = 0; r < rows; r++) { + float *row = x + r * cols; + float max_val = row[0]; + for (int c = 1; c < cols; c++) { + if (row[c] > max_val) + max_val = row[c]; + } + float sum = 0.0f; + for (int c = 0; c < cols; c++) { + row[c] = expf(row[c] - max_val); + sum += row[c]; + } + float inv_sum = 1.0f / sum; + for (int c = 0; c < cols; c++) { + row[c] *= inv_sum; + } + } +} diff --git a/liblangdet/alloc.c b/liblangdet/alloc.c new file mode 100644 index 0000000..4bc1496 --- /dev/null +++ b/liblangdet/alloc.c @@ -0,0 +1,65 @@ +#include "alloc.h" + +#include +#include +#include + +/* Sequence-capacity growth policy for the forward-pass workspaces: + * For new growth: + * - never allocate less than the minimum required size. + * - round up to the granularity so small length changes do not trigger a realloc on every call. + * - keep existing capacity if it is already larger. */ +#define LANGDET_MIN_WORKSPACE_SEQ_CAP 16 +#define LANGDET_WORKSPACE_SEQ_GRANULARITY 16 + +int mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; + *out = a * b; + return 0; +} + +int grow_cap(int current, int needed, int *out) { + if (current < 0 || needed <= 0) + return -1; + + int cap = needed; + if (cap < LANGDET_MIN_WORKSPACE_SEQ_CAP) + cap = LANGDET_MIN_WORKSPACE_SEQ_CAP; + + int rem = cap % LANGDET_WORKSPACE_SEQ_GRANULARITY; + if (rem != 0) { + int pad = LANGDET_WORKSPACE_SEQ_GRANULARITY - rem; + if (cap > INT_MAX - pad) + return -1; + cap += pad; + } + + if (cap < current) + cap = current; + + *out = cap; + return 0; +} + +int realloc_floats(float **ptr, size_t count) { + size_t bytes; + if (mul_size(count, sizeof(float), &bytes) != 0) + return -1; + + void *p = realloc(*ptr, bytes); + if (!p && bytes != 0) + return -1; + *ptr = (float *)p; + return 0; +} + +int realloc_floats_2d(float **ptr, int rows, int cols) { + if (rows < 0 || cols < 0) + return -1; + + size_t count; + if (mul_size((size_t)rows, (size_t)cols, &count) != 0) + return -1; + return realloc_floats(ptr, count); +} diff --git a/liblangdet/alloc.h b/liblangdet/alloc.h new file mode 100644 index 0000000..f70f1f7 --- /dev/null +++ b/liblangdet/alloc.h @@ -0,0 +1,22 @@ +#ifndef LANGDET_ALLOC_H +#define LANGDET_ALLOC_H + +#include + +/* Checked size arithmetic and typed allocation helpers. + * Return -1 or NULL on overflow/allocation failure. + * realloc_* leave the original pointer unchanged on failure. */ + +/* out = a * b, or -1 if the product overflows size_t. */ +int mul_size(size_t a, size_t b, size_t *out); + +/* Grow sequence capacity without shrinking. */ +int grow_cap(int current, int needed, int *out); + +/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ +int realloc_floats(float **ptr, size_t count); + +/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ +int realloc_floats_2d(float **ptr, int rows, int cols); + +#endif /* LANGDET_ALLOC_H */ diff --git a/liblangdet/attn.c b/liblangdet/attn.c new file mode 100644 index 0000000..ac0f4d1 --- /dev/null +++ b/liblangdet/attn.c @@ -0,0 +1,242 @@ +#include "kernels.h" +#include "impl.h" +#include "threadpool.h" + +#include + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include +# else +# include +# endif +#endif + +static inline float dot_f32_fast(const float *a, const float *b, int n) { + return dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void vec_scale_inplace(float *dst, float scale, int n) { + vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { + vec_scale_add_impl(dst, src, correction, n); +} + +#define PACKED_ATTN_QUERY_TILE 32 +#define PACKED_ATTN_BLAS_MIN_SEQ 128 + +static void bidirectional_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + vec_scale_inplace(o_row, inv_sum, head_dim); + } + } +} + +#ifdef USE_BLAS +static void bidirectional_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, + kv_hidden, 0.0f, scores, keys); + softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; + float *scores; + int max_seq; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { + bidirectional_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1, scores); + continue; + } +#endif + bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, + t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); + } + } + } +} + +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) + return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) + max_seq = len; + } + if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) + return 0; + if ((size_t)max_seq > + SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) + return 0; + return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { + parallel_for(packed_gqa_attn_worker, &task); + } else { + packed_gqa_attn_worker(0, 1, &task); + } +} diff --git a/liblangdet/config.c b/liblangdet/config.c new file mode 100644 index 0000000..f22a9d7 --- /dev/null +++ b/liblangdet/config.c @@ -0,0 +1,67 @@ +#include "config.h" + +#include + +#include +#include +#include + +static int json_get_int(yyjson_val *obj, const char *key, int fallback) { + yyjson_val *v = yyjson_obj_get(obj, key); + if (!v) + return fallback; + if (!yyjson_is_int(v)) + return -1; + int64_t n = yyjson_get_sint(v); + return n >= 0 && n <= INT_MAX ? (int)n : -1; +} + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "langdet_config: model path too long\n"); + return -1; + } + + yyjson_read_err error; + yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); + if (!doc) { + fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg); + return -1; + } + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *causal = yyjson_obj_get(root, "is_causal"); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || + (causal && !yyjson_is_false(causal))) { + fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n"); + yyjson_doc_free(doc); + return -1; + } + + cfg->hidden_size = json_get_int(root, "hidden_size", 0); + cfg->n_layers = json_get_int(root, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(root, "num_attention_heads", 0); + cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + cfg->intermediate_size = json_get_int(root, "intermediate_size", 0); + cfg->vocab_size = json_get_int(root, "vocab_size", 0); + cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0); + cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1); + yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps"); + cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f; + int pad_id = json_get_int(root, "pad_token_id", 1); + cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; + yyjson_doc_free(doc); + + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS || + cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || + cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || + cfg->position_id_offset < 0 || + cfg->max_position_embeddings <= cfg->position_id_offset || + !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path); + return -1; + } + return 0; +} diff --git a/liblangdet/config.h b/liblangdet/config.h new file mode 100644 index 0000000..79522fc --- /dev/null +++ b/liblangdet/config.h @@ -0,0 +1,8 @@ +#ifndef LANGDET_CONFIG_H +#define LANGDET_CONFIG_H + +#include "langdet.h" + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir); + +#endif /* LANGDET_CONFIG_H */ diff --git a/liblangdet/forward.c b/liblangdet/forward.c new file mode 100644 index 0000000..14b1327 --- /dev/null +++ b/liblangdet/forward.c @@ -0,0 +1,96 @@ +#include "model_internal.h" +#include "kernels.h" + +#include +#include +#include +#include + +static void add_bias_rows(float *x, const float *bias, int rows, int dim) { + for (int row = 0; row < rows; row++) { + float *dst = x + (size_t)row * dim; + for (int d = 0; d < dim; d++) + dst[d] += bias[d]; + } +} + +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding) { + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) + return -1; + + const langdet_config_t *cfg = &model->config; + const langdet_weights_t *w = &model->weights; + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; + int head_dim = cfg->head_dim; + int inter = cfg->intermediate_size; + float eps = cfg->layer_norm_eps; + + if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { + fprintf(stderr, + "langdet: sequence length %d plus position offset %d exceeds max_position_embeddings " + "%d\n", + n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); + return -1; + } + if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || + ensure_buffers(ws, cfg, n_tokens) != 0) + return -1; + int offsets[2] = {0, n_tokens}; + ensure_attention_scores(ws, offsets, 1); + + float *x = ws->x; + float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; + float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; + + for (int i = 0; i < n_tokens; i++) { + int id = token_ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "langdet: invalid token id %d at position %d\n", id, i); + return -1; + } + float *row = x + (size_t)i * hidden; + memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float)); + int pos = cfg->position_id_offset + i; + add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); + add_inplace(row, w->token_type_embeddings, hidden); + } + layer_norm(x, x, w->langdet_ln_w, w->langdet_ln_b, n_tokens, hidden, eps); + + float scale = 1.0f / sqrtf((float)head_dim); + + for (int layer = 0; layer < cfg->n_layers; layer++) { + const langdet_layer_t *l = &w->layers[layer]; + + linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); + linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); + linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden); + add_bias_rows(q_buf, l->q_bias, n_tokens, hidden); + add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); + add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); + + bidirectional_gqa_attention_packed_with_scratch( + attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); + + linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); + add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps); + + linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter); + add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter); + gelu_inplace(ffn_up, n_tokens * inter); + linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden); + add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps); + } + /* The classifier consumes after the encoder's final LayerNorm, without L2 normalization. */ + memcpy(out_embedding, x, (size_t)hidden * sizeof(float)); + return 0; +} diff --git a/liblangdet/gemm.c b/liblangdet/gemm.c new file mode 100644 index 0000000..c750be1 --- /dev/null +++ b/liblangdet/gemm.c @@ -0,0 +1,49 @@ +#include "kernels.h" + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include +# else +# include +# endif +#endif + +void add_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) + a[i] += b[i]; +} + +void linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) { +#ifdef USE_BLAS + if (seq_len == 1) { + cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1); + } else { + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, + in_dim, W, in_dim, 0.0f, y, out_dim); + } + if (b != NULL) { + for (int s = 0; s < seq_len; s++) { + for (int o = 0; o < out_dim; o++) { + y[s * out_dim + o] += b[o]; + } + } + } +#else + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * in_dim; + float *y_row = y + s * out_dim; + for (int o = 0; o < out_dim; o++) { + const float *w_row = W + o * in_dim; + float sum = (b != NULL) ? b[o] : 0.0f; + for (int i = 0; i < in_dim; i++) + sum += x_row[i] * w_row[i]; + y_row[o] = sum; + } + } +#endif +} + +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { + linear(y, x, W, NULL, seq_len, in_dim, out_dim); +} diff --git a/liblangdet/generic.c b/liblangdet/generic.c new file mode 100644 index 0000000..729c610 --- /dev/null +++ b/liblangdet/generic.c @@ -0,0 +1,27 @@ +/* + * Architecture-generic hot kernels / non-SIMD fallback. + */ + +#include "impl.h" + +float dot_f32_generic(const float *a, const float *b, int n) { + float sum = 0.0f; + for (int i = 0; i < n; i++) + sum += a[i] * b[i]; + return sum; +} + +void vec_scale_inplace_generic(float *dst, float scale, int n) { + for (int i = 0; i < n; i++) + dst[i] *= scale; +} + +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { + for (int i = 0; i < n; i++) + dst[i] += alpha * src[i]; +} + +void vec_scale_add_generic(float *dst, const float *src, float correction, int n) { + for (int i = 0; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +} diff --git a/liblangdet/impl.h b/liblangdet/impl.h new file mode 100644 index 0000000..f216518 --- /dev/null +++ b/liblangdet/impl.h @@ -0,0 +1,254 @@ +/* Inline vector operations keep attention rows in registers across reductions. */ + +#ifndef KERNELS_IMPL_H +#define KERNELS_IMPL_H + +#include + +float dot_f32_generic(const float *a, const float *b, int n); +void vec_scale_inplace_generic(float *dst, float scale, int n); +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON + +# include + +static inline float dot_f32_impl(const float *a, const float *b, int n) { + int i = 0; + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; i + 8 <= n; i += 8) { + float32x4_t a0 = vld1q_f32(a + i); + float32x4_t b0 = vld1q_f32(b + i); + float32x4_t a1 = vld1q_f32(a + i + 4); + float32x4_t b1 = vld1q_f32(b + i + 4); + acc0 = vfmaq_f32(acc0, a0, b0); + acc1 = vfmaq_f32(acc1, a1, b1); + } + float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { + int i = 0; + float32x4_t s = vdupq_n_f32(scale); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); + vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); + } + for (; i < n; i++) + dst[i] *= scale; +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { + int i = 0; + float32x4_t a = vdupq_n_f32(alpha); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); + vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { + int i = 0; + float32x4_t c = vdupq_n_f32(correction); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); + vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +} + +#elif defined(__AVX2__) && defined(__FMA__) + +# include + +static inline float dot_f32_impl(const float *a, const float *b, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); + } + __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); + for (; i + 16 <= n; i += 16) { + acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# else + int i = 0; + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); + } + acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + for (; i + 8 <= n; i += 8) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + } + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# endif +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 s = _mm512_set1_ps(scale); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); + _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); + _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# else + int i = 0; + __m256 s = _mm256_set1_ps(scale); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); + _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); + _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# endif +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 a = _mm512_set1_ps(alpha); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, + _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, + _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, + _mm512_loadu_ps(dst + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# else + int i = 0; + __m256 a = _mm256_set1_ps(alpha); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, + _mm256_loadu_ps(dst + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, + _mm256_loadu_ps(dst + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, + _mm256_loadu_ps(dst + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# endif +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 c = _mm512_set1_ps(correction); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, + _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, + _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, + _mm512_loadu_ps(src + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# else + int i = 0; + __m256 c = _mm256_set1_ps(correction); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, + _mm256_loadu_ps(src + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, + _mm256_loadu_ps(src + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, + _mm256_loadu_ps(src + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# endif +} + +#else /* no SIMD path: dispatch to the scalar references */ + +# define dot_f32_impl dot_f32_generic +# define vec_scale_inplace_impl vec_scale_inplace_generic +# define vec_axpy_inplace_impl vec_axpy_inplace_generic +# define vec_scale_add_impl vec_scale_add_generic + +#endif + +#endif /* KERNELS_IMPL_H */ diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h new file mode 100644 index 0000000..1025602 --- /dev/null +++ b/liblangdet/kernels.h @@ -0,0 +1,42 @@ +#ifndef KERNELS_H +#define KERNELS_H + +#include +#include + +void add_inplace(float *a, const float *b, int n); + +/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */ +void linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); + +/* Mean and population variance are reduced over the hidden axis per row. */ +void layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps); + +void softmax(float *x, int rows, int cols); +void gelu_inplace(float *x, int n); + +/* Attention is block-diagonal: each token attends only within its offsets span. + * Missing scratch uses the online-softmax path instead of tiled BLAS. */ +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); +void bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); + +#endif /* KERNELS_H */ diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h new file mode 100644 index 0000000..9c24264 --- /dev/null +++ b/liblangdet/langdet.h @@ -0,0 +1,49 @@ +#ifndef LANGDET_H +#define LANGDET_H + +#include + +#define LANGDET_MAX_LAYERS 64 + +typedef struct { + int hidden_size; + int n_layers; + int n_heads; + int head_dim; + int intermediate_size; + int vocab_size; + int max_position_embeddings; + int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ + int type_vocab_size; + float layer_norm_eps; +} langdet_config_t; + +typedef struct langdet_model langdet_model_t; +typedef struct langdet_workspace langdet_workspace_t; +typedef struct langdet_tok langdet_tok_t; + +/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ +langdet_model_t *langdet_model_load(const char *model_dir); +void langdet_model_free(langdet_model_t *model); +const langdet_config_t *langdet_model_config(const langdet_model_t *model); + +/* A workspace belongs to one model and may serve only one inference call at a time. */ +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); +void langdet_workspace_free(langdet_workspace_t *ws); + +/* Write the first token's final hidden state to out_embedding[hidden_size]. + * Token IDs must include and , without padding. Returns 0 or -1 on error. */ +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); +void langdet_tok_free(langdet_tok_t *t); + +/* Return malloc-owned IDs wrapped with and , or NULL on error. + * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); + +#endif /* LANGDET_H */ diff --git a/liblangdet/model.c b/liblangdet/model.c new file mode 100644 index 0000000..146bbe7 --- /dev/null +++ b/liblangdet/model.c @@ -0,0 +1,159 @@ +#include "model_internal.h" +#include "config.h" + +#include +#include + +int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim) { + if (t->dtype != DTYPE_F32) { + fprintf(stderr, "langdet: expected F32 tensor: %s\n", name); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (t->shape[i] != shape[i]) { + fprintf(stderr, + "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n", + name, i, (long long)t->shape[i], (long long)shape[i]); + return 0; + } + if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { + fprintf(stderr, "langdet: shape too large or empty for %s\n", name); + return 0; + } + numel *= (size_t)shape[i]; + } + size_t bytes; + if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { + fprintf(stderr, "langdet: invalid data size for %s\n", name); + return 0; + } + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static const float *load_weight(multi_safetensors_t *ms, const char *name, + const int64_t *shape, int ndim) { + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "langdet: tensor not found: %s\n", name); + return NULL; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) + return NULL; + return safetensors_data(sf, t); +} + +static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) { + const langdet_config_t *cfg = &model->config; + langdet_weights_t *w = &model->weights; + char name[256]; + int hidden = cfg->hidden_size, inter = cfg->intermediate_size; + + w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", + (int64_t[]){cfg->vocab_size, hidden}, 2); + w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", + (int64_t[]){cfg->max_position_embeddings, hidden}, 2); + w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", + (int64_t[]){cfg->type_vocab_size, hidden}, 2); + w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); + w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); + if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || + !w->langdet_ln_w || !w->langdet_ln_b) + return -1; + + w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t)); + if (!w->layers) + return -1; + + for (int i = 0; i < cfg->n_layers; i++) { + langdet_layer_t *l = &w->layers[i]; + +#define BERT_W(field, suffix, d0, d1) \ + do { \ + const int64_t e[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(ms, name, e, 2); \ + if (!l->field) \ + return -1; \ + } while (0) +#define BERT_V(field, suffix, d0) \ + do { \ + const int64_t e[1] = {(d0)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(ms, name, e, 1); \ + if (!l->field) \ + return -1; \ + } while (0) + + BERT_W(wq, "attention.self.query.weight", hidden, hidden); + BERT_V(q_bias, "attention.self.query.bias", hidden); + BERT_W(wk, "attention.self.key.weight", hidden, hidden); + BERT_V(k_bias, "attention.self.key.bias", hidden); + BERT_W(wv, "attention.self.value.weight", hidden, hidden); + BERT_V(v_bias, "attention.self.value.bias", hidden); + BERT_W(wo, "attention.output.dense.weight", hidden, hidden); + BERT_V(o_bias, "attention.output.dense.bias", hidden); + BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); + BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); + BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); + BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); + BERT_W(down_proj, "output.dense.weight", hidden, inter); + BERT_V(ffn_out_bias, "output.dense.bias", hidden); + BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); + BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); + +#undef BERT_W +#undef BERT_V + } + return 0; +} + +langdet_model_t *langdet_model_load(const char *model_dir) { + if (!model_dir) + return NULL; + langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t)); + if (!model) + return NULL; + if (langdet_config_parse(&model->config, model_dir) != 0) + goto fail; + model->safetensors = multi_safetensors_open(model_dir); + if (!model->safetensors) { + fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir); + goto fail; + } + if (load_bert_weights(model, model->safetensors) != 0) + goto fail; + return model; + +fail: + langdet_model_free(model); + return NULL; +} + +void langdet_model_free(langdet_model_t *model) { + if (!model) + return; + multi_safetensors_close(model->safetensors); + free(model->weights.layers); + free(model); +} + +const langdet_config_t *langdet_model_config(const langdet_model_t *model) { + return model ? &model->config : NULL; +} diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c new file mode 100644 index 0000000..41f9237 --- /dev/null +++ b/liblangdet/model_blas.c @@ -0,0 +1,68 @@ +#include "model_internal.h" +#include "kernels.h" + +#include + +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model) { + if (!model) + return NULL; + langdet_workspace_t *ws = (langdet_workspace_t *)calloc(1, sizeof(langdet_workspace_t)); + if (!ws) + return NULL; + ws->model = model; + return ws; +} + +void langdet_workspace_free(langdet_workspace_t *ws) { + if (!ws) + return; + free(ws->x); + free(ws->q); + free(ws->k); + free(ws->v); + free(ws->attn_out); + free(ws->proj_out); + free(ws->ffn_up); + free(ws->attn_scores); + free(ws); +} + +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) { + if (seq <= ws->buf_seq_cap) + return 0; + + int cap; + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) + return -1; + +#define R(ptr, n) \ + do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ + return -1; \ + } while (0) + + R(ws->x, c->hidden_size); + R(ws->q, c->hidden_size); + R(ws->k, c->hidden_size); + R(ws->v, c->hidden_size); + R(ws->attn_out, c->hidden_size); + R(ws->proj_out, c->hidden_size); + R(ws->ffn_up, c->intermediate_size); + +#undef R + + ws->buf_seq_cap = cap; + return 0; +} + +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) { + size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) + return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) + return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h new file mode 100644 index 0000000..73a7851 --- /dev/null +++ b/liblangdet/model_internal.h @@ -0,0 +1,17 @@ +#ifndef LANGDET_MODEL_INTERNAL_H +#define LANGDET_MODEL_INTERNAL_H + +#include "model_types.h" +#include "alloc.h" + +/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */ +int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim); + +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq); +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch); + +#endif /* LANGDET_MODEL_INTERNAL_H */ diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h new file mode 100644 index 0000000..2170909 --- /dev/null +++ b/liblangdet/model_types.h @@ -0,0 +1,56 @@ +#ifndef LANGDET_MODEL_TYPES_H +#define LANGDET_MODEL_TYPES_H + +#include "langdet.h" +#include "safetensors.h" + +typedef struct { + const float *wq; + const float *wk; + const float *wv; + const float *wo; + const float *q_bias; + const float *k_bias; + const float *v_bias; + const float *o_bias; + const float *input_norm; + const float *attn_ln_bias; + const float *up_proj; + const float *ffn_inter_bias; + const float *down_proj; + const float *ffn_out_bias; + const float *post_attn_norm; + const float *ffn_ln_bias; +} langdet_layer_t; + +typedef struct { + const float *embed_tokens; + langdet_layer_t *layers; + const float *position_embeddings; + const float *token_type_embeddings; + const float *langdet_ln_w; + const float *langdet_ln_b; +} langdet_weights_t; + +struct langdet_model { + langdet_config_t config; + langdet_weights_t weights; + /* All weight pointers borrow this mapping until model teardown. */ + multi_safetensors_t *safetensors; +}; + +struct langdet_workspace { + const langdet_model_t *model; + int buf_seq_cap; + float *x; + float *q; + float *k; + float *v; + float *attn_out; + float *proj_out; + float *ffn_up; + float *attn_scores; + size_t attn_scores_bytes; +}; + +#endif /* LANGDET_MODEL_TYPES_H */ diff --git a/liblangdet/norm.c b/liblangdet/norm.c new file mode 100644 index 0000000..59e8bdb --- /dev/null +++ b/liblangdet/norm.c @@ -0,0 +1,33 @@ +#include "kernels.h" + +#include + +/* Safe in place: each row's mean and variance are reduced before it is rewritten. */ +void layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + float inv_h = 1.0f / (float)hidden; + for (int s = 0; s < seq_len; s++) { + const float *row = x + (size_t)s * hidden; + float *dst = out + (size_t)s * hidden; + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += row[d]; + mean *= inv_h; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = row[d] - mean; + var += diff * diff; + } + var *= inv_h; + float inv_std = 1.0f / sqrtf(var + eps); + for (int d = 0; d < hidden; d++) + dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; + } +} diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c new file mode 100644 index 0000000..cfba76e --- /dev/null +++ b/liblangdet/safetensors.c @@ -0,0 +1,483 @@ +#include "safetensors.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Minimal JSON parser for safetensors header */ +/* TODO: consider replacing with yyjson */ + +static void skip_whitespace(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') + (*p)++; +} + +static int parse_string(const char **p, char *out, size_t max_len) { + skip_whitespace(p); + if (**p != '"') + return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') + out[i++] = '\n'; + else if (**p == 't') + out[i++] = '\t'; + else if (**p == '"') + out[i++] = '"'; + else if (**p == '\\') + out[i++] = '\\'; + else + out[i++] = **p; + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') + return -1; + (*p)++; + return 0; +} + +static int parse_int(const char **p, int64_t *out) { + skip_whitespace(p); + int64_t val = 0; + int neg = 0; + if (**p == '-') { + neg = 1; + (*p)++; + } + if (**p < '0' || **p > '9') + return -1; + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + *out = neg ? -val : val; + return 0; +} + +static safetensor_dtype_t parse_dtype(const char *s) { + if (strcmp(s, "F32") == 0) + return DTYPE_F32; + if (strcmp(s, "F16") == 0) + return DTYPE_F16; + if (strcmp(s, "BF16") == 0) + return DTYPE_BF16; + if (strcmp(s, "I32") == 0) + return DTYPE_I32; + if (strcmp(s, "I64") == 0) + return DTYPE_I64; + if (strcmp(s, "BOOL") == 0) + return DTYPE_BOOL; + return DTYPE_UNKNOWN; +} + +static int parse_tensor_entry(const char **p, safetensor_t *t) { + skip_whitespace(p); + if (**p != '{') + return -1; + (*p)++; + + t->dtype = DTYPE_UNKNOWN; + t->ndim = 0; + t->data_offset = 0; + t->data_size = 0; + + while (**p && **p != '}') { + skip_whitespace(p); + if (**p == ',') { + (*p)++; + continue; + } + + char key[64]; + if (parse_string(p, key, sizeof(key)) != 0) + return -1; + skip_whitespace(p); + if (**p != ':') + return -1; + (*p)++; + skip_whitespace(p); + + if (strcmp(key, "dtype") == 0) { + char dtype_str[32]; + if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) + return -1; + t->dtype = parse_dtype(dtype_str); + } else if (strcmp(key, "shape") == 0) { + if (**p != '[') + return -1; + (*p)++; + t->ndim = 0; + while (**p && **p != ']') { + skip_whitespace(p); + if (**p == ',') { + (*p)++; + continue; + } + if (t->ndim >= 8) + return -1; + if (parse_int(p, &t->shape[t->ndim]) != 0) + return -1; + t->ndim++; + } + if (**p != ']') + return -1; + (*p)++; + } else if (strcmp(key, "data_offsets") == 0) { + if (**p != '[') + return -1; + (*p)++; + skip_whitespace(p); + int64_t start_i; + if (parse_int(p, &start_i) != 0) + return -1; + skip_whitespace(p); + if (**p != ',') + return -1; + (*p)++; + skip_whitespace(p); + int64_t end_i; + if (parse_int(p, &end_i) != 0) + return -1; + if (start_i < 0 || end_i < start_i) + return -1; + t->data_offset = (size_t)start_i; + t->data_size = (size_t)(end_i - start_i); + skip_whitespace(p); + if (**p != ']') + return -1; + (*p)++; + } else { + /* Skip unknown value */ + if (**p == '"') { + (*p)++; + while (**p && **p != '"') { + if (**p == '\\') + (*p)++; + if (**p) + (*p)++; + } + if (**p == '"') + (*p)++; + } else if (**p == '[') { + int depth = 1; + (*p)++; + while (**p && depth > 0) { + if (**p == '[') + depth++; + else if (**p == ']') + depth--; + (*p)++; + } + } else if (**p == '{') { + int depth = 1; + (*p)++; + while (**p && depth > 0) { + if (**p == '{') + depth++; + else if (**p == '}') + depth--; + (*p)++; + } + } else { + while (**p && **p != ',' && **p != '}') + (*p)++; + } + } + } + if (**p != '}') + return -1; + (*p)++; + return 0; +} + +static int parse_header(safetensors_file_t *sf) { + const char *p = sf->header_json; + skip_whitespace(&p); + if (*p != '{') + return -1; + p++; + + sf->num_tensors = 0; + + while (*p && *p != '}') { + skip_whitespace(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; + + char name[256]; + if (parse_string(&p, name, sizeof(name)) != 0) + return -1; + skip_whitespace(&p); + if (*p != ':') + return -1; + p++; + + if (strcmp(name, "__metadata__") == 0) { + skip_whitespace(&p); + if (*p == '{') { + int depth = 1; + p++; + while (*p && depth > 0) { + if (*p == '{') + depth++; + else if (*p == '}') + depth--; + p++; + } + } + continue; + } + + if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { + fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", + sf->path ? sf->path : "", SAFETENSORS_MAX_TENSORS); + return -1; + } + + safetensor_t *t = &sf->tensors[sf->num_tensors]; + snprintf(t->name, sizeof(t->name), "%s", name); + if (parse_tensor_entry(&p, t) != 0) + return -1; + sf->num_tensors++; + } + if (*p != '}') + return -1; + return 0; +} + +/* Single file operations */ + +safetensors_file_t *safetensors_open(const char *path) { + int fd = open(path, O_RDONLY); + if (fd < 0) + return NULL; + + struct stat st; + if (fstat(fd, &st) < 0) { + close(fd); + return NULL; + } + + size_t file_size = (size_t)st.st_size; + if (file_size < 8) { + close(fd); + return NULL; + } + + void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (data == MAP_FAILED) + return NULL; + + uint64_t header_size = 0; + memcpy(&header_size, data, 8); + if (header_size > file_size - 8) { + munmap(data, file_size); + return NULL; + } + + safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); + if (!sf) { + munmap(data, file_size); + return NULL; + } + + sf->path = strdup(path); + sf->data = data; + sf->file_size = file_size; + sf->header_size = (size_t)header_size; + + sf->header_json = malloc(header_size + 1); + if (!sf->header_json) { + safetensors_close(sf); + return NULL; + } + memcpy(sf->header_json, (char *)data + 8, header_size); + sf->header_json[header_size] = '\0'; + + if (parse_header(sf) != 0) { + safetensors_close(sf); + return NULL; + } + + return sf; +} + +void safetensors_close(safetensors_file_t *sf) { + if (!sf) + return; + if (sf->data) + munmap(sf->data, sf->file_size); + free(sf->path); + free(sf->header_json); + free(sf); +} + +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { + return (const char *)sf->data + 8 + sf->header_size + t->data_offset; +} + +/* Multi-shard operations */ + +static int cmp_shard_names(const void *a, const void *b) { + const char *sa = (const char *)a; + const char *sb = (const char *)b; + return strcmp(sa, sb); +} + +static int parse_shard_name(const char *name, int *idx, int *total) { + int i = 0, n = 0, consumed = 0; + if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) + return -1; + if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) + return -1; + *idx = i; + *total = n; + return 0; +} + +multi_safetensors_t *multi_safetensors_open(const char *model_dir) { + multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); + if (!ms) + return NULL; + + char path[4096]; + + /* Try single file first */ + snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); + safetensors_file_t *sf = safetensors_open(path); + if (sf) { + ms->shards[0] = sf; + ms->num_shards = 1; + return ms; + } + + /* Scan directory for shard files */ + DIR *dir = opendir(model_dir); + if (!dir) { + free(ms); + return NULL; + } + + struct dirent *entry; + char shard_names[SAFETENSORS_MAX_SHARDS][256]; + int seen[SAFETENSORS_MAX_SHARDS] = {0}; + int n_shards = 0; + int expected_total = 0; + + while ((entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "model-", 6) == 0 && + strstr(entry->d_name, ".safetensors") != NULL) { + int shard_idx = 0, shard_total = 0; + if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { + fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (shard_total > SAFETENSORS_MAX_SHARDS) { + fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, + SAFETENSORS_MAX_SHARDS); + closedir(dir); + free(ms); + return NULL; + } + if (expected_total == 0) { + expected_total = shard_total; + } else if (expected_total != shard_total) { + fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", + entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (seen[shard_idx - 1]) { + fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); + closedir(dir); + free(ms); + return NULL; + } + seen[shard_idx - 1] = 1; + + snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); + n_shards++; + } + } + closedir(dir); + + if (n_shards == 0) { + fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); + free(ms); + return NULL; + } + if (expected_total != n_shards) { + fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", + expected_total, n_shards, model_dir); + free(ms); + return NULL; + } + for (int i = 0; i < expected_total; i++) { + if (!seen[i]) { + fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, + expected_total, model_dir); + free(ms); + return NULL; + } + } + + /* Sort shard names to ensure consistent ordering */ + qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); + + /* Open each shard */ + for (int i = 0; i < n_shards; i++) { + snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); + ms->shards[i] = safetensors_open(path); + if (!ms->shards[i]) { + fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); + multi_safetensors_close(ms); + return NULL; + } + } + ms->num_shards = n_shards; + return ms; +} + +void multi_safetensors_close(multi_safetensors_t *ms) { + if (!ms) + return; + for (int i = 0; i < ms->num_shards; i++) { + safetensors_close(ms->shards[i]); + } + free(ms); +} + +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { + for (int s = 0; s < ms->num_shards; s++) { + safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) { + if (out_sf) + *out_sf = sf; + return &sf->tensors[i]; + } + } + } + return NULL; +} diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h new file mode 100644 index 0000000..dca68a7 --- /dev/null +++ b/liblangdet/safetensors.h @@ -0,0 +1,64 @@ +/* + * Safetensors file format reader with multi-shard support. + */ + +#ifndef SAFETENSORS_H +#define SAFETENSORS_H + +#include +#include + +#define SAFETENSORS_MAX_TENSORS 1024 +#define SAFETENSORS_MAX_SHARDS 8 + +typedef enum { + DTYPE_F32 = 0, + DTYPE_F16 = 1, + DTYPE_BF16 = 2, + DTYPE_I32 = 3, + DTYPE_I64 = 4, + DTYPE_BOOL = 5, + DTYPE_UNKNOWN = -1 +} safetensor_dtype_t; + +typedef struct { + char name[256]; + safetensor_dtype_t dtype; + int ndim; + int64_t shape[8]; + size_t data_offset; + size_t data_size; +} safetensor_t; + +typedef struct { + char *path; + void *data; + size_t file_size; + size_t header_size; + char *header_json; + int num_tensors; + safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; +} safetensors_file_t; + +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ +typedef struct { + safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; + int num_shards; +} multi_safetensors_t; + +/* Open a single safetensors file (memory-mapped) */ +safetensors_file_t *safetensors_open(const char *path); +void safetensors_close(safetensors_file_t *sf); + +/* Open model from directory (auto-detects single file or multi-shard) */ +multi_safetensors_t *multi_safetensors_open(const char *model_dir); +void multi_safetensors_close(multi_safetensors_t *ms); + +/* Find a tensor by name across all shards */ +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); + +/* Get raw pointer to tensor data (within mmap'd region) */ +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); + +#endif /* SAFETENSORS_H */ diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c new file mode 100644 index 0000000..6c30ba1 --- /dev/null +++ b/liblangdet/sentencepiece.c @@ -0,0 +1,905 @@ +/* + * SentencePiece Unigram tokenizer. + * + * The implementation reads the small subset of ModelProto needed at runtime: + * pieces, scores, piece types, trainer model_type, and normalizer options. The + * normalizer supports SentencePiece's precompiled charsmap format used by + * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program + * over UTF-8 byte offsets. + */ + +#include "sentencepiece.h" + +#include +#include +#include +#include +#include +#include + +enum { + SP_MODEL_UNIGRAM = 1, + SP_PIECE_NORMAL = 1, + SP_PIECE_UNKNOWN = 2, + SP_PIECE_CONTROL = 3, + SP_PIECE_USER_DEFINED = 4, + SP_PIECE_UNUSED = 5, + SP_PIECE_BYTE = 6, +}; + +typedef struct { + const char *key; + uint16_t len; + int val; +} sp_entry_t; + +typedef struct { + int id; + int prev; + double score; +} sp_best_t; + +struct tok_spm_workspace { + char *norm; + size_t norm_len; + size_t norm_cap; + sp_best_t *best; + int best_cap; + int *ids; + int ids_cap; +}; + +static uint64_t fnv1a_bytes(const char *s, size_t n) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < n; i++) { + h ^= (unsigned char)s[i]; + h *= 1099511628211ULL; + } + return h; +} + +static int next_pow2(int x) { + int p = 1; + while (p < x && p <= (1 << 29)) + p <<= 1; + return p; +} + +static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) { + if (len <= 0 || len > UINT16_MAX) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) { + map[pos].val = val; + return 0; + } + pos = (int)((pos + 1) & (int)mask); + } + map[pos].key = key; + map[pos].len = (uint16_t)len; + map[pos].val = val; + return 0; +} + +static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { + if (!map || len <= 0) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) + return map[pos].val; + pos = (int)((pos + 1) & (int)mask); + } + return -1; +} + +static int is_tokenizable_type(int type) { + return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || + type == SP_PIECE_BYTE; +} + +static int reserve_chars(char **buf, size_t *cap, size_t need) { + if (need <= *cap) + return 0; + size_t nc = *cap ? *cap : 128; + while (nc < need) { + if (nc > SIZE_MAX / 2) + return -1; + nc *= 2; + } + char *p = (char *)realloc(*buf, nc); + if (!p) + return -1; + *buf = p; + *cap = nc; + return 0; +} + +static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) { + if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0) + return -1; + memcpy(*buf + *len, s, n); + *len += n; + (*buf)[*len] = '\0'; + return 0; +} + +static int reserve_best(tok_spm_workspace_t *ws, int need) { + if (need <= ws->best_cap) + return 0; + int nc = ws->best_cap ? ws->best_cap : 128; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->best = p; + ws->best_cap = nc; + return 0; +} + +static int reserve_ids(tok_spm_workspace_t *ws, int need) { + if (need <= ws->ids_cap) + return 0; + int nc = ws->ids_cap ? ws->ids_cap : 64; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->ids = p; + ws->ids_cap = nc; + return 0; +} + +static int utf8_len(unsigned char c) { + if (c < 0x80) + return 1; + if ((c & 0xE0) == 0xC0) + return 2; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xF8) == 0xF0) + return 4; + return 1; +} + +static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) { + if (off >= n) + return 0; + const unsigned char *p = (const unsigned char *)s + off; + int len = utf8_len(p[0]); + if (off + (size_t)len > n) { + *len_out = 1; + return 0; + } + if (len > 1) { + for (int i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) { + *len_out = 1; + return 0; + } + } + } + *len_out = (size_t)len; + return 1; +} + +static uint32_t read_le32(const unsigned char *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) { + uint64_t v = 0; + int shift = 0; + while (*p < end && shift <= 63) { + unsigned char c = *(*p)++; + v |= (uint64_t)(c & 0x7f) << shift; + if (!(c & 0x80)) { + *out = v; + return 0; + } + shift += 7; + } + return -1; +} + +static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) { + uint64_t key = 0; + if (pb_read_varint(p, end, &key) != 0) + return -1; + *field = (int)(key >> 3); + *wire = (int)(key & 7); + return *field > 0 ? 0 : -1; +} + +static int pb_read_len(const unsigned char **p, + const unsigned char *end, + const unsigned char **data, + size_t *len) { + uint64_t n = 0; + if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p)) + return -1; + *data = *p; + *len = (size_t)n; + *p += n; + return 0; +} + +static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) { + uint64_t v = 0; + const unsigned char *data = NULL; + size_t len = 0; + switch (wire) { + case 0: + return pb_read_varint(p, end, &v); + case 1: + if ((size_t)(end - *p) < 8) + return -1; + *p += 8; + return 0; + case 2: + return pb_read_len(p, end, &data, &len); + case 5: + if ((size_t)(end - *p) < 4) + return -1; + *p += 4; + return 0; + default: + return -1; + } +} + +static char *memdup0(const unsigned char *data, size_t len) { + char *s = (char *)malloc(len + 1); + if (!s) + return NULL; + memcpy(s, data, len); + s[len] = '\0'; + return s; +} + +static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + char *piece = NULL; + float score = 0.0f; + int type = SP_PIECE_NORMAL; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + goto fail; + if (field == 1 && wire == 2) { + const unsigned char *s = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX) + goto fail; + free(piece); + piece = memdup0(s, n); + if (!piece) + goto fail; + tok->piece_lens[idx] = (uint16_t)n; + } else if (field == 2 && wire == 5) { + if ((size_t)(end - p) < 4) + goto fail; + uint32_t bits = read_le32(p); + memcpy(&score, &bits, sizeof(score)); + p += 4; + } else if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX) + goto fail; + type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + goto fail; + } + } + if (!piece) + goto fail; + tok->id_to_piece[idx] = piece; + tok->scores[idx] = score; + tok->types[idx] = (uint8_t)type; + if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) + tok->max_piece_len = tok->piece_lens[idx]; + return 0; +fail: + free(piece); + return -1; +} + +static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int model_type = SP_MODEL_UNIGRAM; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX) + return -1; + model_type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (model_type != SP_MODEL_UNIGRAM) { + fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n", + model_type); + return -1; + } + (void)tok; + return 0; +} + +static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if ((field == 3 || field == 4 || field == 5) && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0) + return -1; + if (field == 3) + tok->add_dummy_prefix = v != 0; + else if (field == 4) + tok->remove_extra_whitespaces = v != 0; + else + tok->escape_whitespaces = v != 0; + } else if (field == 2 && wire == 2) { + const unsigned char *blob = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &blob, &n) != 0) + return -1; + if (n >= 4) { + uint32_t xcda_bytes = read_le32(blob); + if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) { + tok->xcda_count = xcda_bytes / 4; + tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda)); + if (!tok->xcda) + return -1; + for (size_t i = 0; i < tok->xcda_count; i++) + tok->xcda[i] = read_le32(blob + 4 + i * 4); + tok->prefix_replacements_len = n - 4 - xcda_bytes; + tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1); + if (!tok->prefix_replacements) + return -1; + memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes, + tok->prefix_replacements_len); + tok->prefix_replacements[tok->prefix_replacements_len] = '\0'; + } + } + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return 0; +} + +static int count_pieces(const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int n = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 1 && wire == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0) + return -1; + n++; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return n; +} + +static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { + int n_pieces = count_pieces(data, len); + if (n_pieces <= 0) + return -1; + tok->piece_count = n_pieces; + tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); + tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); + tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); + tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); + if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) + return -1; + + const unsigned char *p = data, *end = data + len; + int piece_idx = 0; + int saw_trainer = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (wire == 2 && field == 1) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces || + parse_piece(tok, piece_idx++, msg, msg_len) != 0) + return -1; + } else if (wire == 2 && field == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) + return -1; + saw_trainer = 1; + } else if (wire == 2 && field == 3) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0) + return -1; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (piece_idx != n_pieces || !saw_trainer) + return -1; + return 0; +} + +static int raw_piece_id(const tok_spm_t *tok, const char *piece) { + if (!tok || !piece) + return -1; + return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); +} + +static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { + if (!tok || raw_id < 0 || raw_id >= tok->piece_count) + return -1; + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { + if (raw_id == 1) + return 0; /* */ + if (raw_id == 2) + return 2; /* */ + if (raw_id == 0) + return 3; /* */ + return raw_id + 1; + } + return raw_id; +} + +static int file_contains_xlm_roberta(const char *model_dir, const char *name) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; + FILE *f = fopen(path, "rb"); + if (!f) + return 0; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return 0; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return 0; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return 0; + } + size_t got = fread(buf, 1, (size_t)sz, f); + fclose(f); + buf[got] = '\0'; + int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || + strstr(buf, "XLMRobertaTokenizerFast") != NULL || + strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; + free(buf); + return yes; +} + +static int detect_xlm_roberta(const char *model_dir) { + return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || + file_contains_xlm_roberta(model_dir, "config.json"); +} + +static int find_model_file(const char *model_dir, char path[1024]) { + static const char *names[] = { + "sentencepiece.bpe.model", + "tokenizer.model", + "spiece.model", + }; + for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { + int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); + if (n < 0 || n >= 1024) + return -1; + FILE *f = fopen(path, "rb"); + if (f) { + fclose(f); + return 0; + } + if (errno != ENOENT) + return -1; + } + return -1; +} + +static int load_file(const char *path, unsigned char **out, size_t *out_len) { + FILE *f = fopen(path, "rb"); + if (!f) + return -1; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + unsigned char *buf = (unsigned char *)malloc((size_t)sz); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); + free(buf); + return -1; + } + fclose(f); + *out = buf; + *out_len = (size_t)sz; + return 0; +} + +static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) { + uint32_t v = tok->xcda[idx]; + return (v >> 10) << ((v & (1u << 9)) >> 6); +} + +static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) | 0xffu); +} + +static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; } + +static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) - 1u); +} + +typedef struct { + const char *data; + size_t len; + size_t consumed; +} norm_prefix_t; + +static norm_prefix_t +normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) { + static const char repl[] = "\xef\xbf\xbd"; + if (off >= input_len) + return (norm_prefix_t){input + off, 0, 0}; + + size_t best_len = 0; + size_t best_repl = 0; + if (tok->xcda && tok->xcda_count > 0) { + uint32_t node = xcda_base(tok, 0); + for (size_t p = off; p < input_len; p++) { + unsigned char c = (unsigned char)input[p]; + if (c == 0) + break; + node ^= c; + if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c) + break; + int leaf = xcda_leaf(tok, node); + node ^= xcda_base(tok, node); + if (leaf) { + best_len = p - off + 1; + best_repl = xcda_value(tok, node); + } + } + } + if (best_len > 0 && best_repl < tok->prefix_replacements_len) { + const char *r = tok->prefix_replacements + best_repl; + size_t max = tok->prefix_replacements_len - best_repl; + size_t n = strnlen(r, max); + return (norm_prefix_t){r, n, best_len}; + } + + size_t clen = 1; + if (valid_utf8_one(input, input_len, off, &clen)) + return (norm_prefix_t){input + off, clen, clen}; + return (norm_prefix_t){repl, 3, 1}; +} + +static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) { + ws->norm_len = 0; + if (!text) + return -1; + size_t input_len = strlen(text); + const char space[] = "\xe2\x96\x81"; /* U+2581 */ + const char *space_s = tok->escape_whitespaces ? space : " "; + size_t space_n = tok->escape_whitespaces ? 3 : 1; + int processing_non_ws = 0; + int space_prepended = 0; + + for (size_t off = 0; off < input_len;) { + norm_prefix_t r = normalize_prefix(tok, text, input_len, off); + for (size_t i = 0; i < r.len; i++) { + char c = r.data[i]; + if (c != ' ') { + if (!processing_non_ws) { + processing_non_ws = 1; + if ((tok->add_dummy_prefix && !space_prepended) || + tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != + 0) + return -1; + space_prepended = 1; + } + } + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) + return -1; + } else { + processing_non_ws = 0; + if (!tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) + return -1; + } + } + } + off += r.consumed ? r.consumed : 1; + } + if (ws->norm) + ws->norm[ws->norm_len] = '\0'; + return 0; +} + +static double compute_unknown_score(const tok_spm_t *tok) { + float min_score = FLT_MAX; + for (int i = 0; i < tok->piece_count; i++) { + if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) + min_score = tok->scores[i]; + } + if (min_score == FLT_MAX) + min_score = -10.0f; + return (double)min_score - 10.0; +} + +static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { + size_t n = ws->norm_len; + if (n == 0) + return 0; + if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 || + reserve_ids(ws, (int)n + 1) != 0) + return -1; + for (size_t i = 0; i <= n; i++) + ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; + ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; + double unk_score = tok->unknown_score; + + for (size_t off = 0; off < n;) { + size_t cp_len = 1; + valid_utf8_one(ws->norm, n, off, &cp_len); + sp_best_t cur = ws->best[off]; + if (cur.score <= -DBL_MAX / 2.0) { + off += cp_len; + continue; + } + + int single_cp_found = 0; + for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { + int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, + (int)(end - off)); + if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { + if (end - off == cp_len) + single_cp_found = 1; + double sc = + cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; + } + if (end == n) + break; + size_t next_len = 1; + valid_utf8_one(ws->norm, n, end, &next_len); + end += next_len; + } + if (!single_cp_found) { + size_t end = off + cp_len; + double sc = cur.score + unk_score; + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; + } + off += cp_len; + } + + int out_n = 0; + int prev_unk = 0; + for (size_t off = n; off > 0;) { + sp_best_t b = ws->best[off]; + if (b.prev < 0 || b.prev >= (int)off) + return -1; + int is_unk = b.id == tok->unk_id; + if (!(prev_unk && is_unk)) { + if (out_n >= ws->ids_cap) + return -1; + ws->ids[out_n++] = b.id; + } + prev_unk = is_unk; + off = (size_t)b.prev; + } + for (int i = 0; i < out_n / 2; i++) { + int t = ws->ids[i]; + ws->ids[i] = ws->ids[out_n - 1 - i]; + ws->ids[out_n - 1 - i] = t; + } + return out_n; +} + +tok_spm_t *tok_spm_load(const char *model_dir) { + if (!model_dir) + return NULL; + char path[1024]; + if (find_model_file(model_dir, path) != 0) { + fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); + return NULL; + } + unsigned char *data = NULL; + size_t len = 0; + if (load_file(path, &data, &len) != 0) { + fprintf(stderr, "sentencepiece: cannot read %s\n", path); + return NULL; + } + + tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t)); + if (!tok) { + free(data); + return NULL; + } + tok->add_dummy_prefix = 1; + tok->remove_extra_whitespaces = 1; + tok->escape_whitespaces = 1; + tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; + if (parse_model(tok, data, len) != 0) { + fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); + free(data); + tok_spm_free(tok); + return NULL; + } + free(data); + + int cap = next_pow2(tok->piece_count * 2 + 1); + tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); + if (!tok->piece_map) { + tok_spm_free(tok); + return NULL; + } + tok->piece_map_cap = cap; + for (int i = 0; i < tok->piece_count; i++) { + if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], + tok->piece_lens[i], i) != 0) { + tok_spm_free(tok); + return NULL; + } + } + + tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "")); + tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "")); + tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "")); + tok->pad_id = + tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "")); + tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA + ? tok->piece_count + 1 + : raw_to_model_id(tok, raw_piece_id(tok, "")); + tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; + tok->unknown_score = compute_unknown_score(tok); + if (tok->unk_id < 0 || tok->max_piece_len <= 0) { + fprintf(stderr, "sentencepiece: missing or normal pieces in %s\n", path); + tok_spm_free(tok); + return NULL; + } + return tok; +} + +int tok_spm_token_id(const tok_spm_t *tok, const char *token) { + if (!tok || !token) + return -1; + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { + if (!strcmp(token, "")) + return 0; + if (!strcmp(token, "")) + return 1; + if (!strcmp(token, "")) + return 2; + if (!strcmp(token, "")) + return 3; + if (!strcmp(token, "")) + return tok->piece_count + 1; + } + return raw_to_model_id(tok, raw_piece_id(tok, token)); +} + +static int tok_spm_encode_into(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return -1; + if (normalize_into(tok, ws, text) != 0) + return -1; + int n = segment(tok, ws); + if (n < 0) + return -1; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return 0; + if (!out_ids || out_cap < n) + return -2; + memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); + return 0; +} + +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return NULL; + int n = 0; + int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); + if (rc < 0 && rc != -2) + return NULL; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return NULL; + int *ids = (int *)malloc((size_t)n * sizeof(*ids)); + if (!ids) + return NULL; + memcpy(ids, ws->ids, (size_t)n * sizeof(*ids)); + return ids; +} + +tok_spm_workspace_t *tok_spm_workspace_new(void) { + return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t)); +} + +void tok_spm_workspace_free(tok_spm_workspace_t *ws) { + if (!ws) + return; + free(ws->norm); + free(ws->best); + free(ws->ids); + free(ws); +} + +void tok_spm_free(tok_spm_t *tok) { + if (!tok) + return; + if (tok->id_to_piece) { + for (int i = 0; i < tok->piece_count; i++) + free(tok->id_to_piece[i]); + } + free(tok->id_to_piece); + free(tok->piece_lens); + free(tok->scores); + free(tok->types); + free(tok->piece_map); + free(tok->xcda); + free(tok->prefix_replacements); + free(tok); +} diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h new file mode 100644 index 0000000..2230538 --- /dev/null +++ b/liblangdet/sentencepiece.h @@ -0,0 +1,71 @@ +/* + * SentencePiece tokenizer for XLM-R style models. + * + * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, + * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R + * and multilingual E5. It encodes text only; frontends add the model's special + * token layout, e.g. ... for XLM-R. + */ + +#ifndef TOK_SPM_H +#define TOK_SPM_H + +#include +#include + +typedef enum { + TOK_SPM_IDENTITY = 0, + TOK_SPM_XLM_ROBERTA = 1, +} tok_spm_id_map_t; + +typedef struct { + char **id_to_piece; /* [piece_count] raw SentencePiece ids */ + uint16_t *piece_lens; /* byte length for id_to_piece entries */ + float *scores; /* SentencePiece score per raw id */ + uint8_t *types; /* SentencePiece piece type per raw id */ + int piece_count; /* raw SentencePiece vocabulary size */ + int vocab_size; /* model embedding vocab size after id mapping */ + int unk_id; /* model-space unknown-token id */ + int bos_id; /* model-space beginning-token id, or -1 */ + int eos_id; /* model-space end-token id, or -1 */ + int pad_id; /* model-space pad-token id, or -1 */ + int mask_id; /* model-space mask-token id, or -1 */ + int max_piece_len; /* max normal/user/unused piece byte length */ + double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ + int add_dummy_prefix; + int remove_extra_whitespaces; + int escape_whitespaces; + tok_spm_id_map_t id_map; + + void *piece_map; /* piece bytes -> raw SentencePiece id */ + int piece_map_cap; + + uint32_t *xcda; + size_t xcda_count; + char *prefix_replacements; + size_t prefix_replacements_len; +} tok_spm_t; + +typedef struct tok_spm_workspace tok_spm_workspace_t; + +/* Load the first supported SentencePiece model found in model_dir. Returns NULL + * on missing model file, unsupported model type, malformed protobuf, or OOM. */ +tok_spm_t *tok_spm_load(const char *model_dir); + +/* Resolve a named token (e.g. "", "", "", "") to a + * model-space id, or -1 when absent. */ +int tok_spm_token_id(const tok_spm_t *tok, const char *token); + +tok_spm_workspace_t *tok_spm_workspace_new(void); +void tok_spm_workspace_free(tok_spm_workspace_t *ws); + +/* Return malloc-owned model-space token IDs, reusing workspace scratch. + * Empty text returns NULL with a zero count. */ +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens); + +void tok_spm_free(tok_spm_t *tok); + +#endif /* TOK_SPM_H */ diff --git a/liblangdet/threadpool.c b/liblangdet/threadpool.c new file mode 100644 index 0000000..6cb4252 --- /dev/null +++ b/liblangdet/threadpool.c @@ -0,0 +1,218 @@ +/* + * CPU thread pool. + */ + +#if defined(__linux__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#include "threadpool.h" + +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#else +# ifdef __linux__ +# include +# endif +# include +#endif + +#if defined(USE_OPENBLAS) +void openblas_set_num_threads(int num_threads); +#endif + +#define TP_MAX_THREADS 16 + +static struct { + pthread_t threads[TP_MAX_THREADS - 1]; + int tids[TP_MAX_THREADS - 1]; + int n_threads; + int shutdown; + + parallel_fn_t fn; + void *arg; + uint64_t generation; + + pthread_mutex_t mutex; + pthread_cond_t cond_work; + pthread_cond_t cond_done; + int n_done; +} tp = { + .n_threads = 1, + .shutdown = 0, + .generation = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond_work = PTHREAD_COND_INITIALIZER, + .cond_done = PTHREAD_COND_INITIALIZER, +}; + +static void *worker_loop(void *arg) { + int tid = *(int *)arg; + uint64_t my_gen = 0; + + for (;;) { + pthread_mutex_lock(&tp.mutex); + while (tp.generation == my_gen && !tp.shutdown) + pthread_cond_wait(&tp.cond_work, &tp.mutex); + if (tp.shutdown) { + pthread_mutex_unlock(&tp.mutex); + return NULL; + } + my_gen = tp.generation; + parallel_fn_t fn = tp.fn; + void *a = tp.arg; + int nt = tp.n_threads; + pthread_mutex_unlock(&tp.mutex); + + fn(tid, nt, a); + + pthread_mutex_lock(&tp.mutex); + if (++tp.n_done >= tp.n_threads - 1) + pthread_cond_signal(&tp.cond_done); + pthread_mutex_unlock(&tp.mutex); + } +} + +void tp_set_threads(int n) { + if (n < 1) + n = 1; + if (n > TP_MAX_THREADS) + n = TP_MAX_THREADS; + +#if defined(USE_OPENBLAS) + /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the + * auxiliary pool serial to avoid nested thread contention. */ + int blas_threads = n; + openblas_set_num_threads(blas_threads); + n = 1; +#endif + + /* Shutdown existing workers */ + if (tp.n_threads > 1) { + pthread_mutex_lock(&tp.mutex); + tp.shutdown = 1; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + for (int i = 0; i < tp.n_threads - 1; i++) + pthread_join(tp.threads[i], NULL); + tp.shutdown = 0; + tp.generation = 0; + } + + tp.n_threads = n; + if (n <= 1) + return; + + for (int i = 0; i < n - 1; i++) { + tp.tids[i] = i + 1; + pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); + } +} + +#ifndef __APPLE__ +/* CPU quota available to this process, in whole cores, or 0 if uncapped. + * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host + * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process + * can actually use. + * + * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: + * threads use the period budget early, then get throttled for the rest of the + * period, causing large bursty stalls. Example: a 16-thread default against a + * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. + * + * Floor instead of ceil, so a fully busy pool stays under quota and avoids + * throttling. + */ +static int cgroup_cpu_quota(void) { + /* cgroup v2: "/sys/fs/cgroup/cpu.max" is " ", or + * "max " when unlimited. */ + FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r"); + if (f) { + char quota[32]; + long period = 0; + int got = fscanf(f, "%31s %ld", quota, &period); + fclose(f); + if (got == 2 && period > 0 && strcmp(quota, "max") != 0) { + long q = atol(quota); + if (q > 0) + return q / period < 1 ? 1 : (int)(q / period); + } + } + /* cgroup v1: separate files; quota -1 means unlimited. */ + long q = -1, period = 0; + f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r"); + if (f) { + if (fscanf(f, "%ld", &q) != 1) + q = -1; + fclose(f); + } + f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r"); + if (f) { + if (fscanf(f, "%ld", &period) != 1) + period = 0; + fclose(f); + } + if (q > 0 && period > 0) + return q / period < 1 ? 1 : (int)(q / period); + return 0; +} +#endif + +int tp_get_num_cpus(void) { +#ifdef __APPLE__ + int n = 0; + size_t len = sizeof(n); + sysctlbyname("hw.ncpu", &n, &len, NULL, 0); + return n > 0 ? n : 1; +#else + int online = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (online < 1) + online = 1; +# ifdef __linux__ + cpu_set_t set; + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + int affinity = CPU_COUNT(&set); + if (affinity >= 1 && affinity < online) + online = affinity; + } +# endif + /* Honor a cgroup CPU quota when it is tighter than the visible core count, + * so the default thread pool does not oversubscribe a CPU-limited + * container and get throttled. */ + int quota = cgroup_cpu_quota(); + if (quota >= 1 && quota < online) + return quota; + return online; +#endif +} + +int tp_num_threads(void) { return tp.n_threads; } + +/* Dispatch work to all threads; main thread is tid=0 */ +void parallel_for(parallel_fn_t fn, void *arg) { + if (tp.n_threads <= 1) { + fn(0, 1, arg); + return; + } + + pthread_mutex_lock(&tp.mutex); + tp.fn = fn; + tp.arg = arg; + tp.n_done = 0; + tp.generation++; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + + fn(0, tp.n_threads, arg); + + pthread_mutex_lock(&tp.mutex); + while (tp.n_done < tp.n_threads - 1) + pthread_cond_wait(&tp.cond_done, &tp.mutex); + pthread_mutex_unlock(&tp.mutex); +} diff --git a/liblangdet/threadpool.h b/liblangdet/threadpool.h new file mode 100644 index 0000000..f9739a1 --- /dev/null +++ b/liblangdet/threadpool.h @@ -0,0 +1,30 @@ +#ifndef TP_THREADPOOL_H +#define TP_THREADPOOL_H + +/* Persistent worker pool for CPU kernels. + * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. + * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. + * + * Kept separate from the kernels so they depend only on parallel_for and + * tp_num_threads, not pool internals. + * + * kernels.h includes this, so kernel callers need no extra include. */ + +/* Worker callback: (thread index, total threads, user arg). */ +typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); + +/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call + * before inference. */ +void tp_set_threads(int n); + +/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */ +int tp_get_num_cpus(void); + +/* Current pool size (1 == serial). Kernels gate parallelism on this. */ +int tp_num_threads(void); + +/* Run fn across the pool; the calling thread participates as tid 0. Blocks + * until every worker has finished. */ +void parallel_for(parallel_fn_t fn, void *arg); + +#endif /* TP_THREADPOOL_H */ diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c new file mode 100644 index 0000000..5e26a18 --- /dev/null +++ b/liblangdet/tokenize.c @@ -0,0 +1,85 @@ +#include "langdet.h" +#include "sentencepiece.h" + +#include +#include +#include +#include + +struct langdet_tok { + tok_spm_t *spm; + tok_spm_workspace_t *workspace; + int cls_id; + int sep_id; +}; + +static void set_err(char *err, size_t errlen, const char *msg) { + if (err && errlen) + snprintf(err, errlen, "%s", msg); +} + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { + if (!model_dir) { + set_err(err, errlen, "model_dir is NULL"); + return NULL; + } + langdet_tok_t *t = calloc(1, sizeof(*t)); + if (!t) { + set_err(err, errlen, "out of memory"); + return NULL; + } + t->spm = tok_spm_load(model_dir); + if (!t->spm) { + set_err(err, errlen, "failed to load SentencePiece tokenizer"); + goto fail; + } + t->workspace = tok_spm_workspace_new(); + if (!t->workspace) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_spm_token_id(t->spm, ""); + t->sep_id = tok_spm_token_id(t->spm, ""); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "SentencePiece vocab missing /"); + goto fail; + } + return t; + +fail: + langdet_tok_free(t); + return NULL; +} + +void langdet_tok_free(langdet_tok_t *t) { + if (!t) + return; + tok_spm_workspace_free(t->workspace); + tok_spm_free(t->spm); + free(t); +} + +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { + if (n_out) + *n_out = 0; + if (!t || !text) + return NULL; + int n = 0; + int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); + if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { + free(core); + return NULL; + } + /* Empty text still produces the two special tokens. */ + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return NULL; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = t->cls_id; + ids[n + 1] = t->sep_id; + if (n_out) + *n_out = n + 2; + return ids; +} -- cgit v1.3.1