aboutsummaryrefslogtreecommitdiff
path: root/liblangdet
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet')
-rw-r--r--liblangdet/LICENSE20
-rw-r--r--liblangdet/act.c29
-rw-r--r--liblangdet/alloc.c65
-rw-r--r--liblangdet/alloc.h22
-rw-r--r--liblangdet/attn.c242
-rw-r--r--liblangdet/config.c67
-rw-r--r--liblangdet/config.h8
-rw-r--r--liblangdet/forward.c96
-rw-r--r--liblangdet/gemm.c49
-rw-r--r--liblangdet/generic.c27
-rw-r--r--liblangdet/impl.h254
-rw-r--r--liblangdet/kernels.h42
-rw-r--r--liblangdet/langdet.h49
-rw-r--r--liblangdet/model.c159
-rw-r--r--liblangdet/model_blas.c68
-rw-r--r--liblangdet/model_internal.h17
-rw-r--r--liblangdet/model_types.h56
-rw-r--r--liblangdet/norm.c33
-rw-r--r--liblangdet/safetensors.c483
-rw-r--r--liblangdet/safetensors.h64
-rw-r--r--liblangdet/sentencepiece.c905
-rw-r--r--liblangdet/sentencepiece.h71
-rw-r--r--liblangdet/threadpool.c218
-rw-r--r--liblangdet/threadpool.h30
-rw-r--r--liblangdet/tokenize.c85
25 files changed, 3159 insertions, 0 deletions
diff --git a/liblangdet/LICENSE b/liblangdet/LICENSE
new file mode 100644
index 0000000..a00caa7
--- /dev/null
+++ b/liblangdet/LICENSE
@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2026 The ffwd authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/liblangdet/act.c b/liblangdet/act.c
new file mode 100644
index 0000000..3912f2f
--- /dev/null
+++ b/liblangdet/act.c
@@ -0,0 +1,29 @@
+#include "kernels.h"
+
+#include <math.h>
+
+void gelu_inplace(float *x, int n) {
+ /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */
+ for (int i = 0; i < n; i++)
+ x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
+}
+
+void softmax(float *x, int rows, int cols) {
+ for (int r = 0; r < rows; r++) {
+ float *row = x + r * cols;
+ float max_val = row[0];
+ for (int c = 1; c < cols; c++) {
+ if (row[c] > max_val)
+ max_val = row[c];
+ }
+ float sum = 0.0f;
+ for (int c = 0; c < cols; c++) {
+ row[c] = expf(row[c] - max_val);
+ sum += row[c];
+ }
+ float inv_sum = 1.0f / sum;
+ for (int c = 0; c < cols; c++) {
+ row[c] *= inv_sum;
+ }
+ }
+}
diff --git a/liblangdet/alloc.c b/liblangdet/alloc.c
new file mode 100644
index 0000000..4bc1496
--- /dev/null
+++ b/liblangdet/alloc.c
@@ -0,0 +1,65 @@
+#include "alloc.h"
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <limits.h>
+
+/* Sequence-capacity growth policy for the forward-pass workspaces:
+ * For new growth:
+ * - never allocate less than the minimum required size.
+ * - round up to the granularity so small length changes do not trigger a realloc on every call.
+ * - keep existing capacity if it is already larger. */
+#define LANGDET_MIN_WORKSPACE_SEQ_CAP 16
+#define LANGDET_WORKSPACE_SEQ_GRANULARITY 16
+
+int mul_size(size_t a, size_t b, size_t *out) {
+ if (a != 0 && b > SIZE_MAX / a)
+ return -1;
+ *out = a * b;
+ return 0;
+}
+
+int grow_cap(int current, int needed, int *out) {
+ if (current < 0 || needed <= 0)
+ return -1;
+
+ int cap = needed;
+ if (cap < LANGDET_MIN_WORKSPACE_SEQ_CAP)
+ cap = LANGDET_MIN_WORKSPACE_SEQ_CAP;
+
+ int rem = cap % LANGDET_WORKSPACE_SEQ_GRANULARITY;
+ if (rem != 0) {
+ int pad = LANGDET_WORKSPACE_SEQ_GRANULARITY - rem;
+ if (cap > INT_MAX - pad)
+ return -1;
+ cap += pad;
+ }
+
+ if (cap < current)
+ cap = current;
+
+ *out = cap;
+ return 0;
+}
+
+int realloc_floats(float **ptr, size_t count) {
+ size_t bytes;
+ if (mul_size(count, sizeof(float), &bytes) != 0)
+ return -1;
+
+ void *p = realloc(*ptr, bytes);
+ if (!p && bytes != 0)
+ return -1;
+ *ptr = (float *)p;
+ return 0;
+}
+
+int realloc_floats_2d(float **ptr, int rows, int cols) {
+ if (rows < 0 || cols < 0)
+ return -1;
+
+ size_t count;
+ if (mul_size((size_t)rows, (size_t)cols, &count) != 0)
+ return -1;
+ return realloc_floats(ptr, count);
+}
diff --git a/liblangdet/alloc.h b/liblangdet/alloc.h
new file mode 100644
index 0000000..f70f1f7
--- /dev/null
+++ b/liblangdet/alloc.h
@@ -0,0 +1,22 @@
+#ifndef LANGDET_ALLOC_H
+#define LANGDET_ALLOC_H
+
+#include <stddef.h>
+
+/* Checked size arithmetic and typed allocation helpers.
+ * Return -1 or NULL on overflow/allocation failure.
+ * realloc_* leave the original pointer unchanged on failure. */
+
+/* out = a * b, or -1 if the product overflows size_t. */
+int mul_size(size_t a, size_t b, size_t *out);
+
+/* Grow sequence capacity without shrinking. */
+int grow_cap(int current, int needed, int *out);
+
+/* realloc *ptr to hold count floats; *ptr unchanged on failure. */
+int realloc_floats(float **ptr, size_t count);
+
+/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */
+int realloc_floats_2d(float **ptr, int rows, int cols);
+
+#endif /* LANGDET_ALLOC_H */
diff --git a/liblangdet/attn.c b/liblangdet/attn.c
new file mode 100644
index 0000000..ac0f4d1
--- /dev/null
+++ b/liblangdet/attn.c
@@ -0,0 +1,242 @@
+#include "kernels.h"
+#include "impl.h"
+#include "threadpool.h"
+
+#include <math.h>
+
+#ifdef USE_BLAS
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
+#endif
+
+static inline float dot_f32_fast(const float *a, const float *b, int n) {
+ return dot_f32_impl(a, b, n);
+}
+
+/* dst = dst * scale */
+static inline void vec_scale_inplace(float *dst, float scale, int n) {
+ vec_scale_inplace_impl(dst, scale, n);
+}
+
+/* dst += alpha * src */
+static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
+ vec_axpy_inplace_impl(dst, src, alpha, n);
+}
+
+/* dst = dst * correction + src */
+static inline void vec_scale_add(float *dst, const float *src, float correction, int n) {
+ vec_scale_add_impl(dst, src, correction, n);
+}
+
+#define PACKED_ATTN_QUERY_TILE 32
+#define PACKED_ATTN_BLAS_MIN_SEQ 128
+
+static void bidirectional_gqa_attention_packed_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+
+ for (int i = query_start; i < query_end; i++) {
+ const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
+ float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o_row[d] = 0.0f;
+
+ for (int j = start; j < end; j++) {
+ const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+
+ float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f) {
+ float inv_sum = 1.0f / sum_exp;
+ vec_scale_inplace(o_row, inv_sum, head_dim);
+ }
+ }
+}
+
+#ifdef USE_BLAS
+static void bidirectional_gqa_attention_packed_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+ int rows = query_end - query_start;
+ int keys = end - start;
+
+ const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
+ const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
+ const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
+ float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
+ kv_hidden, 0.0f, scores, keys);
+ softmax(scores, rows, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
+ v, kv_hidden, 0.0f, o, q_hidden);
+}
+#endif
+
+typedef struct {
+ float *out;
+ const float *Q;
+ const float *K;
+ const float *V;
+ const int *offsets;
+ int batch;
+ int n_heads, n_kv_heads;
+ int head_dim;
+ float scale;
+ float *scores;
+ int max_seq;
+} packed_gqa_attn_task_t;
+
+static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
+ packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+ int task_id = 0;
+#ifdef USE_BLAS
+ float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
+#endif
+
+ for (int h = 0; h < t->n_heads; h++) {
+ for (int b = 0; b < t->batch; b++) {
+ int start = t->offsets[b];
+ int end = t->offsets[b + 1];
+ for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
+ int q1 = q0 + PACKED_ATTN_QUERY_TILE;
+ if (q1 > end)
+ q1 = end;
+ if (task_id++ % n_threads != tid)
+ continue;
+#ifdef USE_BLAS
+ if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
+ bidirectional_gqa_attention_packed_blas_rows(
+ t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1, scores);
+ continue;
+ }
+#endif
+ bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end,
+ t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1);
+ }
+ }
+ }
+}
+
+size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
+#ifdef USE_BLAS
+ if (!offsets || batch <= 0)
+ return 0;
+
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ if (len > max_seq)
+ max_seq = len;
+ }
+ if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ)
+ return 0;
+ if ((size_t)max_seq >
+ SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads()))
+ return 0;
+ return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
+#else
+ (void)offsets;
+ (void)batch;
+ return 0;
+#endif
+}
+
+void bidirectional_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
+ long long qk_work = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ qk_work += (long long)len * len * n_heads;
+ if (len > max_seq)
+ max_seq = len;
+ }
+
+ float *scores = NULL;
+#ifdef USE_BLAS
+ size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required)
+ scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
+#endif
+ packed_gqa_attn_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .offsets = offsets,
+ .batch = batch,
+ .n_heads = n_heads,
+ .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores,
+ .max_seq = max_seq};
+ if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
+ parallel_for(packed_gqa_attn_worker, &task);
+ } else {
+ packed_gqa_attn_worker(0, 1, &task);
+ }
+}
diff --git a/liblangdet/config.c b/liblangdet/config.c
new file mode 100644
index 0000000..f22a9d7
--- /dev/null
+++ b/liblangdet/config.c
@@ -0,0 +1,67 @@
+#include "config.h"
+
+#include <yyjson.h>
+
+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+
+static int json_get_int(yyjson_val *obj, const char *key, int fallback) {
+ yyjson_val *v = yyjson_obj_get(obj, key);
+ if (!v)
+ return fallback;
+ if (!yyjson_is_int(v))
+ return -1;
+ int64_t n = yyjson_get_sint(v);
+ return n >= 0 && n <= INT_MAX ? (int)n : -1;
+}
+
+int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ fprintf(stderr, "langdet_config: model path too long\n");
+ return -1;
+ }
+
+ yyjson_read_err error;
+ yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error);
+ if (!doc) {
+ fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg);
+ return -1;
+ }
+ yyjson_val *root = yyjson_doc_get_root(doc);
+ yyjson_val *causal = yyjson_obj_get(root, "is_causal");
+ if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
+ !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") ||
+ (causal && !yyjson_is_false(causal))) {
+ fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n");
+ yyjson_doc_free(doc);
+ return -1;
+ }
+
+ cfg->hidden_size = json_get_int(root, "hidden_size", 0);
+ cfg->n_layers = json_get_int(root, "num_hidden_layers", 0);
+ cfg->n_heads = json_get_int(root, "num_attention_heads", 0);
+ cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
+ cfg->intermediate_size = json_get_int(root, "intermediate_size", 0);
+ cfg->vocab_size = json_get_int(root, "vocab_size", 0);
+ cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0);
+ cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1);
+ yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps");
+ cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f;
+ int pad_id = json_get_int(root, "pad_token_id", 1);
+ cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1;
+ yyjson_doc_free(doc);
+
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS ||
+ cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 ||
+ cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 ||
+ cfg->position_id_offset < 0 ||
+ cfg->max_position_embeddings <= cfg->position_id_offset ||
+ !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) {
+ fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path);
+ return -1;
+ }
+ return 0;
+}
diff --git a/liblangdet/config.h b/liblangdet/config.h
new file mode 100644
index 0000000..79522fc
--- /dev/null
+++ b/liblangdet/config.h
@@ -0,0 +1,8 @@
+#ifndef LANGDET_CONFIG_H
+#define LANGDET_CONFIG_H
+
+#include "langdet.h"
+
+int langdet_config_parse(langdet_config_t *cfg, const char *model_dir);
+
+#endif /* LANGDET_CONFIG_H */
diff --git a/liblangdet/forward.c b/liblangdet/forward.c
new file mode 100644
index 0000000..14b1327
--- /dev/null
+++ b/liblangdet/forward.c
@@ -0,0 +1,96 @@
+#include "model_internal.h"
+#include "kernels.h"
+
+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
+ for (int row = 0; row < rows; row++) {
+ float *dst = x + (size_t)row * dim;
+ for (int d = 0; d < dim; d++)
+ dst[d] += bias[d];
+ }
+}
+
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding)
+ return -1;
+
+ const langdet_config_t *cfg = &model->config;
+ const langdet_weights_t *w = &model->weights;
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
+ int head_dim = cfg->head_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->layer_norm_eps;
+
+ if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) {
+ fprintf(stderr,
+ "langdet: sequence length %d plus position offset %d exceeds max_position_embeddings "
+ "%d\n",
+ n_tokens, cfg->position_id_offset, cfg->max_position_embeddings);
+ return -1;
+ }
+ if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter ||
+ ensure_buffers(ws, cfg, n_tokens) != 0)
+ return -1;
+ int offsets[2] = {0, n_tokens};
+ ensure_attention_scores(ws, offsets, 1);
+
+ float *x = ws->x;
+ float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v;
+ float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up;
+
+ for (int i = 0; i < n_tokens; i++) {
+ int id = token_ids[i];
+ if (id < 0 || id >= cfg->vocab_size) {
+ fprintf(stderr, "langdet: invalid token id %d at position %d\n", id, i);
+ return -1;
+ }
+ float *row = x + (size_t)i * hidden;
+ memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float));
+ int pos = cfg->position_id_offset + i;
+ add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
+ add_inplace(row, w->token_type_embeddings, hidden);
+ }
+ layer_norm(x, x, w->langdet_ln_w, w->langdet_ln_b, n_tokens, hidden, eps);
+
+ float scale = 1.0f / sqrtf((float)head_dim);
+
+ for (int layer = 0; layer < cfg->n_layers; layer++) {
+ const langdet_layer_t *l = &w->layers[layer];
+
+ linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden);
+ linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden);
+ linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden);
+ add_bias_rows(q_buf, l->q_bias, n_tokens, hidden);
+ add_bias_rows(k_buf, l->k_bias, n_tokens, hidden);
+ add_bias_rows(v_buf, l->v_bias, n_tokens, hidden);
+
+ bidirectional_gqa_attention_packed_with_scratch(
+ attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale,
+ ws->attn_scores, ws->attn_scores_bytes);
+
+ linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden);
+ add_bias_rows(proj_out, l->o_bias, n_tokens, hidden);
+ add_inplace(x, proj_out, n_tokens * hidden);
+ layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps);
+
+ linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter);
+ add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter);
+ gelu_inplace(ffn_up, n_tokens * inter);
+ linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden);
+ add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden);
+ add_inplace(x, proj_out, n_tokens * hidden);
+ layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps);
+ }
+ /* The classifier consumes <s> after the encoder's final LayerNorm, without L2 normalization. */
+ memcpy(out_embedding, x, (size_t)hidden * sizeof(float));
+ return 0;
+}
diff --git a/liblangdet/gemm.c b/liblangdet/gemm.c
new file mode 100644
index 0000000..c750be1
--- /dev/null
+++ b/liblangdet/gemm.c
@@ -0,0 +1,49 @@
+#include "kernels.h"
+
+#ifdef USE_BLAS
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
+#endif
+
+void add_inplace(float *a, const float *b, int n) {
+ for (int i = 0; i < n; i++)
+ a[i] += b[i];
+}
+
+void linear(
+ float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) {
+#ifdef USE_BLAS
+ if (seq_len == 1) {
+ cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1);
+ } else {
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x,
+ in_dim, W, in_dim, 0.0f, y, out_dim);
+ }
+ if (b != NULL) {
+ for (int s = 0; s < seq_len; s++) {
+ for (int o = 0; o < out_dim; o++) {
+ y[s * out_dim + o] += b[o];
+ }
+ }
+ }
+#else
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + s * in_dim;
+ float *y_row = y + s * out_dim;
+ for (int o = 0; o < out_dim; o++) {
+ const float *w_row = W + o * in_dim;
+ float sum = (b != NULL) ? b[o] : 0.0f;
+ for (int i = 0; i < in_dim; i++)
+ sum += x_row[i] * w_row[i];
+ y_row[o] = sum;
+ }
+ }
+#endif
+}
+
+void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
+ linear(y, x, W, NULL, seq_len, in_dim, out_dim);
+}
diff --git a/liblangdet/generic.c b/liblangdet/generic.c
new file mode 100644
index 0000000..729c610
--- /dev/null
+++ b/liblangdet/generic.c
@@ -0,0 +1,27 @@
+/*
+ * Architecture-generic hot kernels / non-SIMD fallback.
+ */
+
+#include "impl.h"
+
+float dot_f32_generic(const float *a, const float *b, int n) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+}
+
+void vec_scale_inplace_generic(float *dst, float scale, int n) {
+ for (int i = 0; i < n; i++)
+ dst[i] *= scale;
+}
+
+void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
+ for (int i = 0; i < n; i++)
+ dst[i] += alpha * src[i];
+}
+
+void vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
+ for (int i = 0; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+}
diff --git a/liblangdet/impl.h b/liblangdet/impl.h
new file mode 100644
index 0000000..f216518
--- /dev/null
+++ b/liblangdet/impl.h
@@ -0,0 +1,254 @@
+/* Inline vector operations keep attention rows in registers across reductions. */
+
+#ifndef KERNELS_IMPL_H
+#define KERNELS_IMPL_H
+
+#include <stdint.h>
+
+float dot_f32_generic(const float *a, const float *b, int n);
+void vec_scale_inplace_generic(float *dst, float scale, int n);
+void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
+void vec_scale_add_generic(float *dst, const float *src, float correction, int n);
+
+#ifdef __ARM_NEON
+
+# include <arm_neon.h>
+
+static inline float dot_f32_impl(const float *a, const float *b, int n) {
+ int i = 0;
+ float32x4_t acc0 = vdupq_n_f32(0.0f);
+ float32x4_t acc1 = vdupq_n_f32(0.0f);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t a0 = vld1q_f32(a + i);
+ float32x4_t b0 = vld1q_f32(b + i);
+ float32x4_t a1 = vld1q_f32(a + i + 4);
+ float32x4_t b1 = vld1q_f32(b + i + 4);
+ acc0 = vfmaq_f32(acc0, a0, b0);
+ acc1 = vfmaq_f32(acc1, a1, b1);
+ }
+ float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+}
+
+static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
+ int i = 0;
+ float32x4_t s = vdupq_n_f32(scale);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
+ vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+}
+
+static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
+ int i = 0;
+ float32x4_t a = vdupq_n_f32(alpha);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
+ vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+}
+
+static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
+ int i = 0;
+ float32x4_t c = vdupq_n_f32(correction);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
+ vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+}
+
+#elif defined(__AVX2__) && defined(__FMA__)
+
+# include <immintrin.h>
+
+static inline float dot_f32_impl(const float *a, const float *b, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 acc0 = _mm512_setzero_ps();
+ __m512 acc1 = _mm512_setzero_ps();
+ __m512 acc2 = _mm512_setzero_ps();
+ __m512 acc3 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
+ acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
+ acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
+ acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
+ }
+ __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
+ for (; i + 16 <= n; i += 16) {
+ acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc);
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+# else
+ int i = 0;
+ __m256 acc0 = _mm256_setzero_ps();
+ __m256 acc1 = _mm256_setzero_ps();
+ __m256 acc2 = _mm256_setzero_ps();
+ __m256 acc3 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
+ acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
+ acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
+ acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
+ }
+ acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
+ for (; i + 8 <= n; i += 8) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
+ }
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
+ r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r);
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+# endif
+}
+
+static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 s = _mm512_set1_ps(scale);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
+ _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
+ _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+# else
+ int i = 0;
+ __m256 s = _mm256_set1_ps(scale);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
+ _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
+ _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
+ _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+# endif
+}
+
+static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 a = _mm512_set1_ps(alpha);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
+ _mm512_loadu_ps(dst + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
+ _mm512_loadu_ps(dst + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
+ _mm512_loadu_ps(dst + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+# else
+ int i = 0;
+ __m256 a = _mm256_set1_ps(alpha);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
+ _mm256_loadu_ps(dst + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
+ _mm256_loadu_ps(dst + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
+ _mm256_loadu_ps(dst + i + 24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+# endif
+}
+
+static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 c = _mm512_set1_ps(correction);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
+ _mm512_loadu_ps(src + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
+ _mm512_loadu_ps(src + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
+ _mm512_loadu_ps(src + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+# else
+ int i = 0;
+ __m256 c = _mm256_set1_ps(correction);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
+ _mm256_loadu_ps(src + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
+ _mm256_loadu_ps(src + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
+ _mm256_loadu_ps(src + i + 24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+# endif
+}
+
+#else /* no SIMD path: dispatch to the scalar references */
+
+# define dot_f32_impl dot_f32_generic
+# define vec_scale_inplace_impl vec_scale_inplace_generic
+# define vec_axpy_inplace_impl vec_axpy_inplace_generic
+# define vec_scale_add_impl vec_scale_add_generic
+
+#endif
+
+#endif /* KERNELS_IMPL_H */
diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h
new file mode 100644
index 0000000..1025602
--- /dev/null
+++ b/liblangdet/kernels.h
@@ -0,0 +1,42 @@
+#ifndef KERNELS_H
+#define KERNELS_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+void add_inplace(float *a, const float *b, int n);
+
+/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */
+void linear(
+ float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
+void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
+
+/* Mean and population variance are reduced over the hidden axis per row. */
+void layer_norm(float *out,
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps);
+
+void softmax(float *x, int rows, int cols);
+void gelu_inplace(float *x, int n);
+
+/* Attention is block-diagonal: each token attends only within its offsets span.
+ * Missing scratch uses the online-softmax path instead of tiled BLAS. */
+size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
+void bidirectional_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
+
+#endif /* KERNELS_H */
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h
new file mode 100644
index 0000000..9c24264
--- /dev/null
+++ b/liblangdet/langdet.h
@@ -0,0 +1,49 @@
+#ifndef LANGDET_H
+#define LANGDET_H
+
+#include <stddef.h>
+
+#define LANGDET_MAX_LAYERS 64
+
+typedef struct {
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int head_dim;
+ int intermediate_size;
+ int vocab_size;
+ int max_position_embeddings;
+ int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
+ int type_vocab_size;
+ float layer_norm_eps;
+} langdet_config_t;
+
+typedef struct langdet_model langdet_model_t;
+typedef struct langdet_workspace langdet_workspace_t;
+typedef struct langdet_tok langdet_tok_t;
+
+/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
+langdet_model_t *langdet_model_load(const char *model_dir);
+void langdet_model_free(langdet_model_t *model);
+const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+
+/* A workspace belongs to one model and may serve only one inference call at a time. */
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
+void langdet_workspace_free(langdet_workspace_t *ws);
+
+/* Write the first token's final hidden state to out_embedding[hidden_size].
+ * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
+void langdet_tok_free(langdet_tok_t *t);
+
+/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
+ * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+
+#endif /* LANGDET_H */
diff --git a/liblangdet/model.c b/liblangdet/model.c
new file mode 100644
index 0000000..146bbe7
--- /dev/null
+++ b/liblangdet/model.c
@@ -0,0 +1,159 @@
+#include "model_internal.h"
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim) {
+ if (t->dtype != DTYPE_F32) {
+ fprintf(stderr, "langdet: expected F32 tensor: %s\n", name);
+ return 0;
+ }
+ if (t->ndim != ndim) {
+ fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
+ return 0;
+ }
+
+ size_t numel = 1;
+ for (int i = 0; i < ndim; i++) {
+ if (t->shape[i] != shape[i]) {
+ fprintf(stderr,
+ "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n",
+ name, i, (long long)t->shape[i], (long long)shape[i]);
+ return 0;
+ }
+ if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
+ fprintf(stderr, "langdet: shape too large or empty for %s\n", name);
+ return 0;
+ }
+ numel *= (size_t)shape[i];
+ }
+ size_t bytes;
+ if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) {
+ fprintf(stderr, "langdet: invalid data size for %s\n", name);
+ return 0;
+ }
+ size_t data_start = 8 + sf->header_size;
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
+ t->data_size > sf->file_size - data_start - t->data_offset) {
+ fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name);
+ return 0;
+ }
+ return 1;
+}
+
+static const float *load_weight(multi_safetensors_t *ms, const char *name,
+ const int64_t *shape, int ndim) {
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "langdet: tensor not found: %s\n", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(sf, t, name, shape, ndim))
+ return NULL;
+ return safetensors_data(sf, t);
+}
+
+static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+ const langdet_config_t *cfg = &model->config;
+ langdet_weights_t *w = &model->weights;
+ char name[256];
+ int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
+
+ w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
+ (int64_t[]){cfg->vocab_size, hidden}, 2);
+ w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
+ (int64_t[]){cfg->max_position_embeddings, hidden}, 2);
+ w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
+ (int64_t[]){cfg->type_vocab_size, hidden}, 2);
+ w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
+ !w->langdet_ln_w || !w->langdet_ln_b)
+ return -1;
+
+ w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t));
+ if (!w->layers)
+ return -1;
+
+ for (int i = 0; i < cfg->n_layers; i++) {
+ langdet_layer_t *l = &w->layers[i];
+
+#define BERT_W(field, suffix, d0, d1) \
+ do { \
+ const int64_t e[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(ms, name, e, 2); \
+ if (!l->field) \
+ return -1; \
+ } while (0)
+#define BERT_V(field, suffix, d0) \
+ do { \
+ const int64_t e[1] = {(d0)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(ms, name, e, 1); \
+ if (!l->field) \
+ return -1; \
+ } while (0)
+
+ BERT_W(wq, "attention.self.query.weight", hidden, hidden);
+ BERT_V(q_bias, "attention.self.query.bias", hidden);
+ BERT_W(wk, "attention.self.key.weight", hidden, hidden);
+ BERT_V(k_bias, "attention.self.key.bias", hidden);
+ BERT_W(wv, "attention.self.value.weight", hidden, hidden);
+ BERT_V(v_bias, "attention.self.value.bias", hidden);
+ BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
+ BERT_V(o_bias, "attention.output.dense.bias", hidden);
+ BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
+ BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
+ BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
+ BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
+ BERT_W(down_proj, "output.dense.weight", hidden, inter);
+ BERT_V(ffn_out_bias, "output.dense.bias", hidden);
+ BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
+ BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
+
+#undef BERT_W
+#undef BERT_V
+ }
+ return 0;
+}
+
+langdet_model_t *langdet_model_load(const char *model_dir) {
+ if (!model_dir)
+ return NULL;
+ langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t));
+ if (!model)
+ return NULL;
+ if (langdet_config_parse(&model->config, model_dir) != 0)
+ goto fail;
+ model->safetensors = multi_safetensors_open(model_dir);
+ if (!model->safetensors) {
+ fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
+ goto fail;
+ }
+ if (load_bert_weights(model, model->safetensors) != 0)
+ goto fail;
+ return model;
+
+fail:
+ langdet_model_free(model);
+ return NULL;
+}
+
+void langdet_model_free(langdet_model_t *model) {
+ if (!model)
+ return;
+ multi_safetensors_close(model->safetensors);
+ free(model->weights.layers);
+ free(model);
+}
+
+const langdet_config_t *langdet_model_config(const langdet_model_t *model) {
+ return model ? &model->config : NULL;
+}
diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c
new file mode 100644
index 0000000..41f9237
--- /dev/null
+++ b/liblangdet/model_blas.c
@@ -0,0 +1,68 @@
+#include "model_internal.h"
+#include "kernels.h"
+
+#include <stdlib.h>
+
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model) {
+ if (!model)
+ return NULL;
+ langdet_workspace_t *ws = (langdet_workspace_t *)calloc(1, sizeof(langdet_workspace_t));
+ if (!ws)
+ return NULL;
+ ws->model = model;
+ return ws;
+}
+
+void langdet_workspace_free(langdet_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->x);
+ free(ws->q);
+ free(ws->k);
+ free(ws->v);
+ free(ws->attn_out);
+ free(ws->proj_out);
+ free(ws->ffn_up);
+ free(ws->attn_scores);
+ free(ws);
+}
+
+int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) {
+ if (seq <= ws->buf_seq_cap)
+ return 0;
+
+ int cap;
+ if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
+ return -1;
+
+#define R(ptr, n) \
+ do { \
+ if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
+ return -1; \
+ } while (0)
+
+ R(ws->x, c->hidden_size);
+ R(ws->q, c->hidden_size);
+ R(ws->k, c->hidden_size);
+ R(ws->v, c->hidden_size);
+ R(ws->attn_out, c->hidden_size);
+ R(ws->proj_out, c->hidden_size);
+ R(ws->ffn_up, c->intermediate_size);
+
+#undef R
+
+ ws->buf_seq_cap = cap;
+ return 0;
+}
+
+void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) {
+ size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes)
+ return;
+
+ void *p = realloc(ws->attn_scores, bytes);
+ if (!p)
+ return;
+ ws->attn_scores = (float *)p;
+ ws->attn_scores_bytes = bytes;
+}
diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h
new file mode 100644
index 0000000..73a7851
--- /dev/null
+++ b/liblangdet/model_internal.h
@@ -0,0 +1,17 @@
+#ifndef LANGDET_MODEL_INTERNAL_H
+#define LANGDET_MODEL_INTERNAL_H
+
+#include "model_types.h"
+#include "alloc.h"
+
+/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim);
+
+int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq);
+void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch);
+
+#endif /* LANGDET_MODEL_INTERNAL_H */
diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h
new file mode 100644
index 0000000..2170909
--- /dev/null
+++ b/liblangdet/model_types.h
@@ -0,0 +1,56 @@
+#ifndef LANGDET_MODEL_TYPES_H
+#define LANGDET_MODEL_TYPES_H
+
+#include "langdet.h"
+#include "safetensors.h"
+
+typedef struct {
+ const float *wq;
+ const float *wk;
+ const float *wv;
+ const float *wo;
+ const float *q_bias;
+ const float *k_bias;
+ const float *v_bias;
+ const float *o_bias;
+ const float *input_norm;
+ const float *attn_ln_bias;
+ const float *up_proj;
+ const float *ffn_inter_bias;
+ const float *down_proj;
+ const float *ffn_out_bias;
+ const float *post_attn_norm;
+ const float *ffn_ln_bias;
+} langdet_layer_t;
+
+typedef struct {
+ const float *embed_tokens;
+ langdet_layer_t *layers;
+ const float *position_embeddings;
+ const float *token_type_embeddings;
+ const float *langdet_ln_w;
+ const float *langdet_ln_b;
+} langdet_weights_t;
+
+struct langdet_model {
+ langdet_config_t config;
+ langdet_weights_t weights;
+ /* All weight pointers borrow this mapping until model teardown. */
+ multi_safetensors_t *safetensors;
+};
+
+struct langdet_workspace {
+ const langdet_model_t *model;
+ int buf_seq_cap;
+ float *x;
+ float *q;
+ float *k;
+ float *v;
+ float *attn_out;
+ float *proj_out;
+ float *ffn_up;
+ float *attn_scores;
+ size_t attn_scores_bytes;
+};
+
+#endif /* LANGDET_MODEL_TYPES_H */
diff --git a/liblangdet/norm.c b/liblangdet/norm.c
new file mode 100644
index 0000000..59e8bdb
--- /dev/null
+++ b/liblangdet/norm.c
@@ -0,0 +1,33 @@
+#include "kernels.h"
+
+#include <math.h>
+
+/* Safe in place: each row's mean and variance are reduced before it is rewritten. */
+void layer_norm(float *out,
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps) {
+ if (seq_len <= 0 || hidden <= 0)
+ return;
+ float inv_h = 1.0f / (float)hidden;
+ for (int s = 0; s < seq_len; s++) {
+ const float *row = x + (size_t)s * hidden;
+ float *dst = out + (size_t)s * hidden;
+ float mean = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ mean += row[d];
+ mean *= inv_h;
+ float var = 0.0f;
+ for (int d = 0; d < hidden; d++) {
+ float diff = row[d] - mean;
+ var += diff * diff;
+ }
+ var *= inv_h;
+ float inv_std = 1.0f / sqrtf(var + eps);
+ for (int d = 0; d < hidden; d++)
+ dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
+ }
+}
diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c
new file mode 100644
index 0000000..cfba76e
--- /dev/null
+++ b/liblangdet/safetensors.c
@@ -0,0 +1,483 @@
+#include "safetensors.h"
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+/* Minimal JSON parser for safetensors header */
+/* TODO: consider replacing with yyjson */
+
+static void skip_whitespace(const char **p) {
+ while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
+ (*p)++;
+}
+
+static int parse_string(const char **p, char *out, size_t max_len) {
+ skip_whitespace(p);
+ if (**p != '"')
+ return -1;
+ (*p)++;
+ size_t i = 0;
+ while (**p && **p != '"' && i < max_len - 1) {
+ if (**p == '\\') {
+ (*p)++;
+ if (**p == 'n')
+ out[i++] = '\n';
+ else if (**p == 't')
+ out[i++] = '\t';
+ else if (**p == '"')
+ out[i++] = '"';
+ else if (**p == '\\')
+ out[i++] = '\\';
+ else
+ out[i++] = **p;
+ } else {
+ out[i++] = **p;
+ }
+ (*p)++;
+ }
+ out[i] = '\0';
+ if (**p != '"')
+ return -1;
+ (*p)++;
+ return 0;
+}
+
+static int parse_int(const char **p, int64_t *out) {
+ skip_whitespace(p);
+ int64_t val = 0;
+ int neg = 0;
+ if (**p == '-') {
+ neg = 1;
+ (*p)++;
+ }
+ if (**p < '0' || **p > '9')
+ return -1;
+ while (**p >= '0' && **p <= '9') {
+ val = val * 10 + (**p - '0');
+ (*p)++;
+ }
+ *out = neg ? -val : val;
+ return 0;
+}
+
+static safetensor_dtype_t parse_dtype(const char *s) {
+ if (strcmp(s, "F32") == 0)
+ return DTYPE_F32;
+ if (strcmp(s, "F16") == 0)
+ return DTYPE_F16;
+ if (strcmp(s, "BF16") == 0)
+ return DTYPE_BF16;
+ if (strcmp(s, "I32") == 0)
+ return DTYPE_I32;
+ if (strcmp(s, "I64") == 0)
+ return DTYPE_I64;
+ if (strcmp(s, "BOOL") == 0)
+ return DTYPE_BOOL;
+ return DTYPE_UNKNOWN;
+}
+
+static int parse_tensor_entry(const char **p, safetensor_t *t) {
+ skip_whitespace(p);
+ if (**p != '{')
+ return -1;
+ (*p)++;
+
+ t->dtype = DTYPE_UNKNOWN;
+ t->ndim = 0;
+ t->data_offset = 0;
+ t->data_size = 0;
+
+ while (**p && **p != '}') {
+ skip_whitespace(p);
+ if (**p == ',') {
+ (*p)++;
+ continue;
+ }
+
+ char key[64];
+ if (parse_string(p, key, sizeof(key)) != 0)
+ return -1;
+ skip_whitespace(p);
+ if (**p != ':')
+ return -1;
+ (*p)++;
+ skip_whitespace(p);
+
+ if (strcmp(key, "dtype") == 0) {
+ char dtype_str[32];
+ if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
+ return -1;
+ t->dtype = parse_dtype(dtype_str);
+ } else if (strcmp(key, "shape") == 0) {
+ if (**p != '[')
+ return -1;
+ (*p)++;
+ t->ndim = 0;
+ while (**p && **p != ']') {
+ skip_whitespace(p);
+ if (**p == ',') {
+ (*p)++;
+ continue;
+ }
+ if (t->ndim >= 8)
+ return -1;
+ if (parse_int(p, &t->shape[t->ndim]) != 0)
+ return -1;
+ t->ndim++;
+ }
+ if (**p != ']')
+ return -1;
+ (*p)++;
+ } else if (strcmp(key, "data_offsets") == 0) {
+ if (**p != '[')
+ return -1;
+ (*p)++;
+ skip_whitespace(p);
+ int64_t start_i;
+ if (parse_int(p, &start_i) != 0)
+ return -1;
+ skip_whitespace(p);
+ if (**p != ',')
+ return -1;
+ (*p)++;
+ skip_whitespace(p);
+ int64_t end_i;
+ if (parse_int(p, &end_i) != 0)
+ return -1;
+ if (start_i < 0 || end_i < start_i)
+ return -1;
+ t->data_offset = (size_t)start_i;
+ t->data_size = (size_t)(end_i - start_i);
+ skip_whitespace(p);
+ if (**p != ']')
+ return -1;
+ (*p)++;
+ } else {
+ /* Skip unknown value */
+ if (**p == '"') {
+ (*p)++;
+ while (**p && **p != '"') {
+ if (**p == '\\')
+ (*p)++;
+ if (**p)
+ (*p)++;
+ }
+ if (**p == '"')
+ (*p)++;
+ } else if (**p == '[') {
+ int depth = 1;
+ (*p)++;
+ while (**p && depth > 0) {
+ if (**p == '[')
+ depth++;
+ else if (**p == ']')
+ depth--;
+ (*p)++;
+ }
+ } else if (**p == '{') {
+ int depth = 1;
+ (*p)++;
+ while (**p && depth > 0) {
+ if (**p == '{')
+ depth++;
+ else if (**p == '}')
+ depth--;
+ (*p)++;
+ }
+ } else {
+ while (**p && **p != ',' && **p != '}')
+ (*p)++;
+ }
+ }
+ }
+ if (**p != '}')
+ return -1;
+ (*p)++;
+ return 0;
+}
+
+static int parse_header(safetensors_file_t *sf) {
+ const char *p = sf->header_json;
+ skip_whitespace(&p);
+ if (*p != '{')
+ return -1;
+ p++;
+
+ sf->num_tensors = 0;
+
+ while (*p && *p != '}') {
+ skip_whitespace(&p);
+ if (*p == ',') {
+ p++;
+ continue;
+ }
+ if (*p == '}')
+ break;
+
+ char name[256];
+ if (parse_string(&p, name, sizeof(name)) != 0)
+ return -1;
+ skip_whitespace(&p);
+ if (*p != ':')
+ return -1;
+ p++;
+
+ if (strcmp(name, "__metadata__") == 0) {
+ skip_whitespace(&p);
+ if (*p == '{') {
+ int depth = 1;
+ p++;
+ while (*p && depth > 0) {
+ if (*p == '{')
+ depth++;
+ else if (*p == '}')
+ depth--;
+ p++;
+ }
+ }
+ continue;
+ }
+
+ if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
+ fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
+ sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
+ return -1;
+ }
+
+ safetensor_t *t = &sf->tensors[sf->num_tensors];
+ snprintf(t->name, sizeof(t->name), "%s", name);
+ if (parse_tensor_entry(&p, t) != 0)
+ return -1;
+ sf->num_tensors++;
+ }
+ if (*p != '}')
+ return -1;
+ return 0;
+}
+
+/* Single file operations */
+
+safetensors_file_t *safetensors_open(const char *path) {
+ int fd = open(path, O_RDONLY);
+ if (fd < 0)
+ return NULL;
+
+ struct stat st;
+ if (fstat(fd, &st) < 0) {
+ close(fd);
+ return NULL;
+ }
+
+ size_t file_size = (size_t)st.st_size;
+ if (file_size < 8) {
+ close(fd);
+ return NULL;
+ }
+
+ void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (data == MAP_FAILED)
+ return NULL;
+
+ uint64_t header_size = 0;
+ memcpy(&header_size, data, 8);
+ if (header_size > file_size - 8) {
+ munmap(data, file_size);
+ return NULL;
+ }
+
+ safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
+ if (!sf) {
+ munmap(data, file_size);
+ return NULL;
+ }
+
+ sf->path = strdup(path);
+ sf->data = data;
+ sf->file_size = file_size;
+ sf->header_size = (size_t)header_size;
+
+ sf->header_json = malloc(header_size + 1);
+ if (!sf->header_json) {
+ safetensors_close(sf);
+ return NULL;
+ }
+ memcpy(sf->header_json, (char *)data + 8, header_size);
+ sf->header_json[header_size] = '\0';
+
+ if (parse_header(sf) != 0) {
+ safetensors_close(sf);
+ return NULL;
+ }
+
+ return sf;
+}
+
+void safetensors_close(safetensors_file_t *sf) {
+ if (!sf)
+ return;
+ if (sf->data)
+ munmap(sf->data, sf->file_size);
+ free(sf->path);
+ free(sf->header_json);
+ free(sf);
+}
+
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
+ return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
+}
+
+/* Multi-shard operations */
+
+static int cmp_shard_names(const void *a, const void *b) {
+ const char *sa = (const char *)a;
+ const char *sb = (const char *)b;
+ return strcmp(sa, sb);
+}
+
+static int parse_shard_name(const char *name, int *idx, int *total) {
+ int i = 0, n = 0, consumed = 0;
+ if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
+ return -1;
+ if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
+ return -1;
+ *idx = i;
+ *total = n;
+ return 0;
+}
+
+multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
+ multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
+ if (!ms)
+ return NULL;
+
+ char path[4096];
+
+ /* Try single file first */
+ snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
+ safetensors_file_t *sf = safetensors_open(path);
+ if (sf) {
+ ms->shards[0] = sf;
+ ms->num_shards = 1;
+ return ms;
+ }
+
+ /* Scan directory for shard files */
+ DIR *dir = opendir(model_dir);
+ if (!dir) {
+ free(ms);
+ return NULL;
+ }
+
+ struct dirent *entry;
+ char shard_names[SAFETENSORS_MAX_SHARDS][256];
+ int seen[SAFETENSORS_MAX_SHARDS] = {0};
+ int n_shards = 0;
+ int expected_total = 0;
+
+ while ((entry = readdir(dir)) != NULL) {
+ if (strncmp(entry->d_name, "model-", 6) == 0 &&
+ strstr(entry->d_name, ".safetensors") != NULL) {
+ int shard_idx = 0, shard_total = 0;
+ if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
+ fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (shard_total > SAFETENSORS_MAX_SHARDS) {
+ fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
+ SAFETENSORS_MAX_SHARDS);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (expected_total == 0) {
+ expected_total = shard_total;
+ } else if (expected_total != shard_total) {
+ fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
+ entry->d_name);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (seen[shard_idx - 1]) {
+ fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ seen[shard_idx - 1] = 1;
+
+ snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
+ n_shards++;
+ }
+ }
+ closedir(dir);
+
+ if (n_shards == 0) {
+ fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
+ free(ms);
+ return NULL;
+ }
+ if (expected_total != n_shards) {
+ fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
+ expected_total, n_shards, model_dir);
+ free(ms);
+ return NULL;
+ }
+ for (int i = 0; i < expected_total; i++) {
+ if (!seen[i]) {
+ fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
+ expected_total, model_dir);
+ free(ms);
+ return NULL;
+ }
+ }
+
+ /* Sort shard names to ensure consistent ordering */
+ qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
+
+ /* Open each shard */
+ for (int i = 0; i < n_shards; i++) {
+ snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
+ ms->shards[i] = safetensors_open(path);
+ if (!ms->shards[i]) {
+ fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
+ multi_safetensors_close(ms);
+ return NULL;
+ }
+ }
+ ms->num_shards = n_shards;
+ return ms;
+}
+
+void multi_safetensors_close(multi_safetensors_t *ms) {
+ if (!ms)
+ return;
+ for (int i = 0; i < ms->num_shards; i++) {
+ safetensors_close(ms->shards[i]);
+ }
+ free(ms);
+}
+
+const safetensor_t *
+multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) {
+ for (int s = 0; s < ms->num_shards; s++) {
+ safetensors_file_t *sf = ms->shards[s];
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0) {
+ if (out_sf)
+ *out_sf = sf;
+ return &sf->tensors[i];
+ }
+ }
+ }
+ return NULL;
+}
diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h
new file mode 100644
index 0000000..dca68a7
--- /dev/null
+++ b/liblangdet/safetensors.h
@@ -0,0 +1,64 @@
+/*
+ * Safetensors file format reader with multi-shard support.
+ */
+
+#ifndef SAFETENSORS_H
+#define SAFETENSORS_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define SAFETENSORS_MAX_TENSORS 1024
+#define SAFETENSORS_MAX_SHARDS 8
+
+typedef enum {
+ DTYPE_F32 = 0,
+ DTYPE_F16 = 1,
+ DTYPE_BF16 = 2,
+ DTYPE_I32 = 3,
+ DTYPE_I64 = 4,
+ DTYPE_BOOL = 5,
+ DTYPE_UNKNOWN = -1
+} safetensor_dtype_t;
+
+typedef struct {
+ char name[256];
+ safetensor_dtype_t dtype;
+ int ndim;
+ int64_t shape[8];
+ size_t data_offset;
+ size_t data_size;
+} safetensor_t;
+
+typedef struct {
+ char *path;
+ void *data;
+ size_t file_size;
+ size_t header_size;
+ char *header_json;
+ int num_tensors;
+ safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
+} safetensors_file_t;
+
+/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
+typedef struct {
+ safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
+ int num_shards;
+} multi_safetensors_t;
+
+/* Open a single safetensors file (memory-mapped) */
+safetensors_file_t *safetensors_open(const char *path);
+void safetensors_close(safetensors_file_t *sf);
+
+/* Open model from directory (auto-detects single file or multi-shard) */
+multi_safetensors_t *multi_safetensors_open(const char *model_dir);
+void multi_safetensors_close(multi_safetensors_t *ms);
+
+/* Find a tensor by name across all shards */
+const safetensor_t *
+multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf);
+
+/* Get raw pointer to tensor data (within mmap'd region) */
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
+
+#endif /* SAFETENSORS_H */
diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c
new file mode 100644
index 0000000..6c30ba1
--- /dev/null
+++ b/liblangdet/sentencepiece.c
@@ -0,0 +1,905 @@
+/*
+ * SentencePiece Unigram tokenizer.
+ *
+ * The implementation reads the small subset of ModelProto needed at runtime:
+ * pieces, scores, piece types, trainer model_type, and normalizer options. The
+ * normalizer supports SentencePiece's precompiled charsmap format used by
+ * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program
+ * over UTF-8 byte offsets.
+ */
+
+#include "sentencepiece.h"
+
+#include <errno.h>
+#include <float.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+ SP_MODEL_UNIGRAM = 1,
+ SP_PIECE_NORMAL = 1,
+ SP_PIECE_UNKNOWN = 2,
+ SP_PIECE_CONTROL = 3,
+ SP_PIECE_USER_DEFINED = 4,
+ SP_PIECE_UNUSED = 5,
+ SP_PIECE_BYTE = 6,
+};
+
+typedef struct {
+ const char *key;
+ uint16_t len;
+ int val;
+} sp_entry_t;
+
+typedef struct {
+ int id;
+ int prev;
+ double score;
+} sp_best_t;
+
+struct tok_spm_workspace {
+ char *norm;
+ size_t norm_len;
+ size_t norm_cap;
+ sp_best_t *best;
+ int best_cap;
+ int *ids;
+ int ids_cap;
+};
+
+static uint64_t fnv1a_bytes(const char *s, size_t n) {
+ uint64_t h = 1469598103934665603ULL;
+ for (size_t i = 0; i < n; i++) {
+ h ^= (unsigned char)s[i];
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+static int next_pow2(int x) {
+ int p = 1;
+ while (p < x && p <= (1 << 29))
+ p <<= 1;
+ return p;
+}
+
+static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) {
+ if (len <= 0 || len > UINT16_MAX)
+ return -1;
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
+ while (map[pos].key) {
+ if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) {
+ map[pos].val = val;
+ return 0;
+ }
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ map[pos].key = key;
+ map[pos].len = (uint16_t)len;
+ map[pos].val = val;
+ return 0;
+}
+
+static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
+ if (!map || len <= 0)
+ return -1;
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
+ while (map[pos].key) {
+ if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0)
+ return map[pos].val;
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ return -1;
+}
+
+static int is_tokenizable_type(int type) {
+ return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
+ type == SP_PIECE_BYTE;
+}
+
+static int reserve_chars(char **buf, size_t *cap, size_t need) {
+ if (need <= *cap)
+ return 0;
+ size_t nc = *cap ? *cap : 128;
+ while (nc < need) {
+ if (nc > SIZE_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ char *p = (char *)realloc(*buf, nc);
+ if (!p)
+ return -1;
+ *buf = p;
+ *cap = nc;
+ return 0;
+}
+
+static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) {
+ if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0)
+ return -1;
+ memcpy(*buf + *len, s, n);
+ *len += n;
+ (*buf)[*len] = '\0';
+ return 0;
+}
+
+static int reserve_best(tok_spm_workspace_t *ws, int need) {
+ if (need <= ws->best_cap)
+ return 0;
+ int nc = ws->best_cap ? ws->best_cap : 128;
+ while (nc < need) {
+ if (nc > INT_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p));
+ if (!p)
+ return -1;
+ ws->best = p;
+ ws->best_cap = nc;
+ return 0;
+}
+
+static int reserve_ids(tok_spm_workspace_t *ws, int need) {
+ if (need <= ws->ids_cap)
+ return 0;
+ int nc = ws->ids_cap ? ws->ids_cap : 64;
+ while (nc < need) {
+ if (nc > INT_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p));
+ if (!p)
+ return -1;
+ ws->ids = p;
+ ws->ids_cap = nc;
+ return 0;
+}
+
+static int utf8_len(unsigned char c) {
+ if (c < 0x80)
+ return 1;
+ if ((c & 0xE0) == 0xC0)
+ return 2;
+ if ((c & 0xF0) == 0xE0)
+ return 3;
+ if ((c & 0xF8) == 0xF0)
+ return 4;
+ return 1;
+}
+
+static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) {
+ if (off >= n)
+ return 0;
+ const unsigned char *p = (const unsigned char *)s + off;
+ int len = utf8_len(p[0]);
+ if (off + (size_t)len > n) {
+ *len_out = 1;
+ return 0;
+ }
+ if (len > 1) {
+ for (int i = 1; i < len; i++) {
+ if ((p[i] & 0xC0) != 0x80) {
+ *len_out = 1;
+ return 0;
+ }
+ }
+ }
+ *len_out = (size_t)len;
+ return 1;
+}
+
+static uint32_t read_le32(const unsigned char *p) {
+ return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
+}
+
+static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) {
+ uint64_t v = 0;
+ int shift = 0;
+ while (*p < end && shift <= 63) {
+ unsigned char c = *(*p)++;
+ v |= (uint64_t)(c & 0x7f) << shift;
+ if (!(c & 0x80)) {
+ *out = v;
+ return 0;
+ }
+ shift += 7;
+ }
+ return -1;
+}
+
+static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) {
+ uint64_t key = 0;
+ if (pb_read_varint(p, end, &key) != 0)
+ return -1;
+ *field = (int)(key >> 3);
+ *wire = (int)(key & 7);
+ return *field > 0 ? 0 : -1;
+}
+
+static int pb_read_len(const unsigned char **p,
+ const unsigned char *end,
+ const unsigned char **data,
+ size_t *len) {
+ uint64_t n = 0;
+ if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p))
+ return -1;
+ *data = *p;
+ *len = (size_t)n;
+ *p += n;
+ return 0;
+}
+
+static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) {
+ uint64_t v = 0;
+ const unsigned char *data = NULL;
+ size_t len = 0;
+ switch (wire) {
+ case 0:
+ return pb_read_varint(p, end, &v);
+ case 1:
+ if ((size_t)(end - *p) < 8)
+ return -1;
+ *p += 8;
+ return 0;
+ case 2:
+ return pb_read_len(p, end, &data, &len);
+ case 5:
+ if ((size_t)(end - *p) < 4)
+ return -1;
+ *p += 4;
+ return 0;
+ default:
+ return -1;
+ }
+}
+
+static char *memdup0(const unsigned char *data, size_t len) {
+ char *s = (char *)malloc(len + 1);
+ if (!s)
+ return NULL;
+ memcpy(s, data, len);
+ s[len] = '\0';
+ return s;
+}
+
+static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ char *piece = NULL;
+ float score = 0.0f;
+ int type = SP_PIECE_NORMAL;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ goto fail;
+ if (field == 1 && wire == 2) {
+ const unsigned char *s = NULL;
+ size_t n = 0;
+ if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX)
+ goto fail;
+ free(piece);
+ piece = memdup0(s, n);
+ if (!piece)
+ goto fail;
+ tok->piece_lens[idx] = (uint16_t)n;
+ } else if (field == 2 && wire == 5) {
+ if ((size_t)(end - p) < 4)
+ goto fail;
+ uint32_t bits = read_le32(p);
+ memcpy(&score, &bits, sizeof(score));
+ p += 4;
+ } else if (field == 3 && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX)
+ goto fail;
+ type = (int)v;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ goto fail;
+ }
+ }
+ if (!piece)
+ goto fail;
+ tok->id_to_piece[idx] = piece;
+ tok->scores[idx] = score;
+ tok->types[idx] = (uint8_t)type;
+ if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
+ tok->max_piece_len = tok->piece_lens[idx];
+ return 0;
+fail:
+ free(piece);
+ return -1;
+}
+
+static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ int model_type = SP_MODEL_UNIGRAM;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (field == 3 && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX)
+ return -1;
+ model_type = (int)v;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ if (model_type != SP_MODEL_UNIGRAM) {
+ fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n",
+ model_type);
+ return -1;
+ }
+ (void)tok;
+ return 0;
+}
+
+static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if ((field == 3 || field == 4 || field == 5) && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0)
+ return -1;
+ if (field == 3)
+ tok->add_dummy_prefix = v != 0;
+ else if (field == 4)
+ tok->remove_extra_whitespaces = v != 0;
+ else
+ tok->escape_whitespaces = v != 0;
+ } else if (field == 2 && wire == 2) {
+ const unsigned char *blob = NULL;
+ size_t n = 0;
+ if (pb_read_len(&p, end, &blob, &n) != 0)
+ return -1;
+ if (n >= 4) {
+ uint32_t xcda_bytes = read_le32(blob);
+ if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) {
+ tok->xcda_count = xcda_bytes / 4;
+ tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda));
+ if (!tok->xcda)
+ return -1;
+ for (size_t i = 0; i < tok->xcda_count; i++)
+ tok->xcda[i] = read_le32(blob + 4 + i * 4);
+ tok->prefix_replacements_len = n - 4 - xcda_bytes;
+ tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1);
+ if (!tok->prefix_replacements)
+ return -1;
+ memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes,
+ tok->prefix_replacements_len);
+ tok->prefix_replacements[tok->prefix_replacements_len] = '\0';
+ }
+ }
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int count_pieces(const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ int n = 0;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (field == 1 && wire == 2) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0)
+ return -1;
+ n++;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ return n;
+}
+
+static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
+ int n_pieces = count_pieces(data, len);
+ if (n_pieces <= 0)
+ return -1;
+ tok->piece_count = n_pieces;
+ tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
+ tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
+ tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
+ tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
+ if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
+ return -1;
+
+ const unsigned char *p = data, *end = data + len;
+ int piece_idx = 0;
+ int saw_trainer = 0;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (wire == 2 && field == 1) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces ||
+ parse_piece(tok, piece_idx++, msg, msg_len) != 0)
+ return -1;
+ } else if (wire == 2 && field == 2) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
+ return -1;
+ saw_trainer = 1;
+ } else if (wire == 2 && field == 3) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0)
+ return -1;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ if (piece_idx != n_pieces || !saw_trainer)
+ return -1;
+ return 0;
+}
+
+static int raw_piece_id(const tok_spm_t *tok, const char *piece) {
+ if (!tok || !piece)
+ return -1;
+ return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece));
+}
+
+static int raw_to_model_id(const tok_spm_t *tok, int raw_id) {
+ if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
+ return -1;
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
+ if (raw_id == 1)
+ return 0; /* <s> */
+ if (raw_id == 2)
+ return 2; /* </s> */
+ if (raw_id == 0)
+ return 3; /* <unk> */
+ return raw_id + 1;
+ }
+ return raw_id;
+}
+
+static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return 0;
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return 0;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return 0;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return 0;
+ }
+ size_t got = fread(buf, 1, (size_t)sz, f);
+ fclose(f);
+ buf[got] = '\0';
+ int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
+ strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
+ strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
+ free(buf);
+ return yes;
+}
+
+static int detect_xlm_roberta(const char *model_dir) {
+ return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
+ file_contains_xlm_roberta(model_dir, "config.json");
+}
+
+static int find_model_file(const char *model_dir, char path[1024]) {
+ static const char *names[] = {
+ "sentencepiece.bpe.model",
+ "tokenizer.model",
+ "spiece.model",
+ };
+ for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
+ int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
+ if (n < 0 || n >= 1024)
+ return -1;
+ FILE *f = fopen(path, "rb");
+ if (f) {
+ fclose(f);
+ return 0;
+ }
+ if (errno != ENOENT)
+ return -1;
+ }
+ return -1;
+}
+
+static int load_file(const char *path, unsigned char **out, size_t *out_len) {
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return -1;
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ unsigned char *buf = (unsigned char *)malloc((size_t)sz);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ fclose(f);
+ free(buf);
+ return -1;
+ }
+ fclose(f);
+ *out = buf;
+ *out_len = (size_t)sz;
+ return 0;
+}
+
+static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) {
+ uint32_t v = tok->xcda[idx];
+ return (v >> 10) << ((v & (1u << 9)) >> 6);
+}
+
+static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) {
+ return tok->xcda[idx] & ((1u << 31) | 0xffu);
+}
+
+static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; }
+
+static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) {
+ return tok->xcda[idx] & ((1u << 31) - 1u);
+}
+
+typedef struct {
+ const char *data;
+ size_t len;
+ size_t consumed;
+} norm_prefix_t;
+
+static norm_prefix_t
+normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) {
+ static const char repl[] = "\xef\xbf\xbd";
+ if (off >= input_len)
+ return (norm_prefix_t){input + off, 0, 0};
+
+ size_t best_len = 0;
+ size_t best_repl = 0;
+ if (tok->xcda && tok->xcda_count > 0) {
+ uint32_t node = xcda_base(tok, 0);
+ for (size_t p = off; p < input_len; p++) {
+ unsigned char c = (unsigned char)input[p];
+ if (c == 0)
+ break;
+ node ^= c;
+ if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c)
+ break;
+ int leaf = xcda_leaf(tok, node);
+ node ^= xcda_base(tok, node);
+ if (leaf) {
+ best_len = p - off + 1;
+ best_repl = xcda_value(tok, node);
+ }
+ }
+ }
+ if (best_len > 0 && best_repl < tok->prefix_replacements_len) {
+ const char *r = tok->prefix_replacements + best_repl;
+ size_t max = tok->prefix_replacements_len - best_repl;
+ size_t n = strnlen(r, max);
+ return (norm_prefix_t){r, n, best_len};
+ }
+
+ size_t clen = 1;
+ if (valid_utf8_one(input, input_len, off, &clen))
+ return (norm_prefix_t){input + off, clen, clen};
+ return (norm_prefix_t){repl, 3, 1};
+}
+
+static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) {
+ ws->norm_len = 0;
+ if (!text)
+ return -1;
+ size_t input_len = strlen(text);
+ const char space[] = "\xe2\x96\x81"; /* U+2581 */
+ const char *space_s = tok->escape_whitespaces ? space : " ";
+ size_t space_n = tok->escape_whitespaces ? 3 : 1;
+ int processing_non_ws = 0;
+ int space_prepended = 0;
+
+ for (size_t off = 0; off < input_len;) {
+ norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
+ for (size_t i = 0; i < r.len; i++) {
+ char c = r.data[i];
+ if (c != ' ') {
+ if (!processing_non_ws) {
+ processing_non_ws = 1;
+ if ((tok->add_dummy_prefix && !space_prepended) ||
+ tok->remove_extra_whitespaces) {
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
+ 0)
+ return -1;
+ space_prepended = 1;
+ }
+ }
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
+ return -1;
+ } else {
+ processing_non_ws = 0;
+ if (!tok->remove_extra_whitespaces) {
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0)
+ return -1;
+ }
+ }
+ }
+ off += r.consumed ? r.consumed : 1;
+ }
+ if (ws->norm)
+ ws->norm[ws->norm_len] = '\0';
+ return 0;
+}
+
+static double compute_unknown_score(const tok_spm_t *tok) {
+ float min_score = FLT_MAX;
+ for (int i = 0; i < tok->piece_count; i++) {
+ if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
+ min_score = tok->scores[i];
+ }
+ if (min_score == FLT_MAX)
+ min_score = -10.0f;
+ return (double)min_score - 10.0;
+}
+
+static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
+ size_t n = ws->norm_len;
+ if (n == 0)
+ return 0;
+ if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 ||
+ reserve_ids(ws, (int)n + 1) != 0)
+ return -1;
+ for (size_t i = 0; i <= n; i++)
+ ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
+ ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
+ double unk_score = tok->unknown_score;
+
+ for (size_t off = 0; off < n;) {
+ size_t cp_len = 1;
+ valid_utf8_one(ws->norm, n, off, &cp_len);
+ sp_best_t cur = ws->best[off];
+ if (cur.score <= -DBL_MAX / 2.0) {
+ off += cp_len;
+ continue;
+ }
+
+ int single_cp_found = 0;
+ for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
+ int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off,
+ (int)(end - off));
+ if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
+ if (end - off == cp_len)
+ single_cp_found = 1;
+ double sc =
+ cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
+ if (sc > ws->best[end].score)
+ ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
+ }
+ if (end == n)
+ break;
+ size_t next_len = 1;
+ valid_utf8_one(ws->norm, n, end, &next_len);
+ end += next_len;
+ }
+ if (!single_cp_found) {
+ size_t end = off + cp_len;
+ double sc = cur.score + unk_score;
+ if (sc > ws->best[end].score)
+ ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
+ }
+ off += cp_len;
+ }
+
+ int out_n = 0;
+ int prev_unk = 0;
+ for (size_t off = n; off > 0;) {
+ sp_best_t b = ws->best[off];
+ if (b.prev < 0 || b.prev >= (int)off)
+ return -1;
+ int is_unk = b.id == tok->unk_id;
+ if (!(prev_unk && is_unk)) {
+ if (out_n >= ws->ids_cap)
+ return -1;
+ ws->ids[out_n++] = b.id;
+ }
+ prev_unk = is_unk;
+ off = (size_t)b.prev;
+ }
+ for (int i = 0; i < out_n / 2; i++) {
+ int t = ws->ids[i];
+ ws->ids[i] = ws->ids[out_n - 1 - i];
+ ws->ids[out_n - 1 - i] = t;
+ }
+ return out_n;
+}
+
+tok_spm_t *tok_spm_load(const char *model_dir) {
+ if (!model_dir)
+ return NULL;
+ char path[1024];
+ if (find_model_file(model_dir, path) != 0) {
+ fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
+ return NULL;
+ }
+ unsigned char *data = NULL;
+ size_t len = 0;
+ if (load_file(path, &data, &len) != 0) {
+ fprintf(stderr, "sentencepiece: cannot read %s\n", path);
+ return NULL;
+ }
+
+ tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t));
+ if (!tok) {
+ free(data);
+ return NULL;
+ }
+ tok->add_dummy_prefix = 1;
+ tok->remove_extra_whitespaces = 1;
+ tok->escape_whitespaces = 1;
+ tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY;
+ if (parse_model(tok, data, len) != 0) {
+ fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
+ free(data);
+ tok_spm_free(tok);
+ return NULL;
+ }
+ free(data);
+
+ int cap = next_pow2(tok->piece_count * 2 + 1);
+ tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t));
+ if (!tok->piece_map) {
+ tok_spm_free(tok);
+ return NULL;
+ }
+ tok->piece_map_cap = cap;
+ for (int i = 0; i < tok->piece_count; i++) {
+ if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
+ tok->piece_lens[i], i) != 0) {
+ tok_spm_free(tok);
+ return NULL;
+ }
+ }
+
+ tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
+ tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
+ tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
+ tok->pad_id =
+ tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
+ tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA
+ ? tok->piece_count + 1
+ : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
+ tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
+ tok->unknown_score = compute_unknown_score(tok);
+ if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
+ fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
+ tok_spm_free(tok);
+ return NULL;
+ }
+ return tok;
+}
+
+int tok_spm_token_id(const tok_spm_t *tok, const char *token) {
+ if (!tok || !token)
+ return -1;
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
+ if (!strcmp(token, "<s>"))
+ return 0;
+ if (!strcmp(token, "<pad>"))
+ return 1;
+ if (!strcmp(token, "</s>"))
+ return 2;
+ if (!strcmp(token, "<unk>"))
+ return 3;
+ if (!strcmp(token, "<mask>"))
+ return tok->piece_count + 1;
+ }
+ return raw_to_model_id(tok, raw_piece_id(tok, token));
+}
+
+static int tok_spm_encode_into(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return -1;
+ if (normalize_into(tok, ws, text) != 0)
+ return -1;
+ int n = segment(tok, ws);
+ if (n < 0)
+ return -1;
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ if (n == 0)
+ return 0;
+ if (!out_ids || out_cap < n)
+ return -2;
+ memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
+ return 0;
+}
+
+int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return NULL;
+ int n = 0;
+ int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n);
+ if (rc < 0 && rc != -2)
+ return NULL;
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ if (n == 0)
+ return NULL;
+ int *ids = (int *)malloc((size_t)n * sizeof(*ids));
+ if (!ids)
+ return NULL;
+ memcpy(ids, ws->ids, (size_t)n * sizeof(*ids));
+ return ids;
+}
+
+tok_spm_workspace_t *tok_spm_workspace_new(void) {
+ return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t));
+}
+
+void tok_spm_workspace_free(tok_spm_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->norm);
+ free(ws->best);
+ free(ws->ids);
+ free(ws);
+}
+
+void tok_spm_free(tok_spm_t *tok) {
+ if (!tok)
+ return;
+ if (tok->id_to_piece) {
+ for (int i = 0; i < tok->piece_count; i++)
+ free(tok->id_to_piece[i]);
+ }
+ free(tok->id_to_piece);
+ free(tok->piece_lens);
+ free(tok->scores);
+ free(tok->types);
+ free(tok->piece_map);
+ free(tok->xcda);
+ free(tok->prefix_replacements);
+ free(tok);
+}
diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h
new file mode 100644
index 0000000..2230538
--- /dev/null
+++ b/liblangdet/sentencepiece.h
@@ -0,0 +1,71 @@
+/*
+ * SentencePiece tokenizer for XLM-R style models.
+ *
+ * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model,
+ * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R
+ * and multilingual E5. It encodes text only; frontends add the model's special
+ * token layout, e.g. <s> ... </s> for XLM-R.
+ */
+
+#ifndef TOK_SPM_H
+#define TOK_SPM_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef enum {
+ TOK_SPM_IDENTITY = 0,
+ TOK_SPM_XLM_ROBERTA = 1,
+} tok_spm_id_map_t;
+
+typedef struct {
+ char **id_to_piece; /* [piece_count] raw SentencePiece ids */
+ uint16_t *piece_lens; /* byte length for id_to_piece entries */
+ float *scores; /* SentencePiece score per raw id */
+ uint8_t *types; /* SentencePiece piece type per raw id */
+ int piece_count; /* raw SentencePiece vocabulary size */
+ int vocab_size; /* model embedding vocab size after id mapping */
+ int unk_id; /* model-space unknown-token id */
+ int bos_id; /* model-space beginning-token id, or -1 */
+ int eos_id; /* model-space end-token id, or -1 */
+ int pad_id; /* model-space pad-token id, or -1 */
+ int mask_id; /* model-space mask-token id, or -1 */
+ int max_piece_len; /* max normal/user/unused piece byte length */
+ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
+ int add_dummy_prefix;
+ int remove_extra_whitespaces;
+ int escape_whitespaces;
+ tok_spm_id_map_t id_map;
+
+ void *piece_map; /* piece bytes -> raw SentencePiece id */
+ int piece_map_cap;
+
+ uint32_t *xcda;
+ size_t xcda_count;
+ char *prefix_replacements;
+ size_t prefix_replacements_len;
+} tok_spm_t;
+
+typedef struct tok_spm_workspace tok_spm_workspace_t;
+
+/* Load the first supported SentencePiece model found in model_dir. Returns NULL
+ * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+tok_spm_t *tok_spm_load(const char *model_dir);
+
+/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
+ * model-space id, or -1 when absent. */
+int tok_spm_token_id(const tok_spm_t *tok, const char *token);
+
+tok_spm_workspace_t *tok_spm_workspace_new(void);
+void tok_spm_workspace_free(tok_spm_workspace_t *ws);
+
+/* Return malloc-owned model-space token IDs, reusing workspace scratch.
+ * Empty text returns NULL with a zero count. */
+int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+void tok_spm_free(tok_spm_t *tok);
+
+#endif /* TOK_SPM_H */
diff --git a/liblangdet/threadpool.c b/liblangdet/threadpool.c
new file mode 100644
index 0000000..6cb4252
--- /dev/null
+++ b/liblangdet/threadpool.c
@@ -0,0 +1,218 @@
+/*
+ * CPU thread pool.
+ */
+
+#if defined(__linux__) && !defined(_GNU_SOURCE)
+# define _GNU_SOURCE
+#endif
+
+#include "threadpool.h"
+
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef __APPLE__
+# include <sys/sysctl.h>
+#else
+# ifdef __linux__
+# include <sched.h>
+# endif
+# include <unistd.h>
+#endif
+
+#if defined(USE_OPENBLAS)
+void openblas_set_num_threads(int num_threads);
+#endif
+
+#define TP_MAX_THREADS 16
+
+static struct {
+ pthread_t threads[TP_MAX_THREADS - 1];
+ int tids[TP_MAX_THREADS - 1];
+ int n_threads;
+ int shutdown;
+
+ parallel_fn_t fn;
+ void *arg;
+ uint64_t generation;
+
+ pthread_mutex_t mutex;
+ pthread_cond_t cond_work;
+ pthread_cond_t cond_done;
+ int n_done;
+} tp = {
+ .n_threads = 1,
+ .shutdown = 0,
+ .generation = 0,
+ .mutex = PTHREAD_MUTEX_INITIALIZER,
+ .cond_work = PTHREAD_COND_INITIALIZER,
+ .cond_done = PTHREAD_COND_INITIALIZER,
+};
+
+static void *worker_loop(void *arg) {
+ int tid = *(int *)arg;
+ uint64_t my_gen = 0;
+
+ for (;;) {
+ pthread_mutex_lock(&tp.mutex);
+ while (tp.generation == my_gen && !tp.shutdown)
+ pthread_cond_wait(&tp.cond_work, &tp.mutex);
+ if (tp.shutdown) {
+ pthread_mutex_unlock(&tp.mutex);
+ return NULL;
+ }
+ my_gen = tp.generation;
+ parallel_fn_t fn = tp.fn;
+ void *a = tp.arg;
+ int nt = tp.n_threads;
+ pthread_mutex_unlock(&tp.mutex);
+
+ fn(tid, nt, a);
+
+ pthread_mutex_lock(&tp.mutex);
+ if (++tp.n_done >= tp.n_threads - 1)
+ pthread_cond_signal(&tp.cond_done);
+ pthread_mutex_unlock(&tp.mutex);
+ }
+}
+
+void tp_set_threads(int n) {
+ if (n < 1)
+ n = 1;
+ if (n > TP_MAX_THREADS)
+ n = TP_MAX_THREADS;
+
+#if defined(USE_OPENBLAS)
+ /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the
+ * auxiliary pool serial to avoid nested thread contention. */
+ int blas_threads = n;
+ openblas_set_num_threads(blas_threads);
+ n = 1;
+#endif
+
+ /* Shutdown existing workers */
+ if (tp.n_threads > 1) {
+ pthread_mutex_lock(&tp.mutex);
+ tp.shutdown = 1;
+ pthread_cond_broadcast(&tp.cond_work);
+ pthread_mutex_unlock(&tp.mutex);
+ for (int i = 0; i < tp.n_threads - 1; i++)
+ pthread_join(tp.threads[i], NULL);
+ tp.shutdown = 0;
+ tp.generation = 0;
+ }
+
+ tp.n_threads = n;
+ if (n <= 1)
+ return;
+
+ for (int i = 0; i < n - 1; i++) {
+ tp.tids[i] = i + 1;
+ pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]);
+ }
+}
+
+#ifndef __APPLE__
+/* CPU quota available to this process, in whole cores, or 0 if uncapped.
+ * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host
+ * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process
+ * can actually use.
+ *
+ * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota:
+ * threads use the period budget early, then get throttled for the rest of the
+ * period, causing large bursty stalls. Example: a 16-thread default against a
+ * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool.
+ *
+ * Floor instead of ceil, so a fully busy pool stays under quota and avoids
+ * throttling.
+ */
+static int cgroup_cpu_quota(void) {
+ /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or
+ * "max <period>" when unlimited. */
+ FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r");
+ if (f) {
+ char quota[32];
+ long period = 0;
+ int got = fscanf(f, "%31s %ld", quota, &period);
+ fclose(f);
+ if (got == 2 && period > 0 && strcmp(quota, "max") != 0) {
+ long q = atol(quota);
+ if (q > 0)
+ return q / period < 1 ? 1 : (int)(q / period);
+ }
+ }
+ /* cgroup v1: separate files; quota -1 means unlimited. */
+ long q = -1, period = 0;
+ f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r");
+ if (f) {
+ if (fscanf(f, "%ld", &q) != 1)
+ q = -1;
+ fclose(f);
+ }
+ f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r");
+ if (f) {
+ if (fscanf(f, "%ld", &period) != 1)
+ period = 0;
+ fclose(f);
+ }
+ if (q > 0 && period > 0)
+ return q / period < 1 ? 1 : (int)(q / period);
+ return 0;
+}
+#endif
+
+int tp_get_num_cpus(void) {
+#ifdef __APPLE__
+ int n = 0;
+ size_t len = sizeof(n);
+ sysctlbyname("hw.ncpu", &n, &len, NULL, 0);
+ return n > 0 ? n : 1;
+#else
+ int online = (int)sysconf(_SC_NPROCESSORS_ONLN);
+ if (online < 1)
+ online = 1;
+# ifdef __linux__
+ cpu_set_t set;
+ if (sched_getaffinity(0, sizeof(set), &set) == 0) {
+ int affinity = CPU_COUNT(&set);
+ if (affinity >= 1 && affinity < online)
+ online = affinity;
+ }
+# endif
+ /* Honor a cgroup CPU quota when it is tighter than the visible core count,
+ * so the default thread pool does not oversubscribe a CPU-limited
+ * container and get throttled. */
+ int quota = cgroup_cpu_quota();
+ if (quota >= 1 && quota < online)
+ return quota;
+ return online;
+#endif
+}
+
+int tp_num_threads(void) { return tp.n_threads; }
+
+/* Dispatch work to all threads; main thread is tid=0 */
+void parallel_for(parallel_fn_t fn, void *arg) {
+ if (tp.n_threads <= 1) {
+ fn(0, 1, arg);
+ return;
+ }
+
+ pthread_mutex_lock(&tp.mutex);
+ tp.fn = fn;
+ tp.arg = arg;
+ tp.n_done = 0;
+ tp.generation++;
+ pthread_cond_broadcast(&tp.cond_work);
+ pthread_mutex_unlock(&tp.mutex);
+
+ fn(0, tp.n_threads, arg);
+
+ pthread_mutex_lock(&tp.mutex);
+ while (tp.n_done < tp.n_threads - 1)
+ pthread_cond_wait(&tp.cond_done, &tp.mutex);
+ pthread_mutex_unlock(&tp.mutex);
+}
diff --git a/liblangdet/threadpool.h b/liblangdet/threadpool.h
new file mode 100644
index 0000000..f9739a1
--- /dev/null
+++ b/liblangdet/threadpool.h
@@ -0,0 +1,30 @@
+#ifndef TP_THREADPOOL_H
+#define TP_THREADPOOL_H
+
+/* Persistent worker pool for CPU kernels.
+ * Uses a fixed thread set that sleeps until parallel_for() gives it a callback.
+ * Also provides the cgroup-quota-aware CPU-count helper used to size the pool.
+ *
+ * Kept separate from the kernels so they depend only on parallel_for and
+ * tp_num_threads, not pool internals.
+ *
+ * kernels.h includes this, so kernel callers need no extra include. */
+
+/* Worker callback: (thread index, total threads, user arg). */
+typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg);
+
+/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call
+ * before inference. */
+void tp_set_threads(int n);
+
+/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */
+int tp_get_num_cpus(void);
+
+/* Current pool size (1 == serial). Kernels gate parallelism on this. */
+int tp_num_threads(void);
+
+/* Run fn across the pool; the calling thread participates as tid 0. Blocks
+ * until every worker has finished. */
+void parallel_for(parallel_fn_t fn, void *arg);
+
+#endif /* TP_THREADPOOL_H */
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
new file mode 100644
index 0000000..5e26a18
--- /dev/null
+++ b/liblangdet/tokenize.c
@@ -0,0 +1,85 @@
+#include "langdet.h"
+#include "sentencepiece.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct langdet_tok {
+ tok_spm_t *spm;
+ tok_spm_workspace_t *workspace;
+ int cls_id;
+ int sep_id;
+};
+
+static void set_err(char *err, size_t errlen, const char *msg) {
+ if (err && errlen)
+ snprintf(err, errlen, "%s", msg);
+}
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) {
+ if (!model_dir) {
+ set_err(err, errlen, "model_dir is NULL");
+ return NULL;
+ }
+ langdet_tok_t *t = calloc(1, sizeof(*t));
+ if (!t) {
+ set_err(err, errlen, "out of memory");
+ return NULL;
+ }
+ t->spm = tok_spm_load(model_dir);
+ if (!t->spm) {
+ set_err(err, errlen, "failed to load SentencePiece tokenizer");
+ goto fail;
+ }
+ t->workspace = tok_spm_workspace_new();
+ if (!t->workspace) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_spm_token_id(t->spm, "<s>");
+ t->sep_id = tok_spm_token_id(t->spm, "</s>");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
+ goto fail;
+ }
+ return t;
+
+fail:
+ langdet_tok_free(t);
+ return NULL;
+}
+
+void langdet_tok_free(langdet_tok_t *t) {
+ if (!t)
+ return;
+ tok_spm_workspace_free(t->workspace);
+ tok_spm_free(t->spm);
+ free(t);
+}
+
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
+ if (n_out)
+ *n_out = 0;
+ if (!t || !text)
+ return NULL;
+ int n = 0;
+ int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
+ if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
+ free(core);
+ return NULL;
+ }
+ /* Empty text still produces the two special tokens. */
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ return NULL;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = t->cls_id;
+ ids[n + 1] = t->sep_id;
+ if (n_out)
+ *n_out = n + 2;
+ return ids;
+}