diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet')
| -rw-r--r-- | liblangdet/LICENSE | 20 | ||||
| -rw-r--r-- | liblangdet/act.c | 29 | ||||
| -rw-r--r-- | liblangdet/alloc.c | 65 | ||||
| -rw-r--r-- | liblangdet/alloc.h | 22 | ||||
| -rw-r--r-- | liblangdet/attn.c | 242 | ||||
| -rw-r--r-- | liblangdet/config.c | 67 | ||||
| -rw-r--r-- | liblangdet/config.h | 8 | ||||
| -rw-r--r-- | liblangdet/forward.c | 96 | ||||
| -rw-r--r-- | liblangdet/gemm.c | 49 | ||||
| -rw-r--r-- | liblangdet/generic.c | 27 | ||||
| -rw-r--r-- | liblangdet/impl.h | 254 | ||||
| -rw-r--r-- | liblangdet/kernels.h | 42 | ||||
| -rw-r--r-- | liblangdet/langdet.h | 49 | ||||
| -rw-r--r-- | liblangdet/model.c | 159 | ||||
| -rw-r--r-- | liblangdet/model_blas.c | 68 | ||||
| -rw-r--r-- | liblangdet/model_internal.h | 17 | ||||
| -rw-r--r-- | liblangdet/model_types.h | 56 | ||||
| -rw-r--r-- | liblangdet/norm.c | 33 | ||||
| -rw-r--r-- | liblangdet/safetensors.c | 483 | ||||
| -rw-r--r-- | liblangdet/safetensors.h | 64 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.c | 905 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.h | 71 | ||||
| -rw-r--r-- | liblangdet/threadpool.c | 218 | ||||
| -rw-r--r-- | liblangdet/threadpool.h | 30 | ||||
| -rw-r--r-- | liblangdet/tokenize.c | 85 |
25 files changed, 3159 insertions, 0 deletions
diff --git a/liblangdet/LICENSE b/liblangdet/LICENSE new file mode 100644 index 0000000..a00caa7 --- /dev/null +++ b/liblangdet/LICENSE @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) 2026 The ffwd authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/liblangdet/act.c b/liblangdet/act.c new file mode 100644 index 0000000..3912f2f --- /dev/null +++ b/liblangdet/act.c @@ -0,0 +1,29 @@ +#include "kernels.h" + +#include <math.h> + +void gelu_inplace(float *x, int n) { + /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ + for (int i = 0; i < n; i++) + x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); +} + +void softmax(float *x, int rows, int cols) { + for (int r = 0; r < rows; r++) { + float *row = x + r * cols; + float max_val = row[0]; + for (int c = 1; c < cols; c++) { + if (row[c] > max_val) + max_val = row[c]; + } + float sum = 0.0f; + for (int c = 0; c < cols; c++) { + row[c] = expf(row[c] - max_val); + sum += row[c]; + } + float inv_sum = 1.0f / sum; + for (int c = 0; c < cols; c++) { + row[c] *= inv_sum; + } + } +} diff --git a/liblangdet/alloc.c b/liblangdet/alloc.c new file mode 100644 index 0000000..4bc1496 --- /dev/null +++ b/liblangdet/alloc.c @@ -0,0 +1,65 @@ +#include "alloc.h" + +#include <stdlib.h> +#include <stdint.h> +#include <limits.h> + +/* Sequence-capacity growth policy for the forward-pass workspaces: + * For new growth: + * - never allocate less than the minimum required size. + * - round up to the granularity so small length changes do not trigger a realloc on every call. + * - keep existing capacity if it is already larger. */ +#define LANGDET_MIN_WORKSPACE_SEQ_CAP 16 +#define LANGDET_WORKSPACE_SEQ_GRANULARITY 16 + +int mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; + *out = a * b; + return 0; +} + +int grow_cap(int current, int needed, int *out) { + if (current < 0 || needed <= 0) + return -1; + + int cap = needed; + if (cap < LANGDET_MIN_WORKSPACE_SEQ_CAP) + cap = LANGDET_MIN_WORKSPACE_SEQ_CAP; + + int rem = cap % LANGDET_WORKSPACE_SEQ_GRANULARITY; + if (rem != 0) { + int pad = LANGDET_WORKSPACE_SEQ_GRANULARITY - rem; + if (cap > INT_MAX - pad) + return -1; + cap += pad; + } + + if (cap < current) + cap = current; + + *out = cap; + return 0; +} + +int realloc_floats(float **ptr, size_t count) { + size_t bytes; + if (mul_size(count, sizeof(float), &bytes) != 0) + return -1; + + void *p = realloc(*ptr, bytes); + if (!p && bytes != 0) + return -1; + *ptr = (float *)p; + return 0; +} + +int realloc_floats_2d(float **ptr, int rows, int cols) { + if (rows < 0 || cols < 0) + return -1; + + size_t count; + if (mul_size((size_t)rows, (size_t)cols, &count) != 0) + return -1; + return realloc_floats(ptr, count); +} diff --git a/liblangdet/alloc.h b/liblangdet/alloc.h new file mode 100644 index 0000000..f70f1f7 --- /dev/null +++ b/liblangdet/alloc.h @@ -0,0 +1,22 @@ +#ifndef LANGDET_ALLOC_H +#define LANGDET_ALLOC_H + +#include <stddef.h> + +/* Checked size arithmetic and typed allocation helpers. + * Return -1 or NULL on overflow/allocation failure. + * realloc_* leave the original pointer unchanged on failure. */ + +/* out = a * b, or -1 if the product overflows size_t. */ +int mul_size(size_t a, size_t b, size_t *out); + +/* Grow sequence capacity without shrinking. */ +int grow_cap(int current, int needed, int *out); + +/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ +int realloc_floats(float **ptr, size_t count); + +/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ +int realloc_floats_2d(float **ptr, int rows, int cols); + +#endif /* LANGDET_ALLOC_H */ diff --git a/liblangdet/attn.c b/liblangdet/attn.c new file mode 100644 index 0000000..ac0f4d1 --- /dev/null +++ b/liblangdet/attn.c @@ -0,0 +1,242 @@ +#include "kernels.h" +#include "impl.h" +#include "threadpool.h" + +#include <math.h> + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif +#endif + +static inline float dot_f32_fast(const float *a, const float *b, int n) { + return dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void vec_scale_inplace(float *dst, float scale, int n) { + vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { + vec_scale_add_impl(dst, src, correction, n); +} + +#define PACKED_ATTN_QUERY_TILE 32 +#define PACKED_ATTN_BLAS_MIN_SEQ 128 + +static void bidirectional_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + vec_scale_inplace(o_row, inv_sum, head_dim); + } + } +} + +#ifdef USE_BLAS +static void bidirectional_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, + kv_hidden, 0.0f, scores, keys); + softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; + float *scores; + int max_seq; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { + bidirectional_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1, scores); + continue; + } +#endif + bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, + t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); + } + } + } +} + +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) + return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) + max_seq = len; + } + if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) + return 0; + if ((size_t)max_seq > + SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) + return 0; + return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { + parallel_for(packed_gqa_attn_worker, &task); + } else { + packed_gqa_attn_worker(0, 1, &task); + } +} diff --git a/liblangdet/config.c b/liblangdet/config.c new file mode 100644 index 0000000..f22a9d7 --- /dev/null +++ b/liblangdet/config.c @@ -0,0 +1,67 @@ +#include "config.h" + +#include <yyjson.h> + +#include <limits.h> +#include <math.h> +#include <stdio.h> + +static int json_get_int(yyjson_val *obj, const char *key, int fallback) { + yyjson_val *v = yyjson_obj_get(obj, key); + if (!v) + return fallback; + if (!yyjson_is_int(v)) + return -1; + int64_t n = yyjson_get_sint(v); + return n >= 0 && n <= INT_MAX ? (int)n : -1; +} + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "langdet_config: model path too long\n"); + return -1; + } + + yyjson_read_err error; + yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); + if (!doc) { + fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg); + return -1; + } + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *causal = yyjson_obj_get(root, "is_causal"); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || + (causal && !yyjson_is_false(causal))) { + fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n"); + yyjson_doc_free(doc); + return -1; + } + + cfg->hidden_size = json_get_int(root, "hidden_size", 0); + cfg->n_layers = json_get_int(root, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(root, "num_attention_heads", 0); + cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + cfg->intermediate_size = json_get_int(root, "intermediate_size", 0); + cfg->vocab_size = json_get_int(root, "vocab_size", 0); + cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0); + cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1); + yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps"); + cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f; + int pad_id = json_get_int(root, "pad_token_id", 1); + cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; + yyjson_doc_free(doc); + + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS || + cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || + cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || + cfg->position_id_offset < 0 || + cfg->max_position_embeddings <= cfg->position_id_offset || + !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path); + return -1; + } + return 0; +} diff --git a/liblangdet/config.h b/liblangdet/config.h new file mode 100644 index 0000000..79522fc --- /dev/null +++ b/liblangdet/config.h @@ -0,0 +1,8 @@ +#ifndef LANGDET_CONFIG_H +#define LANGDET_CONFIG_H + +#include "langdet.h" + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir); + +#endif /* LANGDET_CONFIG_H */ diff --git a/liblangdet/forward.c b/liblangdet/forward.c new file mode 100644 index 0000000..14b1327 --- /dev/null +++ b/liblangdet/forward.c @@ -0,0 +1,96 @@ +#include "model_internal.h" +#include "kernels.h" + +#include <limits.h> +#include <math.h> +#include <stdio.h> +#include <string.h> + +static void add_bias_rows(float *x, const float *bias, int rows, int dim) { + for (int row = 0; row < rows; row++) { + float *dst = x + (size_t)row * dim; + for (int d = 0; d < dim; d++) + dst[d] += bias[d]; + } +} + +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding) { + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) + return -1; + + const langdet_config_t *cfg = &model->config; + const langdet_weights_t *w = &model->weights; + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; + int head_dim = cfg->head_dim; + int inter = cfg->intermediate_size; + float eps = cfg->layer_norm_eps; + + if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { + fprintf(stderr, + "langdet: sequence length %d plus position offset %d exceeds max_position_embeddings " + "%d\n", + n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); + return -1; + } + if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || + ensure_buffers(ws, cfg, n_tokens) != 0) + return -1; + int offsets[2] = {0, n_tokens}; + ensure_attention_scores(ws, offsets, 1); + + float *x = ws->x; + float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; + float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; + + for (int i = 0; i < n_tokens; i++) { + int id = token_ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "langdet: invalid token id %d at position %d\n", id, i); + return -1; + } + float *row = x + (size_t)i * hidden; + memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float)); + int pos = cfg->position_id_offset + i; + add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); + add_inplace(row, w->token_type_embeddings, hidden); + } + layer_norm(x, x, w->langdet_ln_w, w->langdet_ln_b, n_tokens, hidden, eps); + + float scale = 1.0f / sqrtf((float)head_dim); + + for (int layer = 0; layer < cfg->n_layers; layer++) { + const langdet_layer_t *l = &w->layers[layer]; + + linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); + linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); + linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden); + add_bias_rows(q_buf, l->q_bias, n_tokens, hidden); + add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); + add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); + + bidirectional_gqa_attention_packed_with_scratch( + attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); + + linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); + add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps); + + linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter); + add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter); + gelu_inplace(ffn_up, n_tokens * inter); + linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden); + add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps); + } + /* The classifier consumes <s> after the encoder's final LayerNorm, without L2 normalization. */ + memcpy(out_embedding, x, (size_t)hidden * sizeof(float)); + return 0; +} diff --git a/liblangdet/gemm.c b/liblangdet/gemm.c new file mode 100644 index 0000000..c750be1 --- /dev/null +++ b/liblangdet/gemm.c @@ -0,0 +1,49 @@ +#include "kernels.h" + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif +#endif + +void add_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) + a[i] += b[i]; +} + +void linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) { +#ifdef USE_BLAS + if (seq_len == 1) { + cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1); + } else { + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, + in_dim, W, in_dim, 0.0f, y, out_dim); + } + if (b != NULL) { + for (int s = 0; s < seq_len; s++) { + for (int o = 0; o < out_dim; o++) { + y[s * out_dim + o] += b[o]; + } + } + } +#else + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * in_dim; + float *y_row = y + s * out_dim; + for (int o = 0; o < out_dim; o++) { + const float *w_row = W + o * in_dim; + float sum = (b != NULL) ? b[o] : 0.0f; + for (int i = 0; i < in_dim; i++) + sum += x_row[i] * w_row[i]; + y_row[o] = sum; + } + } +#endif +} + +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { + linear(y, x, W, NULL, seq_len, in_dim, out_dim); +} diff --git a/liblangdet/generic.c b/liblangdet/generic.c new file mode 100644 index 0000000..729c610 --- /dev/null +++ b/liblangdet/generic.c @@ -0,0 +1,27 @@ +/* + * Architecture-generic hot kernels / non-SIMD fallback. + */ + +#include "impl.h" + +float dot_f32_generic(const float *a, const float *b, int n) { + float sum = 0.0f; + for (int i = 0; i < n; i++) + sum += a[i] * b[i]; + return sum; +} + +void vec_scale_inplace_generic(float *dst, float scale, int n) { + for (int i = 0; i < n; i++) + dst[i] *= scale; +} + +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { + for (int i = 0; i < n; i++) + dst[i] += alpha * src[i]; +} + +void vec_scale_add_generic(float *dst, const float *src, float correction, int n) { + for (int i = 0; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +} diff --git a/liblangdet/impl.h b/liblangdet/impl.h new file mode 100644 index 0000000..f216518 --- /dev/null +++ b/liblangdet/impl.h @@ -0,0 +1,254 @@ +/* Inline vector operations keep attention rows in registers across reductions. */ + +#ifndef KERNELS_IMPL_H +#define KERNELS_IMPL_H + +#include <stdint.h> + +float dot_f32_generic(const float *a, const float *b, int n); +void vec_scale_inplace_generic(float *dst, float scale, int n); +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON + +# include <arm_neon.h> + +static inline float dot_f32_impl(const float *a, const float *b, int n) { + int i = 0; + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; i + 8 <= n; i += 8) { + float32x4_t a0 = vld1q_f32(a + i); + float32x4_t b0 = vld1q_f32(b + i); + float32x4_t a1 = vld1q_f32(a + i + 4); + float32x4_t b1 = vld1q_f32(b + i + 4); + acc0 = vfmaq_f32(acc0, a0, b0); + acc1 = vfmaq_f32(acc1, a1, b1); + } + float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { + int i = 0; + float32x4_t s = vdupq_n_f32(scale); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); + vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); + } + for (; i < n; i++) + dst[i] *= scale; +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { + int i = 0; + float32x4_t a = vdupq_n_f32(alpha); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); + vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { + int i = 0; + float32x4_t c = vdupq_n_f32(correction); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); + vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +} + +#elif defined(__AVX2__) && defined(__FMA__) + +# include <immintrin.h> + +static inline float dot_f32_impl(const float *a, const float *b, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); + } + __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); + for (; i + 16 <= n; i += 16) { + acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# else + int i = 0; + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); + } + acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + for (; i + 8 <= n; i += 8) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + } + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# endif +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 s = _mm512_set1_ps(scale); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); + _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); + _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# else + int i = 0; + __m256 s = _mm256_set1_ps(scale); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); + _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); + _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# endif +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 a = _mm512_set1_ps(alpha); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, + _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, + _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, + _mm512_loadu_ps(dst + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# else + int i = 0; + __m256 a = _mm256_set1_ps(alpha); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, + _mm256_loadu_ps(dst + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, + _mm256_loadu_ps(dst + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, + _mm256_loadu_ps(dst + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# endif +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 c = _mm512_set1_ps(correction); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, + _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, + _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, + _mm512_loadu_ps(src + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# else + int i = 0; + __m256 c = _mm256_set1_ps(correction); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, + _mm256_loadu_ps(src + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, + _mm256_loadu_ps(src + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, + _mm256_loadu_ps(src + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# endif +} + +#else /* no SIMD path: dispatch to the scalar references */ + +# define dot_f32_impl dot_f32_generic +# define vec_scale_inplace_impl vec_scale_inplace_generic +# define vec_axpy_inplace_impl vec_axpy_inplace_generic +# define vec_scale_add_impl vec_scale_add_generic + +#endif + +#endif /* KERNELS_IMPL_H */ diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h new file mode 100644 index 0000000..1025602 --- /dev/null +++ b/liblangdet/kernels.h @@ -0,0 +1,42 @@ +#ifndef KERNELS_H +#define KERNELS_H + +#include <stddef.h> +#include <stdint.h> + +void add_inplace(float *a, const float *b, int n); + +/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */ +void linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); + +/* Mean and population variance are reduced over the hidden axis per row. */ +void layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps); + +void softmax(float *x, int rows, int cols); +void gelu_inplace(float *x, int n); + +/* Attention is block-diagonal: each token attends only within its offsets span. + * Missing scratch uses the online-softmax path instead of tiled BLAS. */ +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); +void bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); + +#endif /* KERNELS_H */ diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h new file mode 100644 index 0000000..9c24264 --- /dev/null +++ b/liblangdet/langdet.h @@ -0,0 +1,49 @@ +#ifndef LANGDET_H +#define LANGDET_H + +#include <stddef.h> + +#define LANGDET_MAX_LAYERS 64 + +typedef struct { + int hidden_size; + int n_layers; + int n_heads; + int head_dim; + int intermediate_size; + int vocab_size; + int max_position_embeddings; + int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ + int type_vocab_size; + float layer_norm_eps; +} langdet_config_t; + +typedef struct langdet_model langdet_model_t; +typedef struct langdet_workspace langdet_workspace_t; +typedef struct langdet_tok langdet_tok_t; + +/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ +langdet_model_t *langdet_model_load(const char *model_dir); +void langdet_model_free(langdet_model_t *model); +const langdet_config_t *langdet_model_config(const langdet_model_t *model); + +/* A workspace belongs to one model and may serve only one inference call at a time. */ +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); +void langdet_workspace_free(langdet_workspace_t *ws); + +/* Write the first token's final hidden state to out_embedding[hidden_size]. + * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); +void langdet_tok_free(langdet_tok_t *t); + +/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. + * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); + +#endif /* LANGDET_H */ diff --git a/liblangdet/model.c b/liblangdet/model.c new file mode 100644 index 0000000..146bbe7 --- /dev/null +++ b/liblangdet/model.c @@ -0,0 +1,159 @@ +#include "model_internal.h" +#include "config.h" + +#include <stdio.h> +#include <stdlib.h> + +int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim) { + if (t->dtype != DTYPE_F32) { + fprintf(stderr, "langdet: expected F32 tensor: %s\n", name); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (t->shape[i] != shape[i]) { + fprintf(stderr, + "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n", + name, i, (long long)t->shape[i], (long long)shape[i]); + return 0; + } + if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { + fprintf(stderr, "langdet: shape too large or empty for %s\n", name); + return 0; + } + numel *= (size_t)shape[i]; + } + size_t bytes; + if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { + fprintf(stderr, "langdet: invalid data size for %s\n", name); + return 0; + } + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static const float *load_weight(multi_safetensors_t *ms, const char *name, + const int64_t *shape, int ndim) { + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "langdet: tensor not found: %s\n", name); + return NULL; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) + return NULL; + return safetensors_data(sf, t); +} + +static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) { + const langdet_config_t *cfg = &model->config; + langdet_weights_t *w = &model->weights; + char name[256]; + int hidden = cfg->hidden_size, inter = cfg->intermediate_size; + + w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", + (int64_t[]){cfg->vocab_size, hidden}, 2); + w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", + (int64_t[]){cfg->max_position_embeddings, hidden}, 2); + w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", + (int64_t[]){cfg->type_vocab_size, hidden}, 2); + w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); + w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); + if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || + !w->langdet_ln_w || !w->langdet_ln_b) + return -1; + + w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t)); + if (!w->layers) + return -1; + + for (int i = 0; i < cfg->n_layers; i++) { + langdet_layer_t *l = &w->layers[i]; + +#define BERT_W(field, suffix, d0, d1) \ + do { \ + const int64_t e[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(ms, name, e, 2); \ + if (!l->field) \ + return -1; \ + } while (0) +#define BERT_V(field, suffix, d0) \ + do { \ + const int64_t e[1] = {(d0)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(ms, name, e, 1); \ + if (!l->field) \ + return -1; \ + } while (0) + + BERT_W(wq, "attention.self.query.weight", hidden, hidden); + BERT_V(q_bias, "attention.self.query.bias", hidden); + BERT_W(wk, "attention.self.key.weight", hidden, hidden); + BERT_V(k_bias, "attention.self.key.bias", hidden); + BERT_W(wv, "attention.self.value.weight", hidden, hidden); + BERT_V(v_bias, "attention.self.value.bias", hidden); + BERT_W(wo, "attention.output.dense.weight", hidden, hidden); + BERT_V(o_bias, "attention.output.dense.bias", hidden); + BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); + BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); + BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); + BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); + BERT_W(down_proj, "output.dense.weight", hidden, inter); + BERT_V(ffn_out_bias, "output.dense.bias", hidden); + BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); + BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); + +#undef BERT_W +#undef BERT_V + } + return 0; +} + +langdet_model_t *langdet_model_load(const char *model_dir) { + if (!model_dir) + return NULL; + langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t)); + if (!model) + return NULL; + if (langdet_config_parse(&model->config, model_dir) != 0) + goto fail; + model->safetensors = multi_safetensors_open(model_dir); + if (!model->safetensors) { + fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir); + goto fail; + } + if (load_bert_weights(model, model->safetensors) != 0) + goto fail; + return model; + +fail: + langdet_model_free(model); + return NULL; +} + +void langdet_model_free(langdet_model_t *model) { + if (!model) + return; + multi_safetensors_close(model->safetensors); + free(model->weights.layers); + free(model); +} + +const langdet_config_t *langdet_model_config(const langdet_model_t *model) { + return model ? &model->config : NULL; +} diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c new file mode 100644 index 0000000..41f9237 --- /dev/null +++ b/liblangdet/model_blas.c @@ -0,0 +1,68 @@ +#include "model_internal.h" +#include "kernels.h" + +#include <stdlib.h> + +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model) { + if (!model) + return NULL; + langdet_workspace_t *ws = (langdet_workspace_t *)calloc(1, sizeof(langdet_workspace_t)); + if (!ws) + return NULL; + ws->model = model; + return ws; +} + +void langdet_workspace_free(langdet_workspace_t *ws) { + if (!ws) + return; + free(ws->x); + free(ws->q); + free(ws->k); + free(ws->v); + free(ws->attn_out); + free(ws->proj_out); + free(ws->ffn_up); + free(ws->attn_scores); + free(ws); +} + +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) { + if (seq <= ws->buf_seq_cap) + return 0; + + int cap; + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) + return -1; + +#define R(ptr, n) \ + do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ + return -1; \ + } while (0) + + R(ws->x, c->hidden_size); + R(ws->q, c->hidden_size); + R(ws->k, c->hidden_size); + R(ws->v, c->hidden_size); + R(ws->attn_out, c->hidden_size); + R(ws->proj_out, c->hidden_size); + R(ws->ffn_up, c->intermediate_size); + +#undef R + + ws->buf_seq_cap = cap; + return 0; +} + +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) { + size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) + return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) + return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h new file mode 100644 index 0000000..73a7851 --- /dev/null +++ b/liblangdet/model_internal.h @@ -0,0 +1,17 @@ +#ifndef LANGDET_MODEL_INTERNAL_H +#define LANGDET_MODEL_INTERNAL_H + +#include "model_types.h" +#include "alloc.h" + +/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */ +int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim); + +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq); +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch); + +#endif /* LANGDET_MODEL_INTERNAL_H */ diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h new file mode 100644 index 0000000..2170909 --- /dev/null +++ b/liblangdet/model_types.h @@ -0,0 +1,56 @@ +#ifndef LANGDET_MODEL_TYPES_H +#define LANGDET_MODEL_TYPES_H + +#include "langdet.h" +#include "safetensors.h" + +typedef struct { + const float *wq; + const float *wk; + const float *wv; + const float *wo; + const float *q_bias; + const float *k_bias; + const float *v_bias; + const float *o_bias; + const float *input_norm; + const float *attn_ln_bias; + const float *up_proj; + const float *ffn_inter_bias; + const float *down_proj; + const float *ffn_out_bias; + const float *post_attn_norm; + const float *ffn_ln_bias; +} langdet_layer_t; + +typedef struct { + const float *embed_tokens; + langdet_layer_t *layers; + const float *position_embeddings; + const float *token_type_embeddings; + const float *langdet_ln_w; + const float *langdet_ln_b; +} langdet_weights_t; + +struct langdet_model { + langdet_config_t config; + langdet_weights_t weights; + /* All weight pointers borrow this mapping until model teardown. */ + multi_safetensors_t *safetensors; +}; + +struct langdet_workspace { + const langdet_model_t *model; + int buf_seq_cap; + float *x; + float *q; + float *k; + float *v; + float *attn_out; + float *proj_out; + float *ffn_up; + float *attn_scores; + size_t attn_scores_bytes; +}; + +#endif /* LANGDET_MODEL_TYPES_H */ diff --git a/liblangdet/norm.c b/liblangdet/norm.c new file mode 100644 index 0000000..59e8bdb --- /dev/null +++ b/liblangdet/norm.c @@ -0,0 +1,33 @@ +#include "kernels.h" + +#include <math.h> + +/* Safe in place: each row's mean and variance are reduced before it is rewritten. */ +void layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + float inv_h = 1.0f / (float)hidden; + for (int s = 0; s < seq_len; s++) { + const float *row = x + (size_t)s * hidden; + float *dst = out + (size_t)s * hidden; + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += row[d]; + mean *= inv_h; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = row[d] - mean; + var += diff * diff; + } + var *= inv_h; + float inv_std = 1.0f / sqrtf(var + eps); + for (int d = 0; d < hidden; d++) + dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; + } +} diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c new file mode 100644 index 0000000..cfba76e --- /dev/null +++ b/liblangdet/safetensors.c @@ -0,0 +1,483 @@ +#include "safetensors.h" + +#include <dirent.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <unistd.h> + +/* Minimal JSON parser for safetensors header */ +/* TODO: consider replacing with yyjson */ + +static void skip_whitespace(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') + (*p)++; +} + +static int parse_string(const char **p, char *out, size_t max_len) { + skip_whitespace(p); + if (**p != '"') + return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') + out[i++] = '\n'; + else if (**p == 't') + out[i++] = '\t'; + else if (**p == '"') + out[i++] = '"'; + else if (**p == '\\') + out[i++] = '\\'; + else + out[i++] = **p; + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') + return -1; + (*p)++; + return 0; +} + +static int parse_int(const char **p, int64_t *out) { + skip_whitespace(p); + int64_t val = 0; + int neg = 0; + if (**p == '-') { + neg = 1; + (*p)++; + } + if (**p < '0' || **p > '9') + return -1; + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + *out = neg ? -val : val; + return 0; +} + +static safetensor_dtype_t parse_dtype(const char *s) { + if (strcmp(s, "F32") == 0) + return DTYPE_F32; + if (strcmp(s, "F16") == 0) + return DTYPE_F16; + if (strcmp(s, "BF16") == 0) + return DTYPE_BF16; + if (strcmp(s, "I32") == 0) + return DTYPE_I32; + if (strcmp(s, "I64") == 0) + return DTYPE_I64; + if (strcmp(s, "BOOL") == 0) + return DTYPE_BOOL; + return DTYPE_UNKNOWN; +} + +static int parse_tensor_entry(const char **p, safetensor_t *t) { + skip_whitespace(p); + if (**p != '{') + return -1; + (*p)++; + + t->dtype = DTYPE_UNKNOWN; + t->ndim = 0; + t->data_offset = 0; + t->data_size = 0; + + while (**p && **p != '}') { + skip_whitespace(p); + if (**p == ',') { + (*p)++; + continue; + } + + char key[64]; + if (parse_string(p, key, sizeof(key)) != 0) + return -1; + skip_whitespace(p); + if (**p != ':') + return -1; + (*p)++; + skip_whitespace(p); + + if (strcmp(key, "dtype") == 0) { + char dtype_str[32]; + if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) + return -1; + t->dtype = parse_dtype(dtype_str); + } else if (strcmp(key, "shape") == 0) { + if (**p != '[') + return -1; + (*p)++; + t->ndim = 0; + while (**p && **p != ']') { + skip_whitespace(p); + if (**p == ',') { + (*p)++; + continue; + } + if (t->ndim >= 8) + return -1; + if (parse_int(p, &t->shape[t->ndim]) != 0) + return -1; + t->ndim++; + } + if (**p != ']') + return -1; + (*p)++; + } else if (strcmp(key, "data_offsets") == 0) { + if (**p != '[') + return -1; + (*p)++; + skip_whitespace(p); + int64_t start_i; + if (parse_int(p, &start_i) != 0) + return -1; + skip_whitespace(p); + if (**p != ',') + return -1; + (*p)++; + skip_whitespace(p); + int64_t end_i; + if (parse_int(p, &end_i) != 0) + return -1; + if (start_i < 0 || end_i < start_i) + return -1; + t->data_offset = (size_t)start_i; + t->data_size = (size_t)(end_i - start_i); + skip_whitespace(p); + if (**p != ']') + return -1; + (*p)++; + } else { + /* Skip unknown value */ + if (**p == '"') { + (*p)++; + while (**p && **p != '"') { + if (**p == '\\') + (*p)++; + if (**p) + (*p)++; + } + if (**p == '"') + (*p)++; + } else if (**p == '[') { + int depth = 1; + (*p)++; + while (**p && depth > 0) { + if (**p == '[') + depth++; + else if (**p == ']') + depth--; + (*p)++; + } + } else if (**p == '{') { + int depth = 1; + (*p)++; + while (**p && depth > 0) { + if (**p == '{') + depth++; + else if (**p == '}') + depth--; + (*p)++; + } + } else { + while (**p && **p != ',' && **p != '}') + (*p)++; + } + } + } + if (**p != '}') + return -1; + (*p)++; + return 0; +} + +static int parse_header(safetensors_file_t *sf) { + const char *p = sf->header_json; + skip_whitespace(&p); + if (*p != '{') + return -1; + p++; + + sf->num_tensors = 0; + + while (*p && *p != '}') { + skip_whitespace(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; + + char name[256]; + if (parse_string(&p, name, sizeof(name)) != 0) + return -1; + skip_whitespace(&p); + if (*p != ':') + return -1; + p++; + + if (strcmp(name, "__metadata__") == 0) { + skip_whitespace(&p); + if (*p == '{') { + int depth = 1; + p++; + while (*p && depth > 0) { + if (*p == '{') + depth++; + else if (*p == '}') + depth--; + p++; + } + } + continue; + } + + if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { + fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", + sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS); + return -1; + } + + safetensor_t *t = &sf->tensors[sf->num_tensors]; + snprintf(t->name, sizeof(t->name), "%s", name); + if (parse_tensor_entry(&p, t) != 0) + return -1; + sf->num_tensors++; + } + if (*p != '}') + return -1; + return 0; +} + +/* Single file operations */ + +safetensors_file_t *safetensors_open(const char *path) { + int fd = open(path, O_RDONLY); + if (fd < 0) + return NULL; + + struct stat st; + if (fstat(fd, &st) < 0) { + close(fd); + return NULL; + } + + size_t file_size = (size_t)st.st_size; + if (file_size < 8) { + close(fd); + return NULL; + } + + void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (data == MAP_FAILED) + return NULL; + + uint64_t header_size = 0; + memcpy(&header_size, data, 8); + if (header_size > file_size - 8) { + munmap(data, file_size); + return NULL; + } + + safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); + if (!sf) { + munmap(data, file_size); + return NULL; + } + + sf->path = strdup(path); + sf->data = data; + sf->file_size = file_size; + sf->header_size = (size_t)header_size; + + sf->header_json = malloc(header_size + 1); + if (!sf->header_json) { + safetensors_close(sf); + return NULL; + } + memcpy(sf->header_json, (char *)data + 8, header_size); + sf->header_json[header_size] = '\0'; + + if (parse_header(sf) != 0) { + safetensors_close(sf); + return NULL; + } + + return sf; +} + +void safetensors_close(safetensors_file_t *sf) { + if (!sf) + return; + if (sf->data) + munmap(sf->data, sf->file_size); + free(sf->path); + free(sf->header_json); + free(sf); +} + +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { + return (const char *)sf->data + 8 + sf->header_size + t->data_offset; +} + +/* Multi-shard operations */ + +static int cmp_shard_names(const void *a, const void *b) { + const char *sa = (const char *)a; + const char *sb = (const char *)b; + return strcmp(sa, sb); +} + +static int parse_shard_name(const char *name, int *idx, int *total) { + int i = 0, n = 0, consumed = 0; + if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) + return -1; + if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) + return -1; + *idx = i; + *total = n; + return 0; +} + +multi_safetensors_t *multi_safetensors_open(const char *model_dir) { + multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); + if (!ms) + return NULL; + + char path[4096]; + + /* Try single file first */ + snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); + safetensors_file_t *sf = safetensors_open(path); + if (sf) { + ms->shards[0] = sf; + ms->num_shards = 1; + return ms; + } + + /* Scan directory for shard files */ + DIR *dir = opendir(model_dir); + if (!dir) { + free(ms); + return NULL; + } + + struct dirent *entry; + char shard_names[SAFETENSORS_MAX_SHARDS][256]; + int seen[SAFETENSORS_MAX_SHARDS] = {0}; + int n_shards = 0; + int expected_total = 0; + + while ((entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "model-", 6) == 0 && + strstr(entry->d_name, ".safetensors") != NULL) { + int shard_idx = 0, shard_total = 0; + if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { + fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (shard_total > SAFETENSORS_MAX_SHARDS) { + fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, + SAFETENSORS_MAX_SHARDS); + closedir(dir); + free(ms); + return NULL; + } + if (expected_total == 0) { + expected_total = shard_total; + } else if (expected_total != shard_total) { + fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", + entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (seen[shard_idx - 1]) { + fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); + closedir(dir); + free(ms); + return NULL; + } + seen[shard_idx - 1] = 1; + + snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); + n_shards++; + } + } + closedir(dir); + + if (n_shards == 0) { + fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); + free(ms); + return NULL; + } + if (expected_total != n_shards) { + fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", + expected_total, n_shards, model_dir); + free(ms); + return NULL; + } + for (int i = 0; i < expected_total; i++) { + if (!seen[i]) { + fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, + expected_total, model_dir); + free(ms); + return NULL; + } + } + + /* Sort shard names to ensure consistent ordering */ + qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); + + /* Open each shard */ + for (int i = 0; i < n_shards; i++) { + snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); + ms->shards[i] = safetensors_open(path); + if (!ms->shards[i]) { + fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); + multi_safetensors_close(ms); + return NULL; + } + } + ms->num_shards = n_shards; + return ms; +} + +void multi_safetensors_close(multi_safetensors_t *ms) { + if (!ms) + return; + for (int i = 0; i < ms->num_shards; i++) { + safetensors_close(ms->shards[i]); + } + free(ms); +} + +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { + for (int s = 0; s < ms->num_shards; s++) { + safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) { + if (out_sf) + *out_sf = sf; + return &sf->tensors[i]; + } + } + } + return NULL; +} diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h new file mode 100644 index 0000000..dca68a7 --- /dev/null +++ b/liblangdet/safetensors.h @@ -0,0 +1,64 @@ +/* + * Safetensors file format reader with multi-shard support. + */ + +#ifndef SAFETENSORS_H +#define SAFETENSORS_H + +#include <stddef.h> +#include <stdint.h> + +#define SAFETENSORS_MAX_TENSORS 1024 +#define SAFETENSORS_MAX_SHARDS 8 + +typedef enum { + DTYPE_F32 = 0, + DTYPE_F16 = 1, + DTYPE_BF16 = 2, + DTYPE_I32 = 3, + DTYPE_I64 = 4, + DTYPE_BOOL = 5, + DTYPE_UNKNOWN = -1 +} safetensor_dtype_t; + +typedef struct { + char name[256]; + safetensor_dtype_t dtype; + int ndim; + int64_t shape[8]; + size_t data_offset; + size_t data_size; +} safetensor_t; + +typedef struct { + char *path; + void *data; + size_t file_size; + size_t header_size; + char *header_json; + int num_tensors; + safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; +} safetensors_file_t; + +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ +typedef struct { + safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; + int num_shards; +} multi_safetensors_t; + +/* Open a single safetensors file (memory-mapped) */ +safetensors_file_t *safetensors_open(const char *path); +void safetensors_close(safetensors_file_t *sf); + +/* Open model from directory (auto-detects single file or multi-shard) */ +multi_safetensors_t *multi_safetensors_open(const char *model_dir); +void multi_safetensors_close(multi_safetensors_t *ms); + +/* Find a tensor by name across all shards */ +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); + +/* Get raw pointer to tensor data (within mmap'd region) */ +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); + +#endif /* SAFETENSORS_H */ diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c new file mode 100644 index 0000000..6c30ba1 --- /dev/null +++ b/liblangdet/sentencepiece.c @@ -0,0 +1,905 @@ +/* + * SentencePiece Unigram tokenizer. + * + * The implementation reads the small subset of ModelProto needed at runtime: + * pieces, scores, piece types, trainer model_type, and normalizer options. The + * normalizer supports SentencePiece's precompiled charsmap format used by + * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program + * over UTF-8 byte offsets. + */ + +#include "sentencepiece.h" + +#include <errno.h> +#include <float.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +enum { + SP_MODEL_UNIGRAM = 1, + SP_PIECE_NORMAL = 1, + SP_PIECE_UNKNOWN = 2, + SP_PIECE_CONTROL = 3, + SP_PIECE_USER_DEFINED = 4, + SP_PIECE_UNUSED = 5, + SP_PIECE_BYTE = 6, +}; + +typedef struct { + const char *key; + uint16_t len; + int val; +} sp_entry_t; + +typedef struct { + int id; + int prev; + double score; +} sp_best_t; + +struct tok_spm_workspace { + char *norm; + size_t norm_len; + size_t norm_cap; + sp_best_t *best; + int best_cap; + int *ids; + int ids_cap; +}; + +static uint64_t fnv1a_bytes(const char *s, size_t n) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < n; i++) { + h ^= (unsigned char)s[i]; + h *= 1099511628211ULL; + } + return h; +} + +static int next_pow2(int x) { + int p = 1; + while (p < x && p <= (1 << 29)) + p <<= 1; + return p; +} + +static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) { + if (len <= 0 || len > UINT16_MAX) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) { + map[pos].val = val; + return 0; + } + pos = (int)((pos + 1) & (int)mask); + } + map[pos].key = key; + map[pos].len = (uint16_t)len; + map[pos].val = val; + return 0; +} + +static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { + if (!map || len <= 0) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) + return map[pos].val; + pos = (int)((pos + 1) & (int)mask); + } + return -1; +} + +static int is_tokenizable_type(int type) { + return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || + type == SP_PIECE_BYTE; +} + +static int reserve_chars(char **buf, size_t *cap, size_t need) { + if (need <= *cap) + return 0; + size_t nc = *cap ? *cap : 128; + while (nc < need) { + if (nc > SIZE_MAX / 2) + return -1; + nc *= 2; + } + char *p = (char *)realloc(*buf, nc); + if (!p) + return -1; + *buf = p; + *cap = nc; + return 0; +} + +static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) { + if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0) + return -1; + memcpy(*buf + *len, s, n); + *len += n; + (*buf)[*len] = '\0'; + return 0; +} + +static int reserve_best(tok_spm_workspace_t *ws, int need) { + if (need <= ws->best_cap) + return 0; + int nc = ws->best_cap ? ws->best_cap : 128; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->best = p; + ws->best_cap = nc; + return 0; +} + +static int reserve_ids(tok_spm_workspace_t *ws, int need) { + if (need <= ws->ids_cap) + return 0; + int nc = ws->ids_cap ? ws->ids_cap : 64; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->ids = p; + ws->ids_cap = nc; + return 0; +} + +static int utf8_len(unsigned char c) { + if (c < 0x80) + return 1; + if ((c & 0xE0) == 0xC0) + return 2; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xF8) == 0xF0) + return 4; + return 1; +} + +static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) { + if (off >= n) + return 0; + const unsigned char *p = (const unsigned char *)s + off; + int len = utf8_len(p[0]); + if (off + (size_t)len > n) { + *len_out = 1; + return 0; + } + if (len > 1) { + for (int i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) { + *len_out = 1; + return 0; + } + } + } + *len_out = (size_t)len; + return 1; +} + +static uint32_t read_le32(const unsigned char *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) { + uint64_t v = 0; + int shift = 0; + while (*p < end && shift <= 63) { + unsigned char c = *(*p)++; + v |= (uint64_t)(c & 0x7f) << shift; + if (!(c & 0x80)) { + *out = v; + return 0; + } + shift += 7; + } + return -1; +} + +static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) { + uint64_t key = 0; + if (pb_read_varint(p, end, &key) != 0) + return -1; + *field = (int)(key >> 3); + *wire = (int)(key & 7); + return *field > 0 ? 0 : -1; +} + +static int pb_read_len(const unsigned char **p, + const unsigned char *end, + const unsigned char **data, + size_t *len) { + uint64_t n = 0; + if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p)) + return -1; + *data = *p; + *len = (size_t)n; + *p += n; + return 0; +} + +static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) { + uint64_t v = 0; + const unsigned char *data = NULL; + size_t len = 0; + switch (wire) { + case 0: + return pb_read_varint(p, end, &v); + case 1: + if ((size_t)(end - *p) < 8) + return -1; + *p += 8; + return 0; + case 2: + return pb_read_len(p, end, &data, &len); + case 5: + if ((size_t)(end - *p) < 4) + return -1; + *p += 4; + return 0; + default: + return -1; + } +} + +static char *memdup0(const unsigned char *data, size_t len) { + char *s = (char *)malloc(len + 1); + if (!s) + return NULL; + memcpy(s, data, len); + s[len] = '\0'; + return s; +} + +static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + char *piece = NULL; + float score = 0.0f; + int type = SP_PIECE_NORMAL; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + goto fail; + if (field == 1 && wire == 2) { + const unsigned char *s = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX) + goto fail; + free(piece); + piece = memdup0(s, n); + if (!piece) + goto fail; + tok->piece_lens[idx] = (uint16_t)n; + } else if (field == 2 && wire == 5) { + if ((size_t)(end - p) < 4) + goto fail; + uint32_t bits = read_le32(p); + memcpy(&score, &bits, sizeof(score)); + p += 4; + } else if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX) + goto fail; + type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + goto fail; + } + } + if (!piece) + goto fail; + tok->id_to_piece[idx] = piece; + tok->scores[idx] = score; + tok->types[idx] = (uint8_t)type; + if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) + tok->max_piece_len = tok->piece_lens[idx]; + return 0; +fail: + free(piece); + return -1; +} + +static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int model_type = SP_MODEL_UNIGRAM; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX) + return -1; + model_type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (model_type != SP_MODEL_UNIGRAM) { + fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n", + model_type); + return -1; + } + (void)tok; + return 0; +} + +static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if ((field == 3 || field == 4 || field == 5) && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0) + return -1; + if (field == 3) + tok->add_dummy_prefix = v != 0; + else if (field == 4) + tok->remove_extra_whitespaces = v != 0; + else + tok->escape_whitespaces = v != 0; + } else if (field == 2 && wire == 2) { + const unsigned char *blob = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &blob, &n) != 0) + return -1; + if (n >= 4) { + uint32_t xcda_bytes = read_le32(blob); + if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) { + tok->xcda_count = xcda_bytes / 4; + tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda)); + if (!tok->xcda) + return -1; + for (size_t i = 0; i < tok->xcda_count; i++) + tok->xcda[i] = read_le32(blob + 4 + i * 4); + tok->prefix_replacements_len = n - 4 - xcda_bytes; + tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1); + if (!tok->prefix_replacements) + return -1; + memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes, + tok->prefix_replacements_len); + tok->prefix_replacements[tok->prefix_replacements_len] = '\0'; + } + } + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return 0; +} + +static int count_pieces(const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int n = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 1 && wire == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0) + return -1; + n++; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return n; +} + +static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { + int n_pieces = count_pieces(data, len); + if (n_pieces <= 0) + return -1; + tok->piece_count = n_pieces; + tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); + tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); + tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); + tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); + if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) + return -1; + + const unsigned char *p = data, *end = data + len; + int piece_idx = 0; + int saw_trainer = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (wire == 2 && field == 1) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces || + parse_piece(tok, piece_idx++, msg, msg_len) != 0) + return -1; + } else if (wire == 2 && field == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) + return -1; + saw_trainer = 1; + } else if (wire == 2 && field == 3) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0) + return -1; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (piece_idx != n_pieces || !saw_trainer) + return -1; + return 0; +} + +static int raw_piece_id(const tok_spm_t *tok, const char *piece) { + if (!tok || !piece) + return -1; + return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); +} + +static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { + if (!tok || raw_id < 0 || raw_id >= tok->piece_count) + return -1; + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { + if (raw_id == 1) + return 0; /* <s> */ + if (raw_id == 2) + return 2; /* </s> */ + if (raw_id == 0) + return 3; /* <unk> */ + return raw_id + 1; + } + return raw_id; +} + +static int file_contains_xlm_roberta(const char *model_dir, const char *name) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; + FILE *f = fopen(path, "rb"); + if (!f) + return 0; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return 0; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return 0; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return 0; + } + size_t got = fread(buf, 1, (size_t)sz, f); + fclose(f); + buf[got] = '\0'; + int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || + strstr(buf, "XLMRobertaTokenizerFast") != NULL || + strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; + free(buf); + return yes; +} + +static int detect_xlm_roberta(const char *model_dir) { + return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || + file_contains_xlm_roberta(model_dir, "config.json"); +} + +static int find_model_file(const char *model_dir, char path[1024]) { + static const char *names[] = { + "sentencepiece.bpe.model", + "tokenizer.model", + "spiece.model", + }; + for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { + int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); + if (n < 0 || n >= 1024) + return -1; + FILE *f = fopen(path, "rb"); + if (f) { + fclose(f); + return 0; + } + if (errno != ENOENT) + return -1; + } + return -1; +} + +static int load_file(const char *path, unsigned char **out, size_t *out_len) { + FILE *f = fopen(path, "rb"); + if (!f) + return -1; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + unsigned char *buf = (unsigned char *)malloc((size_t)sz); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); + free(buf); + return -1; + } + fclose(f); + *out = buf; + *out_len = (size_t)sz; + return 0; +} + +static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) { + uint32_t v = tok->xcda[idx]; + return (v >> 10) << ((v & (1u << 9)) >> 6); +} + +static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) | 0xffu); +} + +static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; } + +static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) - 1u); +} + +typedef struct { + const char *data; + size_t len; + size_t consumed; +} norm_prefix_t; + +static norm_prefix_t +normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) { + static const char repl[] = "\xef\xbf\xbd"; + if (off >= input_len) + return (norm_prefix_t){input + off, 0, 0}; + + size_t best_len = 0; + size_t best_repl = 0; + if (tok->xcda && tok->xcda_count > 0) { + uint32_t node = xcda_base(tok, 0); + for (size_t p = off; p < input_len; p++) { + unsigned char c = (unsigned char)input[p]; + if (c == 0) + break; + node ^= c; + if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c) + break; + int leaf = xcda_leaf(tok, node); + node ^= xcda_base(tok, node); + if (leaf) { + best_len = p - off + 1; + best_repl = xcda_value(tok, node); + } + } + } + if (best_len > 0 && best_repl < tok->prefix_replacements_len) { + const char *r = tok->prefix_replacements + best_repl; + size_t max = tok->prefix_replacements_len - best_repl; + size_t n = strnlen(r, max); + return (norm_prefix_t){r, n, best_len}; + } + + size_t clen = 1; + if (valid_utf8_one(input, input_len, off, &clen)) + return (norm_prefix_t){input + off, clen, clen}; + return (norm_prefix_t){repl, 3, 1}; +} + +static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) { + ws->norm_len = 0; + if (!text) + return -1; + size_t input_len = strlen(text); + const char space[] = "\xe2\x96\x81"; /* U+2581 */ + const char *space_s = tok->escape_whitespaces ? space : " "; + size_t space_n = tok->escape_whitespaces ? 3 : 1; + int processing_non_ws = 0; + int space_prepended = 0; + + for (size_t off = 0; off < input_len;) { + norm_prefix_t r = normalize_prefix(tok, text, input_len, off); + for (size_t i = 0; i < r.len; i++) { + char c = r.data[i]; + if (c != ' ') { + if (!processing_non_ws) { + processing_non_ws = 1; + if ((tok->add_dummy_prefix && !space_prepended) || + tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != + 0) + return -1; + space_prepended = 1; + } + } + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) + return -1; + } else { + processing_non_ws = 0; + if (!tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) + return -1; + } + } + } + off += r.consumed ? r.consumed : 1; + } + if (ws->norm) + ws->norm[ws->norm_len] = '\0'; + return 0; +} + +static double compute_unknown_score(const tok_spm_t *tok) { + float min_score = FLT_MAX; + for (int i = 0; i < tok->piece_count; i++) { + if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) + min_score = tok->scores[i]; + } + if (min_score == FLT_MAX) + min_score = -10.0f; + return (double)min_score - 10.0; +} + +static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { + size_t n = ws->norm_len; + if (n == 0) + return 0; + if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 || + reserve_ids(ws, (int)n + 1) != 0) + return -1; + for (size_t i = 0; i <= n; i++) + ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; + ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; + double unk_score = tok->unknown_score; + + for (size_t off = 0; off < n;) { + size_t cp_len = 1; + valid_utf8_one(ws->norm, n, off, &cp_len); + sp_best_t cur = ws->best[off]; + if (cur.score <= -DBL_MAX / 2.0) { + off += cp_len; + continue; + } + + int single_cp_found = 0; + for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { + int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, + (int)(end - off)); + if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { + if (end - off == cp_len) + single_cp_found = 1; + double sc = + cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; + } + if (end == n) + break; + size_t next_len = 1; + valid_utf8_one(ws->norm, n, end, &next_len); + end += next_len; + } + if (!single_cp_found) { + size_t end = off + cp_len; + double sc = cur.score + unk_score; + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; + } + off += cp_len; + } + + int out_n = 0; + int prev_unk = 0; + for (size_t off = n; off > 0;) { + sp_best_t b = ws->best[off]; + if (b.prev < 0 || b.prev >= (int)off) + return -1; + int is_unk = b.id == tok->unk_id; + if (!(prev_unk && is_unk)) { + if (out_n >= ws->ids_cap) + return -1; + ws->ids[out_n++] = b.id; + } + prev_unk = is_unk; + off = (size_t)b.prev; + } + for (int i = 0; i < out_n / 2; i++) { + int t = ws->ids[i]; + ws->ids[i] = ws->ids[out_n - 1 - i]; + ws->ids[out_n - 1 - i] = t; + } + return out_n; +} + +tok_spm_t *tok_spm_load(const char *model_dir) { + if (!model_dir) + return NULL; + char path[1024]; + if (find_model_file(model_dir, path) != 0) { + fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); + return NULL; + } + unsigned char *data = NULL; + size_t len = 0; + if (load_file(path, &data, &len) != 0) { + fprintf(stderr, "sentencepiece: cannot read %s\n", path); + return NULL; + } + + tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t)); + if (!tok) { + free(data); + return NULL; + } + tok->add_dummy_prefix = 1; + tok->remove_extra_whitespaces = 1; + tok->escape_whitespaces = 1; + tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; + if (parse_model(tok, data, len) != 0) { + fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); + free(data); + tok_spm_free(tok); + return NULL; + } + free(data); + + int cap = next_pow2(tok->piece_count * 2 + 1); + tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); + if (!tok->piece_map) { + tok_spm_free(tok); + return NULL; + } + tok->piece_map_cap = cap; + for (int i = 0; i < tok->piece_count; i++) { + if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], + tok->piece_lens[i], i) != 0) { + tok_spm_free(tok); + return NULL; + } + } + + tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>")); + tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>")); + tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>")); + tok->pad_id = + tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); + tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA + ? tok->piece_count + 1 + : raw_to_model_id(tok, raw_piece_id(tok, "<mask>")); + tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; + tok->unknown_score = compute_unknown_score(tok); + if (tok->unk_id < 0 || tok->max_piece_len <= 0) { + fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path); + tok_spm_free(tok); + return NULL; + } + return tok; +} + +int tok_spm_token_id(const tok_spm_t *tok, const char *token) { + if (!tok || !token) + return -1; + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { + if (!strcmp(token, "<s>")) + return 0; + if (!strcmp(token, "<pad>")) + return 1; + if (!strcmp(token, "</s>")) + return 2; + if (!strcmp(token, "<unk>")) + return 3; + if (!strcmp(token, "<mask>")) + return tok->piece_count + 1; + } + return raw_to_model_id(tok, raw_piece_id(tok, token)); +} + +static int tok_spm_encode_into(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return -1; + if (normalize_into(tok, ws, text) != 0) + return -1; + int n = segment(tok, ws); + if (n < 0) + return -1; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return 0; + if (!out_ids || out_cap < n) + return -2; + memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); + return 0; +} + +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return NULL; + int n = 0; + int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); + if (rc < 0 && rc != -2) + return NULL; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return NULL; + int *ids = (int *)malloc((size_t)n * sizeof(*ids)); + if (!ids) + return NULL; + memcpy(ids, ws->ids, (size_t)n * sizeof(*ids)); + return ids; +} + +tok_spm_workspace_t *tok_spm_workspace_new(void) { + return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t)); +} + +void tok_spm_workspace_free(tok_spm_workspace_t *ws) { + if (!ws) + return; + free(ws->norm); + free(ws->best); + free(ws->ids); + free(ws); +} + +void tok_spm_free(tok_spm_t *tok) { + if (!tok) + return; + if (tok->id_to_piece) { + for (int i = 0; i < tok->piece_count; i++) + free(tok->id_to_piece[i]); + } + free(tok->id_to_piece); + free(tok->piece_lens); + free(tok->scores); + free(tok->types); + free(tok->piece_map); + free(tok->xcda); + free(tok->prefix_replacements); + free(tok); +} diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h new file mode 100644 index 0000000..2230538 --- /dev/null +++ b/liblangdet/sentencepiece.h @@ -0,0 +1,71 @@ +/* + * SentencePiece tokenizer for XLM-R style models. + * + * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, + * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R + * and multilingual E5. It encodes text only; frontends add the model's special + * token layout, e.g. <s> ... </s> for XLM-R. + */ + +#ifndef TOK_SPM_H +#define TOK_SPM_H + +#include <stddef.h> +#include <stdint.h> + +typedef enum { + TOK_SPM_IDENTITY = 0, + TOK_SPM_XLM_ROBERTA = 1, +} tok_spm_id_map_t; + +typedef struct { + char **id_to_piece; /* [piece_count] raw SentencePiece ids */ + uint16_t *piece_lens; /* byte length for id_to_piece entries */ + float *scores; /* SentencePiece score per raw id */ + uint8_t *types; /* SentencePiece piece type per raw id */ + int piece_count; /* raw SentencePiece vocabulary size */ + int vocab_size; /* model embedding vocab size after id mapping */ + int unk_id; /* model-space unknown-token id */ + int bos_id; /* model-space beginning-token id, or -1 */ + int eos_id; /* model-space end-token id, or -1 */ + int pad_id; /* model-space pad-token id, or -1 */ + int mask_id; /* model-space mask-token id, or -1 */ + int max_piece_len; /* max normal/user/unused piece byte length */ + double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ + int add_dummy_prefix; + int remove_extra_whitespaces; + int escape_whitespaces; + tok_spm_id_map_t id_map; + + void *piece_map; /* piece bytes -> raw SentencePiece id */ + int piece_map_cap; + + uint32_t *xcda; + size_t xcda_count; + char *prefix_replacements; + size_t prefix_replacements_len; +} tok_spm_t; + +typedef struct tok_spm_workspace tok_spm_workspace_t; + +/* Load the first supported SentencePiece model found in model_dir. Returns NULL + * on missing model file, unsupported model type, malformed protobuf, or OOM. */ +tok_spm_t *tok_spm_load(const char *model_dir); + +/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a + * model-space id, or -1 when absent. */ +int tok_spm_token_id(const tok_spm_t *tok, const char *token); + +tok_spm_workspace_t *tok_spm_workspace_new(void); +void tok_spm_workspace_free(tok_spm_workspace_t *ws); + +/* Return malloc-owned model-space token IDs, reusing workspace scratch. + * Empty text returns NULL with a zero count. */ +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens); + +void tok_spm_free(tok_spm_t *tok); + +#endif /* TOK_SPM_H */ diff --git a/liblangdet/threadpool.c b/liblangdet/threadpool.c new file mode 100644 index 0000000..6cb4252 --- /dev/null +++ b/liblangdet/threadpool.c @@ -0,0 +1,218 @@ +/* + * CPU thread pool. + */ + +#if defined(__linux__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#include "threadpool.h" + +#include <pthread.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef __APPLE__ +# include <sys/sysctl.h> +#else +# ifdef __linux__ +# include <sched.h> +# endif +# include <unistd.h> +#endif + +#if defined(USE_OPENBLAS) +void openblas_set_num_threads(int num_threads); +#endif + +#define TP_MAX_THREADS 16 + +static struct { + pthread_t threads[TP_MAX_THREADS - 1]; + int tids[TP_MAX_THREADS - 1]; + int n_threads; + int shutdown; + + parallel_fn_t fn; + void *arg; + uint64_t generation; + + pthread_mutex_t mutex; + pthread_cond_t cond_work; + pthread_cond_t cond_done; + int n_done; +} tp = { + .n_threads = 1, + .shutdown = 0, + .generation = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond_work = PTHREAD_COND_INITIALIZER, + .cond_done = PTHREAD_COND_INITIALIZER, +}; + +static void *worker_loop(void *arg) { + int tid = *(int *)arg; + uint64_t my_gen = 0; + + for (;;) { + pthread_mutex_lock(&tp.mutex); + while (tp.generation == my_gen && !tp.shutdown) + pthread_cond_wait(&tp.cond_work, &tp.mutex); + if (tp.shutdown) { + pthread_mutex_unlock(&tp.mutex); + return NULL; + } + my_gen = tp.generation; + parallel_fn_t fn = tp.fn; + void *a = tp.arg; + int nt = tp.n_threads; + pthread_mutex_unlock(&tp.mutex); + + fn(tid, nt, a); + + pthread_mutex_lock(&tp.mutex); + if (++tp.n_done >= tp.n_threads - 1) + pthread_cond_signal(&tp.cond_done); + pthread_mutex_unlock(&tp.mutex); + } +} + +void tp_set_threads(int n) { + if (n < 1) + n = 1; + if (n > TP_MAX_THREADS) + n = TP_MAX_THREADS; + +#if defined(USE_OPENBLAS) + /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the + * auxiliary pool serial to avoid nested thread contention. */ + int blas_threads = n; + openblas_set_num_threads(blas_threads); + n = 1; +#endif + + /* Shutdown existing workers */ + if (tp.n_threads > 1) { + pthread_mutex_lock(&tp.mutex); + tp.shutdown = 1; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + for (int i = 0; i < tp.n_threads - 1; i++) + pthread_join(tp.threads[i], NULL); + tp.shutdown = 0; + tp.generation = 0; + } + + tp.n_threads = n; + if (n <= 1) + return; + + for (int i = 0; i < n - 1; i++) { + tp.tids[i] = i + 1; + pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); + } +} + +#ifndef __APPLE__ +/* CPU quota available to this process, in whole cores, or 0 if uncapped. + * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host + * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process + * can actually use. + * + * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: + * threads use the period budget early, then get throttled for the rest of the + * period, causing large bursty stalls. Example: a 16-thread default against a + * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. + * + * Floor instead of ceil, so a fully busy pool stays under quota and avoids + * throttling. + */ +static int cgroup_cpu_quota(void) { + /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or + * "max <period>" when unlimited. */ + FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r"); + if (f) { + char quota[32]; + long period = 0; + int got = fscanf(f, "%31s %ld", quota, &period); + fclose(f); + if (got == 2 && period > 0 && strcmp(quota, "max") != 0) { + long q = atol(quota); + if (q > 0) + return q / period < 1 ? 1 : (int)(q / period); + } + } + /* cgroup v1: separate files; quota -1 means unlimited. */ + long q = -1, period = 0; + f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r"); + if (f) { + if (fscanf(f, "%ld", &q) != 1) + q = -1; + fclose(f); + } + f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r"); + if (f) { + if (fscanf(f, "%ld", &period) != 1) + period = 0; + fclose(f); + } + if (q > 0 && period > 0) + return q / period < 1 ? 1 : (int)(q / period); + return 0; +} +#endif + +int tp_get_num_cpus(void) { +#ifdef __APPLE__ + int n = 0; + size_t len = sizeof(n); + sysctlbyname("hw.ncpu", &n, &len, NULL, 0); + return n > 0 ? n : 1; +#else + int online = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (online < 1) + online = 1; +# ifdef __linux__ + cpu_set_t set; + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + int affinity = CPU_COUNT(&set); + if (affinity >= 1 && affinity < online) + online = affinity; + } +# endif + /* Honor a cgroup CPU quota when it is tighter than the visible core count, + * so the default thread pool does not oversubscribe a CPU-limited + * container and get throttled. */ + int quota = cgroup_cpu_quota(); + if (quota >= 1 && quota < online) + return quota; + return online; +#endif +} + +int tp_num_threads(void) { return tp.n_threads; } + +/* Dispatch work to all threads; main thread is tid=0 */ +void parallel_for(parallel_fn_t fn, void *arg) { + if (tp.n_threads <= 1) { + fn(0, 1, arg); + return; + } + + pthread_mutex_lock(&tp.mutex); + tp.fn = fn; + tp.arg = arg; + tp.n_done = 0; + tp.generation++; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + + fn(0, tp.n_threads, arg); + + pthread_mutex_lock(&tp.mutex); + while (tp.n_done < tp.n_threads - 1) + pthread_cond_wait(&tp.cond_done, &tp.mutex); + pthread_mutex_unlock(&tp.mutex); +} diff --git a/liblangdet/threadpool.h b/liblangdet/threadpool.h new file mode 100644 index 0000000..f9739a1 --- /dev/null +++ b/liblangdet/threadpool.h @@ -0,0 +1,30 @@ +#ifndef TP_THREADPOOL_H +#define TP_THREADPOOL_H + +/* Persistent worker pool for CPU kernels. + * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. + * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. + * + * Kept separate from the kernels so they depend only on parallel_for and + * tp_num_threads, not pool internals. + * + * kernels.h includes this, so kernel callers need no extra include. */ + +/* Worker callback: (thread index, total threads, user arg). */ +typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); + +/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call + * before inference. */ +void tp_set_threads(int n); + +/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */ +int tp_get_num_cpus(void); + +/* Current pool size (1 == serial). Kernels gate parallelism on this. */ +int tp_num_threads(void); + +/* Run fn across the pool; the calling thread participates as tid 0. Blocks + * until every worker has finished. */ +void parallel_for(parallel_fn_t fn, void *arg); + +#endif /* TP_THREADPOOL_H */ diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c new file mode 100644 index 0000000..5e26a18 --- /dev/null +++ b/liblangdet/tokenize.c @@ -0,0 +1,85 @@ +#include "langdet.h" +#include "sentencepiece.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct langdet_tok { + tok_spm_t *spm; + tok_spm_workspace_t *workspace; + int cls_id; + int sep_id; +}; + +static void set_err(char *err, size_t errlen, const char *msg) { + if (err && errlen) + snprintf(err, errlen, "%s", msg); +} + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { + if (!model_dir) { + set_err(err, errlen, "model_dir is NULL"); + return NULL; + } + langdet_tok_t *t = calloc(1, sizeof(*t)); + if (!t) { + set_err(err, errlen, "out of memory"); + return NULL; + } + t->spm = tok_spm_load(model_dir); + if (!t->spm) { + set_err(err, errlen, "failed to load SentencePiece tokenizer"); + goto fail; + } + t->workspace = tok_spm_workspace_new(); + if (!t->workspace) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_spm_token_id(t->spm, "<s>"); + t->sep_id = tok_spm_token_id(t->spm, "</s>"); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); + goto fail; + } + return t; + +fail: + langdet_tok_free(t); + return NULL; +} + +void langdet_tok_free(langdet_tok_t *t) { + if (!t) + return; + tok_spm_workspace_free(t->workspace); + tok_spm_free(t->spm); + free(t); +} + +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { + if (n_out) + *n_out = 0; + if (!t || !text) + return NULL; + int n = 0; + int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); + if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { + free(core); + return NULL; + } + /* Empty text still produces the two special tokens. */ + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return NULL; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = t->cls_id; + ids[n + 1] = t->sep_id; + if (n_out) + *n_out = n + 2; + return ids; +} |
