diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/LICENSE | 20 | ||||
| -rw-r--r-- | libffwd/act.c | 29 | ||||
| -rw-r--r-- | libffwd/alloc.c | 65 | ||||
| -rw-r--r-- | libffwd/alloc.h | 22 | ||||
| -rw-r--r-- | libffwd/attn.c | 242 | ||||
| -rw-r--r-- | libffwd/config.c | 67 | ||||
| -rw-r--r-- | libffwd/config.h | 8 | ||||
| -rw-r--r-- | libffwd/ffwd.h | 49 | ||||
| -rw-r--r-- | libffwd/forward.c | 96 | ||||
| -rw-r--r-- | libffwd/gemm.c | 49 | ||||
| -rw-r--r-- | libffwd/generic.c | 27 | ||||
| -rw-r--r-- | libffwd/impl.h | 254 | ||||
| -rw-r--r-- | libffwd/kernels.h | 42 | ||||
| -rw-r--r-- | libffwd/model.c | 159 | ||||
| -rw-r--r-- | libffwd/model_blas.c | 68 | ||||
| -rw-r--r-- | libffwd/model_internal.h | 17 | ||||
| -rw-r--r-- | libffwd/model_types.h | 56 | ||||
| -rw-r--r-- | libffwd/norm.c | 33 | ||||
| -rw-r--r-- | libffwd/safetensors.c | 483 | ||||
| -rw-r--r-- | libffwd/safetensors.h | 64 | ||||
| -rw-r--r-- | libffwd/sentencepiece.c | 905 | ||||
| -rw-r--r-- | libffwd/sentencepiece.h | 71 | ||||
| -rw-r--r-- | libffwd/threadpool.c | 218 | ||||
| -rw-r--r-- | libffwd/threadpool.h | 30 | ||||
| -rw-r--r-- | libffwd/tokenize.c | 85 |
25 files changed, 0 insertions, 3159 deletions
diff --git a/libffwd/LICENSE b/libffwd/LICENSE deleted file mode 100644 index a00caa7..0000000 --- a/libffwd/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -MIT License - -Copyright (c) 2026 The ffwd authors - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/libffwd/act.c b/libffwd/act.c deleted file mode 100644 index 3912f2f..0000000 --- a/libffwd/act.c +++ /dev/null @@ -1,29 +0,0 @@ -#include "kernels.h" - -#include <math.h> - -void gelu_inplace(float *x, int n) { - /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ - for (int i = 0; i < n; i++) - x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); -} - -void softmax(float *x, int rows, int cols) { - for (int r = 0; r < rows; r++) { - float *row = x + r * cols; - float max_val = row[0]; - for (int c = 1; c < cols; c++) { - if (row[c] > max_val) - max_val = row[c]; - } - float sum = 0.0f; - for (int c = 0; c < cols; c++) { - row[c] = expf(row[c] - max_val); - sum += row[c]; - } - float inv_sum = 1.0f / sum; - for (int c = 0; c < cols; c++) { - row[c] *= inv_sum; - } - } -} diff --git a/libffwd/alloc.c b/libffwd/alloc.c deleted file mode 100644 index ba21fbb..0000000 --- a/libffwd/alloc.c +++ /dev/null @@ -1,65 +0,0 @@ -#include "alloc.h" - -#include <stdlib.h> -#include <stdint.h> -#include <limits.h> - -/* Sequence-capacity growth policy for the forward-pass workspaces: - * For new growth: - * - never allocate less than the minimum required size. - * - round up to the granularity so small length changes do not trigger a realloc on every call. - * - keep existing capacity if it is already larger. */ -#define FFWD_MIN_WORKSPACE_SEQ_CAP 16 -#define FFWD_WORKSPACE_SEQ_GRANULARITY 16 - -int mul_size(size_t a, size_t b, size_t *out) { - if (a != 0 && b > SIZE_MAX / a) - return -1; - *out = a * b; - return 0; -} - -int grow_cap(int current, int needed, int *out) { - if (current < 0 || needed <= 0) - return -1; - - int cap = needed; - if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP) - cap = FFWD_MIN_WORKSPACE_SEQ_CAP; - - int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY; - if (rem != 0) { - int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem; - if (cap > INT_MAX - pad) - return -1; - cap += pad; - } - - if (cap < current) - cap = current; - - *out = cap; - return 0; -} - -int realloc_floats(float **ptr, size_t count) { - size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) - return -1; - - void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) - return -1; - *ptr = (float *)p; - return 0; -} - -int realloc_floats_2d(float **ptr, int rows, int cols) { - if (rows < 0 || cols < 0) - return -1; - - size_t count; - if (mul_size((size_t)rows, (size_t)cols, &count) != 0) - return -1; - return realloc_floats(ptr, count); -} diff --git a/libffwd/alloc.h b/libffwd/alloc.h deleted file mode 100644 index d3ef81a..0000000 --- a/libffwd/alloc.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef FFWD_ALLOC_H -#define FFWD_ALLOC_H - -#include <stddef.h> - -/* Checked size arithmetic and typed allocation helpers. - * Return -1 or NULL on overflow/allocation failure. - * realloc_* leave the original pointer unchanged on failure. */ - -/* out = a * b, or -1 if the product overflows size_t. */ -int mul_size(size_t a, size_t b, size_t *out); - -/* Grow sequence capacity without shrinking. */ -int grow_cap(int current, int needed, int *out); - -/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ -int realloc_floats(float **ptr, size_t count); - -/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ -int realloc_floats_2d(float **ptr, int rows, int cols); - -#endif /* FFWD_ALLOC_H */ diff --git a/libffwd/attn.c b/libffwd/attn.c deleted file mode 100644 index ac0f4d1..0000000 --- a/libffwd/attn.c +++ /dev/null @@ -1,242 +0,0 @@ -#include "kernels.h" -#include "impl.h" -#include "threadpool.h" - -#include <math.h> - -#ifdef USE_BLAS -# ifdef __APPLE__ -# include <Accelerate/Accelerate.h> -# else -# include <cblas.h> -# endif -#endif - -static inline float dot_f32_fast(const float *a, const float *b, int n) { - return dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void vec_scale_inplace(float *dst, float scale, int n) { - vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { - vec_scale_add_impl(dst, src, correction, n); -} - -#define PACKED_ATTN_QUERY_TILE 32 -#define PACKED_ATTN_BLAS_MIN_SEQ 128 - -static void bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - - float score = dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - vec_scale_inplace(o_row, inv_sum, head_dim); - } - } -} - -#ifdef USE_BLAS -static void bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, - kv_hidden, 0.0f, scores, keys); - softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; - int head_dim; - float scale; - float *scores; - int max_seq; -} packed_gqa_attn_task_t; - -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { - bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { -#ifdef USE_BLAS - if (!offsets || batch <= 0) - return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) - return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) - return 0; - return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); -#else - (void)offsets; - (void)batch; - return 0; -#endif -} - -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); - } else { - packed_gqa_attn_worker(0, 1, &task); - } -} diff --git a/libffwd/config.c b/libffwd/config.c deleted file mode 100644 index 6343113..0000000 --- a/libffwd/config.c +++ /dev/null @@ -1,67 +0,0 @@ -#include "config.h" - -#include <yyjson.h> - -#include <limits.h> -#include <math.h> -#include <stdio.h> - -static int json_get_int(yyjson_val *obj, const char *key, int fallback) { - yyjson_val *v = yyjson_obj_get(obj, key); - if (!v) - return fallback; - if (!yyjson_is_int(v)) - return -1; - int64_t n = yyjson_get_sint(v); - return n >= 0 && n <= INT_MAX ? (int)n : -1; -} - -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); - if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "ffwd_config: model path too long\n"); - return -1; - } - - yyjson_read_err error; - yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); - if (!doc) { - fprintf(stderr, "ffwd_config: cannot read %s: %s\n", path, error.msg); - return -1; - } - yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *causal = yyjson_obj_get(root, "is_causal"); - if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || - !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || - (causal && !yyjson_is_false(causal))) { - fprintf(stderr, "ffwd_config: expected bidirectional XLM-R with GeLU activation\n"); - yyjson_doc_free(doc); - return -1; - } - - cfg->hidden_size = json_get_int(root, "hidden_size", 0); - cfg->n_layers = json_get_int(root, "num_hidden_layers", 0); - cfg->n_heads = json_get_int(root, "num_attention_heads", 0); - cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; - cfg->intermediate_size = json_get_int(root, "intermediate_size", 0); - cfg->vocab_size = json_get_int(root, "vocab_size", 0); - cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0); - cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1); - yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps"); - cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f; - int pad_id = json_get_int(root, "pad_token_id", 1); - cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; - yyjson_doc_free(doc); - - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > FFWD_MAX_LAYERS || - cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || - cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || - cfg->position_id_offset < 0 || - cfg->max_position_embeddings <= cfg->position_id_offset || - !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { - fprintf(stderr, "ffwd_config: invalid encoder dimensions or normalization in %s\n", path); - return -1; - } - return 0; -} diff --git a/libffwd/config.h b/libffwd/config.h deleted file mode 100644 index b3e2f67..0000000 --- a/libffwd/config.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef FFWD_CONFIG_H -#define FFWD_CONFIG_H - -#include "ffwd.h" - -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir); - -#endif /* FFWD_CONFIG_H */ diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h deleted file mode 100644 index 527e92f..0000000 --- a/libffwd/ffwd.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef FFWD_H -#define FFWD_H - -#include <stddef.h> - -#define FFWD_MAX_LAYERS 64 - -typedef struct { - int hidden_size; - int n_layers; - int n_heads; - int head_dim; - int intermediate_size; - int vocab_size; - int max_position_embeddings; - int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ - int type_vocab_size; - float layer_norm_eps; -} ffwd_config_t; - -typedef struct ffwd_model ffwd_model_t; -typedef struct ffwd_workspace ffwd_workspace_t; -typedef struct ffwd_tok ffwd_tok_t; - -/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ -ffwd_model_t *ffwd_model_load(const char *model_dir); -void ffwd_model_free(ffwd_model_t *model); -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model); - -/* A workspace belongs to one model and may serve only one inference call at a time. */ -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model); -void ffwd_workspace_free(ffwd_workspace_t *ws); - -/* Write the first token's final hidden state to out_embedding[hidden_size]. - * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen); -void ffwd_tok_free(ffwd_tok_t *t); - -/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. - * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out); - -#endif /* FFWD_H */ diff --git a/libffwd/forward.c b/libffwd/forward.c deleted file mode 100644 index b314642..0000000 --- a/libffwd/forward.c +++ /dev/null @@ -1,96 +0,0 @@ -#include "model_internal.h" -#include "kernels.h" - -#include <limits.h> -#include <math.h> -#include <stdio.h> -#include <string.h> - -static void add_bias_rows(float *x, const float *bias, int rows, int dim) { - for (int row = 0; row < rows; row++) { - float *dst = x + (size_t)row * dim; - for (int d = 0; d < dim; d++) - dst[d] += bias[d]; - } -} - -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { - if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) - return -1; - - const ffwd_config_t *cfg = &model->config; - const ffwd_weights_t *w = &model->weights; - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int head_dim = cfg->head_dim; - int inter = cfg->intermediate_size; - float eps = cfg->layer_norm_eps; - - if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { - fprintf(stderr, - "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings " - "%d\n", - n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); - return -1; - } - if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || - ensure_buffers(ws, cfg, n_tokens) != 0) - return -1; - int offsets[2] = {0, n_tokens}; - ensure_attention_scores(ws, offsets, 1); - - float *x = ws->x; - float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; - float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; - - for (int i = 0; i < n_tokens; i++) { - int id = token_ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i); - return -1; - } - float *row = x + (size_t)i * hidden; - memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float)); - int pos = cfg->position_id_offset + i; - add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); - add_inplace(row, w->token_type_embeddings, hidden); - } - layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps); - - float scale = 1.0f / sqrtf((float)head_dim); - - for (int layer = 0; layer < cfg->n_layers; layer++) { - const ffwd_layer_t *l = &w->layers[layer]; - - linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); - linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); - linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden); - add_bias_rows(q_buf, l->q_bias, n_tokens, hidden); - add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); - add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); - - bidirectional_gqa_attention_packed_with_scratch( - attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, - ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); - add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); - add_inplace(x, proj_out, n_tokens * hidden); - layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps); - - linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter); - add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter); - gelu_inplace(ffn_up, n_tokens * inter); - linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden); - add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden); - add_inplace(x, proj_out, n_tokens * hidden); - layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps); - } - /* The classifier consumes <s> after the encoder's final LayerNorm, without L2 normalization. */ - memcpy(out_embedding, x, (size_t)hidden * sizeof(float)); - return 0; -} diff --git a/libffwd/gemm.c b/libffwd/gemm.c deleted file mode 100644 index c750be1..0000000 --- a/libffwd/gemm.c +++ /dev/null @@ -1,49 +0,0 @@ -#include "kernels.h" - -#ifdef USE_BLAS -# ifdef __APPLE__ -# include <Accelerate/Accelerate.h> -# else -# include <cblas.h> -# endif -#endif - -void add_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) - a[i] += b[i]; -} - -void linear( - float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) { -#ifdef USE_BLAS - if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1); - } else { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, - in_dim, W, in_dim, 0.0f, y, out_dim); - } - if (b != NULL) { - for (int s = 0; s < seq_len; s++) { - for (int o = 0; o < out_dim; o++) { - y[s * out_dim + o] += b[o]; - } - } - } -#else - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * in_dim; - float *y_row = y + s * out_dim; - for (int o = 0; o < out_dim; o++) { - const float *w_row = W + o * in_dim; - float sum = (b != NULL) ? b[o] : 0.0f; - for (int i = 0; i < in_dim; i++) - sum += x_row[i] * w_row[i]; - y_row[o] = sum; - } - } -#endif -} - -void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { - linear(y, x, W, NULL, seq_len, in_dim, out_dim); -} diff --git a/libffwd/generic.c b/libffwd/generic.c deleted file mode 100644 index 729c610..0000000 --- a/libffwd/generic.c +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Architecture-generic hot kernels / non-SIMD fallback. - */ - -#include "impl.h" - -float dot_f32_generic(const float *a, const float *b, int n) { - float sum = 0.0f; - for (int i = 0; i < n; i++) - sum += a[i] * b[i]; - return sum; -} - -void vec_scale_inplace_generic(float *dst, float scale, int n) { - for (int i = 0; i < n; i++) - dst[i] *= scale; -} - -void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { - for (int i = 0; i < n; i++) - dst[i] += alpha * src[i]; -} - -void vec_scale_add_generic(float *dst, const float *src, float correction, int n) { - for (int i = 0; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -} diff --git a/libffwd/impl.h b/libffwd/impl.h deleted file mode 100644 index f216518..0000000 --- a/libffwd/impl.h +++ /dev/null @@ -1,254 +0,0 @@ -/* Inline vector operations keep attention rows in registers across reductions. */ - -#ifndef KERNELS_IMPL_H -#define KERNELS_IMPL_H - -#include <stdint.h> - -float dot_f32_generic(const float *a, const float *b, int n); -void vec_scale_inplace_generic(float *dst, float scale, int n); -void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); -void vec_scale_add_generic(float *dst, const float *src, float correction, int n); - -#ifdef __ARM_NEON - -# include <arm_neon.h> - -static inline float dot_f32_impl(const float *a, const float *b, int n) { - int i = 0; - float32x4_t acc0 = vdupq_n_f32(0.0f); - float32x4_t acc1 = vdupq_n_f32(0.0f); - for (; i + 8 <= n; i += 8) { - float32x4_t a0 = vld1q_f32(a + i); - float32x4_t b0 = vld1q_f32(b + i); - float32x4_t a1 = vld1q_f32(a + i + 4); - float32x4_t b1 = vld1q_f32(b + i + 4); - acc0 = vfmaq_f32(acc0, a0, b0); - acc1 = vfmaq_f32(acc1, a1, b1); - } - float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -} - -static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { - int i = 0; - float32x4_t s = vdupq_n_f32(scale); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); - vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); - } - for (; i < n; i++) - dst[i] *= scale; -} - -static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { - int i = 0; - float32x4_t a = vdupq_n_f32(alpha); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); - vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -} - -static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { - int i = 0; - float32x4_t c = vdupq_n_f32(correction); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); - vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -} - -#elif defined(__AVX2__) && defined(__FMA__) - -# include <immintrin.h> - -static inline float dot_f32_impl(const float *a, const float *b, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 acc0 = _mm512_setzero_ps(); - __m512 acc1 = _mm512_setzero_ps(); - __m512 acc2 = _mm512_setzero_ps(); - __m512 acc3 = _mm512_setzero_ps(); - for (; i + 64 <= n; i += 64) { - acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); - acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); - acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); - acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); - } - __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); - for (; i + 16 <= n; i += 16) { - acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -# else - int i = 0; - __m256 acc0 = _mm256_setzero_ps(); - __m256 acc1 = _mm256_setzero_ps(); - __m256 acc2 = _mm256_setzero_ps(); - __m256 acc3 = _mm256_setzero_ps(); - for (; i + 32 <= n; i += 32) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); - acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); - acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); - } - acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); - for (; i + 8 <= n; i += 8) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - } - __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); - r = _mm_hadd_ps(r, r); - r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -# endif -} - -static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 s = _mm512_set1_ps(scale); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); - _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); - _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -# else - int i = 0; - __m256 s = _mm256_set1_ps(scale); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); - _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); - _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -# endif -} - -static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 a = _mm512_set1_ps(alpha); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, - _mm512_loadu_ps(dst + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, - _mm512_loadu_ps(dst + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, - _mm512_loadu_ps(dst + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -# else - int i = 0; - __m256 a = _mm256_set1_ps(alpha); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, - _mm256_loadu_ps(dst + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, - _mm256_loadu_ps(dst + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, - _mm256_loadu_ps(dst + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -# endif -} - -static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { -# if defined(__AVX512F__) - int i = 0; - __m512 c = _mm512_set1_ps(correction); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, - _mm512_loadu_ps(src + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, - _mm512_loadu_ps(src + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, - _mm512_loadu_ps(src + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -# else - int i = 0; - __m256 c = _mm256_set1_ps(correction); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, - _mm256_loadu_ps(src + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, - _mm256_loadu_ps(src + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, - _mm256_loadu_ps(src + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -# endif -} - -#else /* no SIMD path: dispatch to the scalar references */ - -# define dot_f32_impl dot_f32_generic -# define vec_scale_inplace_impl vec_scale_inplace_generic -# define vec_axpy_inplace_impl vec_axpy_inplace_generic -# define vec_scale_add_impl vec_scale_add_generic - -#endif - -#endif /* KERNELS_IMPL_H */ diff --git a/libffwd/kernels.h b/libffwd/kernels.h deleted file mode 100644 index 1025602..0000000 --- a/libffwd/kernels.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef KERNELS_H -#define KERNELS_H - -#include <stddef.h> -#include <stdint.h> - -void add_inplace(float *a, const float *b, int n); - -/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */ -void linear( - float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); - -/* Mean and population variance are reduced over the hidden axis per row. */ -void layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps); - -void softmax(float *x, int rows, int cols); -void gelu_inplace(float *x, int n); - -/* Attention is block-diagonal: each token attends only within its offsets span. - * Missing scratch uses the online-softmax path instead of tiled BLAS. */ -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); - -#endif /* KERNELS_H */ diff --git a/libffwd/model.c b/libffwd/model.c deleted file mode 100644 index 81073f3..0000000 --- a/libffwd/model.c +++ /dev/null @@ -1,159 +0,0 @@ -#include "model_internal.h" -#include "config.h" - -#include <stdio.h> -#include <stdlib.h> - -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim) { - if (t->dtype != DTYPE_F32) { - fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, - "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "ffwd: shape too large or empty for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - size_t bytes; - if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { - fprintf(stderr, "ffwd: invalid data size for %s\n", name); - return 0; - } - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static const float *load_weight(multi_safetensors_t *ms, const char *name, - const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "ffwd: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) - return NULL; - return safetensors_data(sf, t); -} - -static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { - const ffwd_config_t *cfg = &model->config; - ffwd_weights_t *w = &model->weights; - char name[256]; - int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - - w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", - (int64_t[]){cfg->vocab_size, hidden}, 2); - w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", - (int64_t[]){cfg->max_position_embeddings, hidden}, 2); - w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", - (int64_t[]){cfg->type_vocab_size, hidden}, 2); - w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); - w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); - if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || - !w->ffwd_ln_w || !w->ffwd_ln_b) - return -1; - - w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); - if (!w->layers) - return -1; - - for (int i = 0; i < cfg->n_layers; i++) { - ffwd_layer_t *l = &w->layers[i]; - -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 2); \ - if (!l->field) \ - return -1; \ - } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ - } while (0) - - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); - -#undef BERT_W -#undef BERT_V - } - return 0; -} - -ffwd_model_t *ffwd_model_load(const char *model_dir) { - if (!model_dir) - return NULL; - ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t)); - if (!model) - return NULL; - if (ffwd_config_parse(&model->config, model_dir) != 0) - goto fail; - model->safetensors = multi_safetensors_open(model_dir); - if (!model->safetensors) { - fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir); - goto fail; - } - if (load_bert_weights(model, model->safetensors) != 0) - goto fail; - return model; - -fail: - ffwd_model_free(model); - return NULL; -} - -void ffwd_model_free(ffwd_model_t *model) { - if (!model) - return; - multi_safetensors_close(model->safetensors); - free(model->weights.layers); - free(model); -} - -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) { - return model ? &model->config : NULL; -} diff --git a/libffwd/model_blas.c b/libffwd/model_blas.c deleted file mode 100644 index 409ba66..0000000 --- a/libffwd/model_blas.c +++ /dev/null @@ -1,68 +0,0 @@ -#include "model_internal.h" -#include "kernels.h" - -#include <stdlib.h> - -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) { - if (!model) - return NULL; - ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t)); - if (!ws) - return NULL; - ws->model = model; - return ws; -} - -void ffwd_workspace_free(ffwd_workspace_t *ws) { - if (!ws) - return; - free(ws->x); - free(ws->q); - free(ws->k); - free(ws->v); - free(ws->attn_out); - free(ws->proj_out); - free(ws->ffn_up); - free(ws->attn_scores); - free(ws); -} - -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) { - if (seq <= ws->buf_seq_cap) - return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) - return -1; - -#define R(ptr, n) \ - do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ - return -1; \ - } while (0) - - R(ws->x, c->hidden_size); - R(ws->q, c->hidden_size); - R(ws->k, c->hidden_size); - R(ws->v, c->hidden_size); - R(ws->attn_out, c->hidden_size); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) - return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) - return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} diff --git a/libffwd/model_internal.h b/libffwd/model_internal.h deleted file mode 100644 index aeea52e..0000000 --- a/libffwd/model_internal.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef FFWD_MODEL_INTERNAL_H -#define FFWD_MODEL_INTERNAL_H - -#include "model_types.h" -#include "alloc.h" - -/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */ -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim); - -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int seq); -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); - -#endif /* FFWD_MODEL_INTERNAL_H */ diff --git a/libffwd/model_types.h b/libffwd/model_types.h deleted file mode 100644 index 447b653..0000000 --- a/libffwd/model_types.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef FFWD_MODEL_TYPES_H -#define FFWD_MODEL_TYPES_H - -#include "ffwd.h" -#include "safetensors.h" - -typedef struct { - const float *wq; - const float *wk; - const float *wv; - const float *wo; - const float *q_bias; - const float *k_bias; - const float *v_bias; - const float *o_bias; - const float *input_norm; - const float *attn_ln_bias; - const float *up_proj; - const float *ffn_inter_bias; - const float *down_proj; - const float *ffn_out_bias; - const float *post_attn_norm; - const float *ffn_ln_bias; -} ffwd_layer_t; - -typedef struct { - const float *embed_tokens; - ffwd_layer_t *layers; - const float *position_embeddings; - const float *token_type_embeddings; - const float *ffwd_ln_w; - const float *ffwd_ln_b; -} ffwd_weights_t; - -struct ffwd_model { - ffwd_config_t config; - ffwd_weights_t weights; - /* All weight pointers borrow this mapping until model teardown. */ - multi_safetensors_t *safetensors; -}; - -struct ffwd_workspace { - const ffwd_model_t *model; - int buf_seq_cap; - float *x; - float *q; - float *k; - float *v; - float *attn_out; - float *proj_out; - float *ffn_up; - float *attn_scores; - size_t attn_scores_bytes; -}; - -#endif /* FFWD_MODEL_TYPES_H */ diff --git a/libffwd/norm.c b/libffwd/norm.c deleted file mode 100644 index 59e8bdb..0000000 --- a/libffwd/norm.c +++ /dev/null @@ -1,33 +0,0 @@ -#include "kernels.h" - -#include <math.h> - -/* Safe in place: each row's mean and variance are reduced before it is rewritten. */ -void layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - float inv_h = 1.0f / (float)hidden; - for (int s = 0; s < seq_len; s++) { - const float *row = x + (size_t)s * hidden; - float *dst = out + (size_t)s * hidden; - float mean = 0.0f; - for (int d = 0; d < hidden; d++) - mean += row[d]; - mean *= inv_h; - float var = 0.0f; - for (int d = 0; d < hidden; d++) { - float diff = row[d] - mean; - var += diff * diff; - } - var *= inv_h; - float inv_std = 1.0f / sqrtf(var + eps); - for (int d = 0; d < hidden; d++) - dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; - } -} diff --git a/libffwd/safetensors.c b/libffwd/safetensors.c deleted file mode 100644 index cfba76e..0000000 --- a/libffwd/safetensors.c +++ /dev/null @@ -1,483 +0,0 @@ -#include "safetensors.h" - -#include <dirent.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <unistd.h> - -/* Minimal JSON parser for safetensors header */ -/* TODO: consider replacing with yyjson */ - -static void skip_whitespace(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') - (*p)++; -} - -static int parse_string(const char **p, char *out, size_t max_len) { - skip_whitespace(p); - if (**p != '"') - return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') - out[i++] = '\n'; - else if (**p == 't') - out[i++] = '\t'; - else if (**p == '"') - out[i++] = '"'; - else if (**p == '\\') - out[i++] = '\\'; - else - out[i++] = **p; - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') - return -1; - (*p)++; - return 0; -} - -static int parse_int(const char **p, int64_t *out) { - skip_whitespace(p); - int64_t val = 0; - int neg = 0; - if (**p == '-') { - neg = 1; - (*p)++; - } - if (**p < '0' || **p > '9') - return -1; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - *out = neg ? -val : val; - return 0; -} - -static safetensor_dtype_t parse_dtype(const char *s) { - if (strcmp(s, "F32") == 0) - return DTYPE_F32; - if (strcmp(s, "F16") == 0) - return DTYPE_F16; - if (strcmp(s, "BF16") == 0) - return DTYPE_BF16; - if (strcmp(s, "I32") == 0) - return DTYPE_I32; - if (strcmp(s, "I64") == 0) - return DTYPE_I64; - if (strcmp(s, "BOOL") == 0) - return DTYPE_BOOL; - return DTYPE_UNKNOWN; -} - -static int parse_tensor_entry(const char **p, safetensor_t *t) { - skip_whitespace(p); - if (**p != '{') - return -1; - (*p)++; - - t->dtype = DTYPE_UNKNOWN; - t->ndim = 0; - t->data_offset = 0; - t->data_size = 0; - - while (**p && **p != '}') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - - char key[64]; - if (parse_string(p, key, sizeof(key)) != 0) - return -1; - skip_whitespace(p); - if (**p != ':') - return -1; - (*p)++; - skip_whitespace(p); - - if (strcmp(key, "dtype") == 0) { - char dtype_str[32]; - if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) - return -1; - t->dtype = parse_dtype(dtype_str); - } else if (strcmp(key, "shape") == 0) { - if (**p != '[') - return -1; - (*p)++; - t->ndim = 0; - while (**p && **p != ']') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - if (t->ndim >= 8) - return -1; - if (parse_int(p, &t->shape[t->ndim]) != 0) - return -1; - t->ndim++; - } - if (**p != ']') - return -1; - (*p)++; - } else if (strcmp(key, "data_offsets") == 0) { - if (**p != '[') - return -1; - (*p)++; - skip_whitespace(p); - int64_t start_i; - if (parse_int(p, &start_i) != 0) - return -1; - skip_whitespace(p); - if (**p != ',') - return -1; - (*p)++; - skip_whitespace(p); - int64_t end_i; - if (parse_int(p, &end_i) != 0) - return -1; - if (start_i < 0 || end_i < start_i) - return -1; - t->data_offset = (size_t)start_i; - t->data_size = (size_t)(end_i - start_i); - skip_whitespace(p); - if (**p != ']') - return -1; - (*p)++; - } else { - /* Skip unknown value */ - if (**p == '"') { - (*p)++; - while (**p && **p != '"') { - if (**p == '\\') - (*p)++; - if (**p) - (*p)++; - } - if (**p == '"') - (*p)++; - } else if (**p == '[') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '[') - depth++; - else if (**p == ']') - depth--; - (*p)++; - } - } else if (**p == '{') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '{') - depth++; - else if (**p == '}') - depth--; - (*p)++; - } - } else { - while (**p && **p != ',' && **p != '}') - (*p)++; - } - } - } - if (**p != '}') - return -1; - (*p)++; - return 0; -} - -static int parse_header(safetensors_file_t *sf) { - const char *p = sf->header_json; - skip_whitespace(&p); - if (*p != '{') - return -1; - p++; - - sf->num_tensors = 0; - - while (*p && *p != '}') { - skip_whitespace(&p); - if (*p == ',') { - p++; - continue; - } - if (*p == '}') - break; - - char name[256]; - if (parse_string(&p, name, sizeof(name)) != 0) - return -1; - skip_whitespace(&p); - if (*p != ':') - return -1; - p++; - - if (strcmp(name, "__metadata__") == 0) { - skip_whitespace(&p); - if (*p == '{') { - int depth = 1; - p++; - while (*p && depth > 0) { - if (*p == '{') - depth++; - else if (*p == '}') - depth--; - p++; - } - } - continue; - } - - if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { - fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", - sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS); - return -1; - } - - safetensor_t *t = &sf->tensors[sf->num_tensors]; - snprintf(t->name, sizeof(t->name), "%s", name); - if (parse_tensor_entry(&p, t) != 0) - return -1; - sf->num_tensors++; - } - if (*p != '}') - return -1; - return 0; -} - -/* Single file operations */ - -safetensors_file_t *safetensors_open(const char *path) { - int fd = open(path, O_RDONLY); - if (fd < 0) - return NULL; - - struct stat st; - if (fstat(fd, &st) < 0) { - close(fd); - return NULL; - } - - size_t file_size = (size_t)st.st_size; - if (file_size < 8) { - close(fd); - return NULL; - } - - void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - if (data == MAP_FAILED) - return NULL; - - uint64_t header_size = 0; - memcpy(&header_size, data, 8); - if (header_size > file_size - 8) { - munmap(data, file_size); - return NULL; - } - - safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); - if (!sf) { - munmap(data, file_size); - return NULL; - } - - sf->path = strdup(path); - sf->data = data; - sf->file_size = file_size; - sf->header_size = (size_t)header_size; - - sf->header_json = malloc(header_size + 1); - if (!sf->header_json) { - safetensors_close(sf); - return NULL; - } - memcpy(sf->header_json, (char *)data + 8, header_size); - sf->header_json[header_size] = '\0'; - - if (parse_header(sf) != 0) { - safetensors_close(sf); - return NULL; - } - - return sf; -} - -void safetensors_close(safetensors_file_t *sf) { - if (!sf) - return; - if (sf->data) - munmap(sf->data, sf->file_size); - free(sf->path); - free(sf->header_json); - free(sf); -} - -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { - return (const char *)sf->data + 8 + sf->header_size + t->data_offset; -} - -/* Multi-shard operations */ - -static int cmp_shard_names(const void *a, const void *b) { - const char *sa = (const char *)a; - const char *sb = (const char *)b; - return strcmp(sa, sb); -} - -static int parse_shard_name(const char *name, int *idx, int *total) { - int i = 0, n = 0, consumed = 0; - if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) - return -1; - if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) - return -1; - *idx = i; - *total = n; - return 0; -} - -multi_safetensors_t *multi_safetensors_open(const char *model_dir) { - multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); - if (!ms) - return NULL; - - char path[4096]; - - /* Try single file first */ - snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); - safetensors_file_t *sf = safetensors_open(path); - if (sf) { - ms->shards[0] = sf; - ms->num_shards = 1; - return ms; - } - - /* Scan directory for shard files */ - DIR *dir = opendir(model_dir); - if (!dir) { - free(ms); - return NULL; - } - - struct dirent *entry; - char shard_names[SAFETENSORS_MAX_SHARDS][256]; - int seen[SAFETENSORS_MAX_SHARDS] = {0}; - int n_shards = 0; - int expected_total = 0; - - while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "model-", 6) == 0 && - strstr(entry->d_name, ".safetensors") != NULL) { - int shard_idx = 0, shard_total = 0; - if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { - fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (shard_total > SAFETENSORS_MAX_SHARDS) { - fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, - SAFETENSORS_MAX_SHARDS); - closedir(dir); - free(ms); - return NULL; - } - if (expected_total == 0) { - expected_total = shard_total; - } else if (expected_total != shard_total) { - fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", - entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (seen[shard_idx - 1]) { - fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); - closedir(dir); - free(ms); - return NULL; - } - seen[shard_idx - 1] = 1; - - snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); - n_shards++; - } - } - closedir(dir); - - if (n_shards == 0) { - fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); - free(ms); - return NULL; - } - if (expected_total != n_shards) { - fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", - expected_total, n_shards, model_dir); - free(ms); - return NULL; - } - for (int i = 0; i < expected_total; i++) { - if (!seen[i]) { - fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, - expected_total, model_dir); - free(ms); - return NULL; - } - } - - /* Sort shard names to ensure consistent ordering */ - qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); - - /* Open each shard */ - for (int i = 0; i < n_shards; i++) { - snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); - ms->shards[i] = safetensors_open(path); - if (!ms->shards[i]) { - fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); - multi_safetensors_close(ms); - return NULL; - } - } - ms->num_shards = n_shards; - return ms; -} - -void multi_safetensors_close(multi_safetensors_t *ms) { - if (!ms) - return; - for (int i = 0; i < ms->num_shards; i++) { - safetensors_close(ms->shards[i]); - } - free(ms); -} - -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { - for (int s = 0; s < ms->num_shards; s++) { - safetensors_file_t *sf = ms->shards[s]; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) { - if (out_sf) - *out_sf = sf; - return &sf->tensors[i]; - } - } - } - return NULL; -} diff --git a/libffwd/safetensors.h b/libffwd/safetensors.h deleted file mode 100644 index dca68a7..0000000 --- a/libffwd/safetensors.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Safetensors file format reader with multi-shard support. - */ - -#ifndef SAFETENSORS_H -#define SAFETENSORS_H - -#include <stddef.h> -#include <stdint.h> - -#define SAFETENSORS_MAX_TENSORS 1024 -#define SAFETENSORS_MAX_SHARDS 8 - -typedef enum { - DTYPE_F32 = 0, - DTYPE_F16 = 1, - DTYPE_BF16 = 2, - DTYPE_I32 = 3, - DTYPE_I64 = 4, - DTYPE_BOOL = 5, - DTYPE_UNKNOWN = -1 -} safetensor_dtype_t; - -typedef struct { - char name[256]; - safetensor_dtype_t dtype; - int ndim; - int64_t shape[8]; - size_t data_offset; - size_t data_size; -} safetensor_t; - -typedef struct { - char *path; - void *data; - size_t file_size; - size_t header_size; - char *header_json; - int num_tensors; - safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; -} safetensors_file_t; - -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ -typedef struct { - safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; - int num_shards; -} multi_safetensors_t; - -/* Open a single safetensors file (memory-mapped) */ -safetensors_file_t *safetensors_open(const char *path); -void safetensors_close(safetensors_file_t *sf); - -/* Open model from directory (auto-detects single file or multi-shard) */ -multi_safetensors_t *multi_safetensors_open(const char *model_dir); -void multi_safetensors_close(multi_safetensors_t *ms); - -/* Find a tensor by name across all shards */ -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); - -/* Get raw pointer to tensor data (within mmap'd region) */ -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); - -#endif /* SAFETENSORS_H */ diff --git a/libffwd/sentencepiece.c b/libffwd/sentencepiece.c deleted file mode 100644 index 6c30ba1..0000000 --- a/libffwd/sentencepiece.c +++ /dev/null @@ -1,905 +0,0 @@ -/* - * SentencePiece Unigram tokenizer. - * - * The implementation reads the small subset of ModelProto needed at runtime: - * pieces, scores, piece types, trainer model_type, and normalizer options. The - * normalizer supports SentencePiece's precompiled charsmap format used by - * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program - * over UTF-8 byte offsets. - */ - -#include "sentencepiece.h" - -#include <errno.h> -#include <float.h> -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -enum { - SP_MODEL_UNIGRAM = 1, - SP_PIECE_NORMAL = 1, - SP_PIECE_UNKNOWN = 2, - SP_PIECE_CONTROL = 3, - SP_PIECE_USER_DEFINED = 4, - SP_PIECE_UNUSED = 5, - SP_PIECE_BYTE = 6, -}; - -typedef struct { - const char *key; - uint16_t len; - int val; -} sp_entry_t; - -typedef struct { - int id; - int prev; - double score; -} sp_best_t; - -struct tok_spm_workspace { - char *norm; - size_t norm_len; - size_t norm_cap; - sp_best_t *best; - int best_cap; - int *ids; - int ids_cap; -}; - -static uint64_t fnv1a_bytes(const char *s, size_t n) { - uint64_t h = 1469598103934665603ULL; - for (size_t i = 0; i < n; i++) { - h ^= (unsigned char)s[i]; - h *= 1099511628211ULL; - } - return h; -} - -static int next_pow2(int x) { - int p = 1; - while (p < x && p <= (1 << 29)) - p <<= 1; - return p; -} - -static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) { - if (len <= 0 || len > UINT16_MAX) - return -1; - uint64_t mask = (uint64_t)cap - 1; - int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); - while (map[pos].key) { - if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) { - map[pos].val = val; - return 0; - } - pos = (int)((pos + 1) & (int)mask); - } - map[pos].key = key; - map[pos].len = (uint16_t)len; - map[pos].val = val; - return 0; -} - -static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { - if (!map || len <= 0) - return -1; - uint64_t mask = (uint64_t)cap - 1; - int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); - while (map[pos].key) { - if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) - return map[pos].val; - pos = (int)((pos + 1) & (int)mask); - } - return -1; -} - -static int is_tokenizable_type(int type) { - return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || - type == SP_PIECE_BYTE; -} - -static int reserve_chars(char **buf, size_t *cap, size_t need) { - if (need <= *cap) - return 0; - size_t nc = *cap ? *cap : 128; - while (nc < need) { - if (nc > SIZE_MAX / 2) - return -1; - nc *= 2; - } - char *p = (char *)realloc(*buf, nc); - if (!p) - return -1; - *buf = p; - *cap = nc; - return 0; -} - -static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) { - if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0) - return -1; - memcpy(*buf + *len, s, n); - *len += n; - (*buf)[*len] = '\0'; - return 0; -} - -static int reserve_best(tok_spm_workspace_t *ws, int need) { - if (need <= ws->best_cap) - return 0; - int nc = ws->best_cap ? ws->best_cap : 128; - while (nc < need) { - if (nc > INT_MAX / 2) - return -1; - nc *= 2; - } - sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p)); - if (!p) - return -1; - ws->best = p; - ws->best_cap = nc; - return 0; -} - -static int reserve_ids(tok_spm_workspace_t *ws, int need) { - if (need <= ws->ids_cap) - return 0; - int nc = ws->ids_cap ? ws->ids_cap : 64; - while (nc < need) { - if (nc > INT_MAX / 2) - return -1; - nc *= 2; - } - int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p)); - if (!p) - return -1; - ws->ids = p; - ws->ids_cap = nc; - return 0; -} - -static int utf8_len(unsigned char c) { - if (c < 0x80) - return 1; - if ((c & 0xE0) == 0xC0) - return 2; - if ((c & 0xF0) == 0xE0) - return 3; - if ((c & 0xF8) == 0xF0) - return 4; - return 1; -} - -static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) { - if (off >= n) - return 0; - const unsigned char *p = (const unsigned char *)s + off; - int len = utf8_len(p[0]); - if (off + (size_t)len > n) { - *len_out = 1; - return 0; - } - if (len > 1) { - for (int i = 1; i < len; i++) { - if ((p[i] & 0xC0) != 0x80) { - *len_out = 1; - return 0; - } - } - } - *len_out = (size_t)len; - return 1; -} - -static uint32_t read_le32(const unsigned char *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) { - uint64_t v = 0; - int shift = 0; - while (*p < end && shift <= 63) { - unsigned char c = *(*p)++; - v |= (uint64_t)(c & 0x7f) << shift; - if (!(c & 0x80)) { - *out = v; - return 0; - } - shift += 7; - } - return -1; -} - -static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) { - uint64_t key = 0; - if (pb_read_varint(p, end, &key) != 0) - return -1; - *field = (int)(key >> 3); - *wire = (int)(key & 7); - return *field > 0 ? 0 : -1; -} - -static int pb_read_len(const unsigned char **p, - const unsigned char *end, - const unsigned char **data, - size_t *len) { - uint64_t n = 0; - if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p)) - return -1; - *data = *p; - *len = (size_t)n; - *p += n; - return 0; -} - -static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) { - uint64_t v = 0; - const unsigned char *data = NULL; - size_t len = 0; - switch (wire) { - case 0: - return pb_read_varint(p, end, &v); - case 1: - if ((size_t)(end - *p) < 8) - return -1; - *p += 8; - return 0; - case 2: - return pb_read_len(p, end, &data, &len); - case 5: - if ((size_t)(end - *p) < 4) - return -1; - *p += 4; - return 0; - default: - return -1; - } -} - -static char *memdup0(const unsigned char *data, size_t len) { - char *s = (char *)malloc(len + 1); - if (!s) - return NULL; - memcpy(s, data, len); - s[len] = '\0'; - return s; -} - -static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - char *piece = NULL; - float score = 0.0f; - int type = SP_PIECE_NORMAL; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - goto fail; - if (field == 1 && wire == 2) { - const unsigned char *s = NULL; - size_t n = 0; - if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX) - goto fail; - free(piece); - piece = memdup0(s, n); - if (!piece) - goto fail; - tok->piece_lens[idx] = (uint16_t)n; - } else if (field == 2 && wire == 5) { - if ((size_t)(end - p) < 4) - goto fail; - uint32_t bits = read_le32(p); - memcpy(&score, &bits, sizeof(score)); - p += 4; - } else if (field == 3 && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX) - goto fail; - type = (int)v; - } else if (pb_skip(&p, end, wire) != 0) { - goto fail; - } - } - if (!piece) - goto fail; - tok->id_to_piece[idx] = piece; - tok->scores[idx] = score; - tok->types[idx] = (uint8_t)type; - if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) - tok->max_piece_len = tok->piece_lens[idx]; - return 0; -fail: - free(piece); - return -1; -} - -static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - int model_type = SP_MODEL_UNIGRAM; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (field == 3 && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX) - return -1; - model_type = (int)v; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - if (model_type != SP_MODEL_UNIGRAM) { - fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n", - model_type); - return -1; - } - (void)tok; - return 0; -} - -static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if ((field == 3 || field == 4 || field == 5) && wire == 0) { - uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0) - return -1; - if (field == 3) - tok->add_dummy_prefix = v != 0; - else if (field == 4) - tok->remove_extra_whitespaces = v != 0; - else - tok->escape_whitespaces = v != 0; - } else if (field == 2 && wire == 2) { - const unsigned char *blob = NULL; - size_t n = 0; - if (pb_read_len(&p, end, &blob, &n) != 0) - return -1; - if (n >= 4) { - uint32_t xcda_bytes = read_le32(blob); - if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) { - tok->xcda_count = xcda_bytes / 4; - tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda)); - if (!tok->xcda) - return -1; - for (size_t i = 0; i < tok->xcda_count; i++) - tok->xcda[i] = read_le32(blob + 4 + i * 4); - tok->prefix_replacements_len = n - 4 - xcda_bytes; - tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1); - if (!tok->prefix_replacements) - return -1; - memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes, - tok->prefix_replacements_len); - tok->prefix_replacements[tok->prefix_replacements_len] = '\0'; - } - } - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - return 0; -} - -static int count_pieces(const unsigned char *data, size_t len) { - const unsigned char *p = data, *end = data + len; - int n = 0; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (field == 1 && wire == 2) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0) - return -1; - n++; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - return n; -} - -static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { - int n_pieces = count_pieces(data, len); - if (n_pieces <= 0) - return -1; - tok->piece_count = n_pieces; - tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); - tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); - tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); - tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); - if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) - return -1; - - const unsigned char *p = data, *end = data + len; - int piece_idx = 0; - int saw_trainer = 0; - while (p < end) { - int field = 0, wire = 0; - if (pb_read_key(&p, end, &field, &wire) != 0) - return -1; - if (wire == 2 && field == 1) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces || - parse_piece(tok, piece_idx++, msg, msg_len) != 0) - return -1; - } else if (wire == 2 && field == 2) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) - return -1; - saw_trainer = 1; - } else if (wire == 2 && field == 3) { - const unsigned char *msg = NULL; - size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0) - return -1; - } else if (pb_skip(&p, end, wire) != 0) { - return -1; - } - } - if (piece_idx != n_pieces || !saw_trainer) - return -1; - return 0; -} - -static int raw_piece_id(const tok_spm_t *tok, const char *piece) { - if (!tok || !piece) - return -1; - return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); -} - -static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { - if (!tok || raw_id < 0 || raw_id >= tok->piece_count) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (raw_id == 1) - return 0; /* <s> */ - if (raw_id == 2) - return 2; /* </s> */ - if (raw_id == 0) - return 3; /* <unk> */ - return raw_id + 1; - } - return raw_id; -} - -static int file_contains_xlm_roberta(const char *model_dir, const char *name) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); - if (n < 0 || (size_t)n >= sizeof(path)) - return 0; - FILE *f = fopen(path, "rb"); - if (!f) - return 0; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return 0; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return 0; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf) { - fclose(f); - return 0; - } - size_t got = fread(buf, 1, (size_t)sz, f); - fclose(f); - buf[got] = '\0'; - int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || - strstr(buf, "XLMRobertaTokenizerFast") != NULL || - strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; - free(buf); - return yes; -} - -static int detect_xlm_roberta(const char *model_dir) { - return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || - file_contains_xlm_roberta(model_dir, "config.json"); -} - -static int find_model_file(const char *model_dir, char path[1024]) { - static const char *names[] = { - "sentencepiece.bpe.model", - "tokenizer.model", - "spiece.model", - }; - for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { - int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); - if (n < 0 || n >= 1024) - return -1; - FILE *f = fopen(path, "rb"); - if (f) { - fclose(f); - return 0; - } - if (errno != ENOENT) - return -1; - } - return -1; -} - -static int load_file(const char *path, unsigned char **out, size_t *out_len) { - FILE *f = fopen(path, "rb"); - if (!f) - return -1; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - unsigned char *buf = (unsigned char *)malloc((size_t)sz); - if (!buf) { - fclose(f); - return -1; - } - if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); - free(buf); - return -1; - } - fclose(f); - *out = buf; - *out_len = (size_t)sz; - return 0; -} - -static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) { - uint32_t v = tok->xcda[idx]; - return (v >> 10) << ((v & (1u << 9)) >> 6); -} - -static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) { - return tok->xcda[idx] & ((1u << 31) | 0xffu); -} - -static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; } - -static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) { - return tok->xcda[idx] & ((1u << 31) - 1u); -} - -typedef struct { - const char *data; - size_t len; - size_t consumed; -} norm_prefix_t; - -static norm_prefix_t -normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) { - static const char repl[] = "\xef\xbf\xbd"; - if (off >= input_len) - return (norm_prefix_t){input + off, 0, 0}; - - size_t best_len = 0; - size_t best_repl = 0; - if (tok->xcda && tok->xcda_count > 0) { - uint32_t node = xcda_base(tok, 0); - for (size_t p = off; p < input_len; p++) { - unsigned char c = (unsigned char)input[p]; - if (c == 0) - break; - node ^= c; - if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c) - break; - int leaf = xcda_leaf(tok, node); - node ^= xcda_base(tok, node); - if (leaf) { - best_len = p - off + 1; - best_repl = xcda_value(tok, node); - } - } - } - if (best_len > 0 && best_repl < tok->prefix_replacements_len) { - const char *r = tok->prefix_replacements + best_repl; - size_t max = tok->prefix_replacements_len - best_repl; - size_t n = strnlen(r, max); - return (norm_prefix_t){r, n, best_len}; - } - - size_t clen = 1; - if (valid_utf8_one(input, input_len, off, &clen)) - return (norm_prefix_t){input + off, clen, clen}; - return (norm_prefix_t){repl, 3, 1}; -} - -static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) { - ws->norm_len = 0; - if (!text) - return -1; - size_t input_len = strlen(text); - const char space[] = "\xe2\x96\x81"; /* U+2581 */ - const char *space_s = tok->escape_whitespaces ? space : " "; - size_t space_n = tok->escape_whitespaces ? 3 : 1; - int processing_non_ws = 0; - int space_prepended = 0; - - for (size_t off = 0; off < input_len;) { - norm_prefix_t r = normalize_prefix(tok, text, input_len, off); - for (size_t i = 0; i < r.len; i++) { - char c = r.data[i]; - if (c != ' ') { - if (!processing_non_ws) { - processing_non_ws = 1; - if ((tok->add_dummy_prefix && !space_prepended) || - tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != - 0) - return -1; - space_prepended = 1; - } - } - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) - return -1; - } else { - processing_non_ws = 0; - if (!tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) - return -1; - } - } - } - off += r.consumed ? r.consumed : 1; - } - if (ws->norm) - ws->norm[ws->norm_len] = '\0'; - return 0; -} - -static double compute_unknown_score(const tok_spm_t *tok) { - float min_score = FLT_MAX; - for (int i = 0; i < tok->piece_count; i++) { - if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) - min_score = tok->scores[i]; - } - if (min_score == FLT_MAX) - min_score = -10.0f; - return (double)min_score - 10.0; -} - -static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { - size_t n = ws->norm_len; - if (n == 0) - return 0; - if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 || - reserve_ids(ws, (int)n + 1) != 0) - return -1; - for (size_t i = 0; i <= n; i++) - ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; - ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; - double unk_score = tok->unknown_score; - - for (size_t off = 0; off < n;) { - size_t cp_len = 1; - valid_utf8_one(ws->norm, n, off, &cp_len); - sp_best_t cur = ws->best[off]; - if (cur.score <= -DBL_MAX / 2.0) { - off += cp_len; - continue; - } - - int single_cp_found = 0; - for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { - int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, - (int)(end - off)); - if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { - if (end - off == cp_len) - single_cp_found = 1; - double sc = - cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); - if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; - } - if (end == n) - break; - size_t next_len = 1; - valid_utf8_one(ws->norm, n, end, &next_len); - end += next_len; - } - if (!single_cp_found) { - size_t end = off + cp_len; - double sc = cur.score + unk_score; - if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; - } - off += cp_len; - } - - int out_n = 0; - int prev_unk = 0; - for (size_t off = n; off > 0;) { - sp_best_t b = ws->best[off]; - if (b.prev < 0 || b.prev >= (int)off) - return -1; - int is_unk = b.id == tok->unk_id; - if (!(prev_unk && is_unk)) { - if (out_n >= ws->ids_cap) - return -1; - ws->ids[out_n++] = b.id; - } - prev_unk = is_unk; - off = (size_t)b.prev; - } - for (int i = 0; i < out_n / 2; i++) { - int t = ws->ids[i]; - ws->ids[i] = ws->ids[out_n - 1 - i]; - ws->ids[out_n - 1 - i] = t; - } - return out_n; -} - -tok_spm_t *tok_spm_load(const char *model_dir) { - if (!model_dir) - return NULL; - char path[1024]; - if (find_model_file(model_dir, path) != 0) { - fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); - return NULL; - } - unsigned char *data = NULL; - size_t len = 0; - if (load_file(path, &data, &len) != 0) { - fprintf(stderr, "sentencepiece: cannot read %s\n", path); - return NULL; - } - - tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t)); - if (!tok) { - free(data); - return NULL; - } - tok->add_dummy_prefix = 1; - tok->remove_extra_whitespaces = 1; - tok->escape_whitespaces = 1; - tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; - if (parse_model(tok, data, len) != 0) { - fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); - free(data); - tok_spm_free(tok); - return NULL; - } - free(data); - - int cap = next_pow2(tok->piece_count * 2 + 1); - tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); - if (!tok->piece_map) { - tok_spm_free(tok); - return NULL; - } - tok->piece_map_cap = cap; - for (int i = 0; i < tok->piece_count; i++) { - if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], - tok->piece_lens[i], i) != 0) { - tok_spm_free(tok); - return NULL; - } - } - - tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>")); - tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>")); - tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>")); - tok->pad_id = - tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); - tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA - ? tok->piece_count + 1 - : raw_to_model_id(tok, raw_piece_id(tok, "<mask>")); - tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; - tok->unknown_score = compute_unknown_score(tok); - if (tok->unk_id < 0 || tok->max_piece_len <= 0) { - fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path); - tok_spm_free(tok); - return NULL; - } - return tok; -} - -int tok_spm_token_id(const tok_spm_t *tok, const char *token) { - if (!tok || !token) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (!strcmp(token, "<s>")) - return 0; - if (!strcmp(token, "<pad>")) - return 1; - if (!strcmp(token, "</s>")) - return 2; - if (!strcmp(token, "<unk>")) - return 3; - if (!strcmp(token, "<mask>")) - return tok->piece_count + 1; - } - return raw_to_model_id(tok, raw_piece_id(tok, token)); -} - -static int tok_spm_encode_into(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text) - return -1; - if (normalize_into(tok, ws, text) != 0) - return -1; - int n = segment(tok, ws); - if (n < 0) - return -1; - if (out_n_tokens) - *out_n_tokens = n; - if (n == 0) - return 0; - if (!out_ids || out_cap < n) - return -2; - memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); - return 0; -} - -int *tok_spm_encode_with_workspace(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text) - return NULL; - int n = 0; - int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); - if (rc < 0 && rc != -2) - return NULL; - if (out_n_tokens) - *out_n_tokens = n; - if (n == 0) - return NULL; - int *ids = (int *)malloc((size_t)n * sizeof(*ids)); - if (!ids) - return NULL; - memcpy(ids, ws->ids, (size_t)n * sizeof(*ids)); - return ids; -} - -tok_spm_workspace_t *tok_spm_workspace_new(void) { - return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t)); -} - -void tok_spm_workspace_free(tok_spm_workspace_t *ws) { - if (!ws) - return; - free(ws->norm); - free(ws->best); - free(ws->ids); - free(ws); -} - -void tok_spm_free(tok_spm_t *tok) { - if (!tok) - return; - if (tok->id_to_piece) { - for (int i = 0; i < tok->piece_count; i++) - free(tok->id_to_piece[i]); - } - free(tok->id_to_piece); - free(tok->piece_lens); - free(tok->scores); - free(tok->types); - free(tok->piece_map); - free(tok->xcda); - free(tok->prefix_replacements); - free(tok); -} diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h deleted file mode 100644 index 2230538..0000000 --- a/libffwd/sentencepiece.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SentencePiece tokenizer for XLM-R style models. - * - * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, - * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R - * and multilingual E5. It encodes text only; frontends add the model's special - * token layout, e.g. <s> ... </s> for XLM-R. - */ - -#ifndef TOK_SPM_H -#define TOK_SPM_H - -#include <stddef.h> -#include <stdint.h> - -typedef enum { - TOK_SPM_IDENTITY = 0, - TOK_SPM_XLM_ROBERTA = 1, -} tok_spm_id_map_t; - -typedef struct { - char **id_to_piece; /* [piece_count] raw SentencePiece ids */ - uint16_t *piece_lens; /* byte length for id_to_piece entries */ - float *scores; /* SentencePiece score per raw id */ - uint8_t *types; /* SentencePiece piece type per raw id */ - int piece_count; /* raw SentencePiece vocabulary size */ - int vocab_size; /* model embedding vocab size after id mapping */ - int unk_id; /* model-space unknown-token id */ - int bos_id; /* model-space beginning-token id, or -1 */ - int eos_id; /* model-space end-token id, or -1 */ - int pad_id; /* model-space pad-token id, or -1 */ - int mask_id; /* model-space mask-token id, or -1 */ - int max_piece_len; /* max normal/user/unused piece byte length */ - double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ - int add_dummy_prefix; - int remove_extra_whitespaces; - int escape_whitespaces; - tok_spm_id_map_t id_map; - - void *piece_map; /* piece bytes -> raw SentencePiece id */ - int piece_map_cap; - - uint32_t *xcda; - size_t xcda_count; - char *prefix_replacements; - size_t prefix_replacements_len; -} tok_spm_t; - -typedef struct tok_spm_workspace tok_spm_workspace_t; - -/* Load the first supported SentencePiece model found in model_dir. Returns NULL - * on missing model file, unsupported model type, malformed protobuf, or OOM. */ -tok_spm_t *tok_spm_load(const char *model_dir); - -/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a - * model-space id, or -1 when absent. */ -int tok_spm_token_id(const tok_spm_t *tok, const char *token); - -tok_spm_workspace_t *tok_spm_workspace_new(void); -void tok_spm_workspace_free(tok_spm_workspace_t *ws); - -/* Return malloc-owned model-space token IDs, reusing workspace scratch. - * Empty text returns NULL with a zero count. */ -int *tok_spm_encode_with_workspace(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_n_tokens); - -void tok_spm_free(tok_spm_t *tok); - -#endif /* TOK_SPM_H */ diff --git a/libffwd/threadpool.c b/libffwd/threadpool.c deleted file mode 100644 index 6cb4252..0000000 --- a/libffwd/threadpool.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * CPU thread pool. - */ - -#if defined(__linux__) && !defined(_GNU_SOURCE) -# define _GNU_SOURCE -#endif - -#include "threadpool.h" - -#include <pthread.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#ifdef __APPLE__ -# include <sys/sysctl.h> -#else -# ifdef __linux__ -# include <sched.h> -# endif -# include <unistd.h> -#endif - -#if defined(USE_OPENBLAS) -void openblas_set_num_threads(int num_threads); -#endif - -#define TP_MAX_THREADS 16 - -static struct { - pthread_t threads[TP_MAX_THREADS - 1]; - int tids[TP_MAX_THREADS - 1]; - int n_threads; - int shutdown; - - parallel_fn_t fn; - void *arg; - uint64_t generation; - - pthread_mutex_t mutex; - pthread_cond_t cond_work; - pthread_cond_t cond_done; - int n_done; -} tp = { - .n_threads = 1, - .shutdown = 0, - .generation = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER, - .cond_work = PTHREAD_COND_INITIALIZER, - .cond_done = PTHREAD_COND_INITIALIZER, -}; - -static void *worker_loop(void *arg) { - int tid = *(int *)arg; - uint64_t my_gen = 0; - - for (;;) { - pthread_mutex_lock(&tp.mutex); - while (tp.generation == my_gen && !tp.shutdown) - pthread_cond_wait(&tp.cond_work, &tp.mutex); - if (tp.shutdown) { - pthread_mutex_unlock(&tp.mutex); - return NULL; - } - my_gen = tp.generation; - parallel_fn_t fn = tp.fn; - void *a = tp.arg; - int nt = tp.n_threads; - pthread_mutex_unlock(&tp.mutex); - - fn(tid, nt, a); - - pthread_mutex_lock(&tp.mutex); - if (++tp.n_done >= tp.n_threads - 1) - pthread_cond_signal(&tp.cond_done); - pthread_mutex_unlock(&tp.mutex); - } -} - -void tp_set_threads(int n) { - if (n < 1) - n = 1; - if (n > TP_MAX_THREADS) - n = TP_MAX_THREADS; - -#if defined(USE_OPENBLAS) - /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the - * auxiliary pool serial to avoid nested thread contention. */ - int blas_threads = n; - openblas_set_num_threads(blas_threads); - n = 1; -#endif - - /* Shutdown existing workers */ - if (tp.n_threads > 1) { - pthread_mutex_lock(&tp.mutex); - tp.shutdown = 1; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - for (int i = 0; i < tp.n_threads - 1; i++) - pthread_join(tp.threads[i], NULL); - tp.shutdown = 0; - tp.generation = 0; - } - - tp.n_threads = n; - if (n <= 1) - return; - - for (int i = 0; i < n - 1; i++) { - tp.tids[i] = i + 1; - pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); - } -} - -#ifndef __APPLE__ -/* CPU quota available to this process, in whole cores, or 0 if uncapped. - * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host - * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process - * can actually use. - * - * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: - * threads use the period budget early, then get throttled for the rest of the - * period, causing large bursty stalls. Example: a 16-thread default against a - * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. - * - * Floor instead of ceil, so a fully busy pool stays under quota and avoids - * throttling. - */ -static int cgroup_cpu_quota(void) { - /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or - * "max <period>" when unlimited. */ - FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r"); - if (f) { - char quota[32]; - long period = 0; - int got = fscanf(f, "%31s %ld", quota, &period); - fclose(f); - if (got == 2 && period > 0 && strcmp(quota, "max") != 0) { - long q = atol(quota); - if (q > 0) - return q / period < 1 ? 1 : (int)(q / period); - } - } - /* cgroup v1: separate files; quota -1 means unlimited. */ - long q = -1, period = 0; - f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r"); - if (f) { - if (fscanf(f, "%ld", &q) != 1) - q = -1; - fclose(f); - } - f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r"); - if (f) { - if (fscanf(f, "%ld", &period) != 1) - period = 0; - fclose(f); - } - if (q > 0 && period > 0) - return q / period < 1 ? 1 : (int)(q / period); - return 0; -} -#endif - -int tp_get_num_cpus(void) { -#ifdef __APPLE__ - int n = 0; - size_t len = sizeof(n); - sysctlbyname("hw.ncpu", &n, &len, NULL, 0); - return n > 0 ? n : 1; -#else - int online = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (online < 1) - online = 1; -# ifdef __linux__ - cpu_set_t set; - if (sched_getaffinity(0, sizeof(set), &set) == 0) { - int affinity = CPU_COUNT(&set); - if (affinity >= 1 && affinity < online) - online = affinity; - } -# endif - /* Honor a cgroup CPU quota when it is tighter than the visible core count, - * so the default thread pool does not oversubscribe a CPU-limited - * container and get throttled. */ - int quota = cgroup_cpu_quota(); - if (quota >= 1 && quota < online) - return quota; - return online; -#endif -} - -int tp_num_threads(void) { return tp.n_threads; } - -/* Dispatch work to all threads; main thread is tid=0 */ -void parallel_for(parallel_fn_t fn, void *arg) { - if (tp.n_threads <= 1) { - fn(0, 1, arg); - return; - } - - pthread_mutex_lock(&tp.mutex); - tp.fn = fn; - tp.arg = arg; - tp.n_done = 0; - tp.generation++; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - - fn(0, tp.n_threads, arg); - - pthread_mutex_lock(&tp.mutex); - while (tp.n_done < tp.n_threads - 1) - pthread_cond_wait(&tp.cond_done, &tp.mutex); - pthread_mutex_unlock(&tp.mutex); -} diff --git a/libffwd/threadpool.h b/libffwd/threadpool.h deleted file mode 100644 index f9739a1..0000000 --- a/libffwd/threadpool.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef TP_THREADPOOL_H -#define TP_THREADPOOL_H - -/* Persistent worker pool for CPU kernels. - * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. - * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. - * - * Kept separate from the kernels so they depend only on parallel_for and - * tp_num_threads, not pool internals. - * - * kernels.h includes this, so kernel callers need no extra include. */ - -/* Worker callback: (thread index, total threads, user arg). */ -typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); - -/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call - * before inference. */ -void tp_set_threads(int n); - -/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */ -int tp_get_num_cpus(void); - -/* Current pool size (1 == serial). Kernels gate parallelism on this. */ -int tp_num_threads(void); - -/* Run fn across the pool; the calling thread participates as tid 0. Blocks - * until every worker has finished. */ -void parallel_for(parallel_fn_t fn, void *arg); - -#endif /* TP_THREADPOOL_H */ diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c deleted file mode 100644 index 8e1000f..0000000 --- a/libffwd/tokenize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "ffwd.h" -#include "sentencepiece.h" - -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -struct ffwd_tok { - tok_spm_t *spm; - tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; -}; - -static void set_err(char *err, size_t errlen, const char *msg) { - if (err && errlen) - snprintf(err, errlen, "%s", msg); -} - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { - if (!model_dir) { - set_err(err, errlen, "model_dir is NULL"); - return NULL; - } - ffwd_tok_t *t = calloc(1, sizeof(*t)); - if (!t) { - set_err(err, errlen, "out of memory"); - return NULL; - } - t->spm = tok_spm_load(model_dir); - if (!t->spm) { - set_err(err, errlen, "failed to load SentencePiece tokenizer"); - goto fail; - } - t->workspace = tok_spm_workspace_new(); - if (!t->workspace) { - set_err(err, errlen, "out of memory"); - goto fail; - } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } - return t; - -fail: - ffwd_tok_free(t); - return NULL; -} - -void ffwd_tok_free(ffwd_tok_t *t) { - if (!t) - return; - tok_spm_workspace_free(t->workspace); - tok_spm_free(t->spm); - free(t); -} - -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { - if (n_out) - *n_out = 0; - if (!t || !text) - return NULL; - int n = 0; - int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); - if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { - free(core); - return NULL; - } - /* Empty text still produces the two special tokens. */ - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return NULL; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; - if (n_out) - *n_out = n + 2; - return ids; -} |
