aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/LICENSE20
-rw-r--r--libffwd/act.c29
-rw-r--r--libffwd/alloc.c65
-rw-r--r--libffwd/alloc.h22
-rw-r--r--libffwd/attn.c242
-rw-r--r--libffwd/config.c67
-rw-r--r--libffwd/config.h8
-rw-r--r--libffwd/ffwd.h49
-rw-r--r--libffwd/forward.c96
-rw-r--r--libffwd/gemm.c49
-rw-r--r--libffwd/generic.c27
-rw-r--r--libffwd/impl.h254
-rw-r--r--libffwd/kernels.h42
-rw-r--r--libffwd/model.c159
-rw-r--r--libffwd/model_blas.c68
-rw-r--r--libffwd/model_internal.h17
-rw-r--r--libffwd/model_types.h56
-rw-r--r--libffwd/norm.c33
-rw-r--r--libffwd/safetensors.c483
-rw-r--r--libffwd/safetensors.h64
-rw-r--r--libffwd/sentencepiece.c905
-rw-r--r--libffwd/sentencepiece.h71
-rw-r--r--libffwd/threadpool.c218
-rw-r--r--libffwd/threadpool.h30
-rw-r--r--libffwd/tokenize.c85
25 files changed, 0 insertions, 3159 deletions
diff --git a/libffwd/LICENSE b/libffwd/LICENSE
deleted file mode 100644
index a00caa7..0000000
--- a/libffwd/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-MIT License
-
-Copyright (c) 2026 The ffwd authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/libffwd/act.c b/libffwd/act.c
deleted file mode 100644
index 3912f2f..0000000
--- a/libffwd/act.c
+++ /dev/null
@@ -1,29 +0,0 @@
-#include "kernels.h"
-
-#include <math.h>
-
-void gelu_inplace(float *x, int n) {
- /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */
- for (int i = 0; i < n; i++)
- x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
-}
-
-void softmax(float *x, int rows, int cols) {
- for (int r = 0; r < rows; r++) {
- float *row = x + r * cols;
- float max_val = row[0];
- for (int c = 1; c < cols; c++) {
- if (row[c] > max_val)
- max_val = row[c];
- }
- float sum = 0.0f;
- for (int c = 0; c < cols; c++) {
- row[c] = expf(row[c] - max_val);
- sum += row[c];
- }
- float inv_sum = 1.0f / sum;
- for (int c = 0; c < cols; c++) {
- row[c] *= inv_sum;
- }
- }
-}
diff --git a/libffwd/alloc.c b/libffwd/alloc.c
deleted file mode 100644
index ba21fbb..0000000
--- a/libffwd/alloc.c
+++ /dev/null
@@ -1,65 +0,0 @@
-#include "alloc.h"
-
-#include <stdlib.h>
-#include <stdint.h>
-#include <limits.h>
-
-/* Sequence-capacity growth policy for the forward-pass workspaces:
- * For new growth:
- * - never allocate less than the minimum required size.
- * - round up to the granularity so small length changes do not trigger a realloc on every call.
- * - keep existing capacity if it is already larger. */
-#define FFWD_MIN_WORKSPACE_SEQ_CAP 16
-#define FFWD_WORKSPACE_SEQ_GRANULARITY 16
-
-int mul_size(size_t a, size_t b, size_t *out) {
- if (a != 0 && b > SIZE_MAX / a)
- return -1;
- *out = a * b;
- return 0;
-}
-
-int grow_cap(int current, int needed, int *out) {
- if (current < 0 || needed <= 0)
- return -1;
-
- int cap = needed;
- if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP)
- cap = FFWD_MIN_WORKSPACE_SEQ_CAP;
-
- int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY;
- if (rem != 0) {
- int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem;
- if (cap > INT_MAX - pad)
- return -1;
- cap += pad;
- }
-
- if (cap < current)
- cap = current;
-
- *out = cap;
- return 0;
-}
-
-int realloc_floats(float **ptr, size_t count) {
- size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0)
- return -1;
-
- void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0)
- return -1;
- *ptr = (float *)p;
- return 0;
-}
-
-int realloc_floats_2d(float **ptr, int rows, int cols) {
- if (rows < 0 || cols < 0)
- return -1;
-
- size_t count;
- if (mul_size((size_t)rows, (size_t)cols, &count) != 0)
- return -1;
- return realloc_floats(ptr, count);
-}
diff --git a/libffwd/alloc.h b/libffwd/alloc.h
deleted file mode 100644
index d3ef81a..0000000
--- a/libffwd/alloc.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef FFWD_ALLOC_H
-#define FFWD_ALLOC_H
-
-#include <stddef.h>
-
-/* Checked size arithmetic and typed allocation helpers.
- * Return -1 or NULL on overflow/allocation failure.
- * realloc_* leave the original pointer unchanged on failure. */
-
-/* out = a * b, or -1 if the product overflows size_t. */
-int mul_size(size_t a, size_t b, size_t *out);
-
-/* Grow sequence capacity without shrinking. */
-int grow_cap(int current, int needed, int *out);
-
-/* realloc *ptr to hold count floats; *ptr unchanged on failure. */
-int realloc_floats(float **ptr, size_t count);
-
-/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */
-int realloc_floats_2d(float **ptr, int rows, int cols);
-
-#endif /* FFWD_ALLOC_H */
diff --git a/libffwd/attn.c b/libffwd/attn.c
deleted file mode 100644
index ac0f4d1..0000000
--- a/libffwd/attn.c
+++ /dev/null
@@ -1,242 +0,0 @@
-#include "kernels.h"
-#include "impl.h"
-#include "threadpool.h"
-
-#include <math.h>
-
-#ifdef USE_BLAS
-# ifdef __APPLE__
-# include <Accelerate/Accelerate.h>
-# else
-# include <cblas.h>
-# endif
-#endif
-
-static inline float dot_f32_fast(const float *a, const float *b, int n) {
- return dot_f32_impl(a, b, n);
-}
-
-/* dst = dst * scale */
-static inline void vec_scale_inplace(float *dst, float scale, int n) {
- vec_scale_inplace_impl(dst, scale, n);
-}
-
-/* dst += alpha * src */
-static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
- vec_axpy_inplace_impl(dst, src, alpha, n);
-}
-
-/* dst = dst * correction + src */
-static inline void vec_scale_add(float *dst, const float *src, float correction, int n) {
- vec_scale_add_impl(dst, src, correction, n);
-}
-
-#define PACKED_ATTN_QUERY_TILE 32
-#define PACKED_ATTN_BLAS_MIN_SEQ 128
-
-static void bidirectional_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j < end; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
-
- float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f) {
- float inv_sum = 1.0f / sum_exp;
- vec_scale_inplace(o_row, inv_sum, head_dim);
- }
- }
-}
-
-#ifdef USE_BLAS
-static void bidirectional_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
- kv_hidden, 0.0f, scores, keys);
- softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-typedef struct {
- float *out;
- const float *Q;
- const float *K;
- const float *V;
- const int *offsets;
- int batch;
- int n_heads, n_kv_heads;
- int head_dim;
- float scale;
- float *scores;
- int max_seq;
-} packed_gqa_attn_task_t;
-
-static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
- bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end,
- t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
- }
- }
- }
-}
-
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
-#ifdef USE_BLAS
- if (!offsets || batch <= 0)
- return 0;
-
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- if (len > max_seq)
- max_seq = len;
- }
- if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ)
- return 0;
- if ((size_t)max_seq >
- SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads()))
- return 0;
- return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
-#else
- (void)offsets;
- (void)batch;
- return 0;
-#endif
-}
-
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * len * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
- parallel_for(packed_gqa_attn_worker, &task);
- } else {
- packed_gqa_attn_worker(0, 1, &task);
- }
-}
diff --git a/libffwd/config.c b/libffwd/config.c
deleted file mode 100644
index 6343113..0000000
--- a/libffwd/config.c
+++ /dev/null
@@ -1,67 +0,0 @@
-#include "config.h"
-
-#include <yyjson.h>
-
-#include <limits.h>
-#include <math.h>
-#include <stdio.h>
-
-static int json_get_int(yyjson_val *obj, const char *key, int fallback) {
- yyjson_val *v = yyjson_obj_get(obj, key);
- if (!v)
- return fallback;
- if (!yyjson_is_int(v))
- return -1;
- int64_t n = yyjson_get_sint(v);
- return n >= 0 && n <= INT_MAX ? (int)n : -1;
-}
-
-int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "ffwd_config: model path too long\n");
- return -1;
- }
-
- yyjson_read_err error;
- yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error);
- if (!doc) {
- fprintf(stderr, "ffwd_config: cannot read %s: %s\n", path, error.msg);
- return -1;
- }
- yyjson_val *root = yyjson_doc_get_root(doc);
- yyjson_val *causal = yyjson_obj_get(root, "is_causal");
- if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
- !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") ||
- (causal && !yyjson_is_false(causal))) {
- fprintf(stderr, "ffwd_config: expected bidirectional XLM-R with GeLU activation\n");
- yyjson_doc_free(doc);
- return -1;
- }
-
- cfg->hidden_size = json_get_int(root, "hidden_size", 0);
- cfg->n_layers = json_get_int(root, "num_hidden_layers", 0);
- cfg->n_heads = json_get_int(root, "num_attention_heads", 0);
- cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
- cfg->intermediate_size = json_get_int(root, "intermediate_size", 0);
- cfg->vocab_size = json_get_int(root, "vocab_size", 0);
- cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0);
- cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1);
- yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps");
- cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f;
- int pad_id = json_get_int(root, "pad_token_id", 1);
- cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1;
- yyjson_doc_free(doc);
-
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > FFWD_MAX_LAYERS ||
- cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 ||
- cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 ||
- cfg->position_id_offset < 0 ||
- cfg->max_position_embeddings <= cfg->position_id_offset ||
- !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) {
- fprintf(stderr, "ffwd_config: invalid encoder dimensions or normalization in %s\n", path);
- return -1;
- }
- return 0;
-}
diff --git a/libffwd/config.h b/libffwd/config.h
deleted file mode 100644
index b3e2f67..0000000
--- a/libffwd/config.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef FFWD_CONFIG_H
-#define FFWD_CONFIG_H
-
-#include "ffwd.h"
-
-int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir);
-
-#endif /* FFWD_CONFIG_H */
diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h
deleted file mode 100644
index 527e92f..0000000
--- a/libffwd/ffwd.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#ifndef FFWD_H
-#define FFWD_H
-
-#include <stddef.h>
-
-#define FFWD_MAX_LAYERS 64
-
-typedef struct {
- int hidden_size;
- int n_layers;
- int n_heads;
- int head_dim;
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings;
- int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
- int type_vocab_size;
- float layer_norm_eps;
-} ffwd_config_t;
-
-typedef struct ffwd_model ffwd_model_t;
-typedef struct ffwd_workspace ffwd_workspace_t;
-typedef struct ffwd_tok ffwd_tok_t;
-
-/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
-ffwd_model_t *ffwd_model_load(const char *model_dir);
-void ffwd_model_free(ffwd_model_t *model);
-const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model);
-
-/* A workspace belongs to one model and may serve only one inference call at a time. */
-ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model);
-void ffwd_workspace_free(ffwd_workspace_t *ws);
-
-/* Write the first token's final hidden state to out_embedding[hidden_size].
- * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
-int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
-
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen);
-void ffwd_tok_free(ffwd_tok_t *t);
-
-/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
- * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out);
-
-#endif /* FFWD_H */
diff --git a/libffwd/forward.c b/libffwd/forward.c
deleted file mode 100644
index b314642..0000000
--- a/libffwd/forward.c
+++ /dev/null
@@ -1,96 +0,0 @@
-#include "model_internal.h"
-#include "kernels.h"
-
-#include <limits.h>
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
-
-static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
- for (int row = 0; row < rows; row++) {
- float *dst = x + (size_t)row * dim;
- for (int d = 0; d < dim; d++)
- dst[d] += bias[d];
- }
-}
-
-int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding)
- return -1;
-
- const ffwd_config_t *cfg = &model->config;
- const ffwd_weights_t *w = &model->weights;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
- int head_dim = cfg->head_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->layer_norm_eps;
-
- if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) {
- fprintf(stderr,
- "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings "
- "%d\n",
- n_tokens, cfg->position_id_offset, cfg->max_position_embeddings);
- return -1;
- }
- if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter ||
- ensure_buffers(ws, cfg, n_tokens) != 0)
- return -1;
- int offsets[2] = {0, n_tokens};
- ensure_attention_scores(ws, offsets, 1);
-
- float *x = ws->x;
- float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v;
- float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up;
-
- for (int i = 0; i < n_tokens; i++) {
- int id = token_ids[i];
- if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i);
- return -1;
- }
- float *row = x + (size_t)i * hidden;
- memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float));
- int pos = cfg->position_id_offset + i;
- add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
- add_inplace(row, w->token_type_embeddings, hidden);
- }
- layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps);
-
- float scale = 1.0f / sqrtf((float)head_dim);
-
- for (int layer = 0; layer < cfg->n_layers; layer++) {
- const ffwd_layer_t *l = &w->layers[layer];
-
- linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden);
- linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden);
- linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden);
- add_bias_rows(q_buf, l->q_bias, n_tokens, hidden);
- add_bias_rows(k_buf, l->k_bias, n_tokens, hidden);
- add_bias_rows(v_buf, l->v_bias, n_tokens, hidden);
-
- bidirectional_gqa_attention_packed_with_scratch(
- attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale,
- ws->attn_scores, ws->attn_scores_bytes);
-
- linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden);
- add_bias_rows(proj_out, l->o_bias, n_tokens, hidden);
- add_inplace(x, proj_out, n_tokens * hidden);
- layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps);
-
- linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter);
- add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter);
- gelu_inplace(ffn_up, n_tokens * inter);
- linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden);
- add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden);
- add_inplace(x, proj_out, n_tokens * hidden);
- layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps);
- }
- /* The classifier consumes <s> after the encoder's final LayerNorm, without L2 normalization. */
- memcpy(out_embedding, x, (size_t)hidden * sizeof(float));
- return 0;
-}
diff --git a/libffwd/gemm.c b/libffwd/gemm.c
deleted file mode 100644
index c750be1..0000000
--- a/libffwd/gemm.c
+++ /dev/null
@@ -1,49 +0,0 @@
-#include "kernels.h"
-
-#ifdef USE_BLAS
-# ifdef __APPLE__
-# include <Accelerate/Accelerate.h>
-# else
-# include <cblas.h>
-# endif
-#endif
-
-void add_inplace(float *a, const float *b, int n) {
- for (int i = 0; i < n; i++)
- a[i] += b[i];
-}
-
-void linear(
- float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) {
-#ifdef USE_BLAS
- if (seq_len == 1) {
- cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1);
- } else {
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x,
- in_dim, W, in_dim, 0.0f, y, out_dim);
- }
- if (b != NULL) {
- for (int s = 0; s < seq_len; s++) {
- for (int o = 0; o < out_dim; o++) {
- y[s * out_dim + o] += b[o];
- }
- }
- }
-#else
- for (int s = 0; s < seq_len; s++) {
- const float *x_row = x + s * in_dim;
- float *y_row = y + s * out_dim;
- for (int o = 0; o < out_dim; o++) {
- const float *w_row = W + o * in_dim;
- float sum = (b != NULL) ? b[o] : 0.0f;
- for (int i = 0; i < in_dim; i++)
- sum += x_row[i] * w_row[i];
- y_row[o] = sum;
- }
- }
-#endif
-}
-
-void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
- linear(y, x, W, NULL, seq_len, in_dim, out_dim);
-}
diff --git a/libffwd/generic.c b/libffwd/generic.c
deleted file mode 100644
index 729c610..0000000
--- a/libffwd/generic.c
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Architecture-generic hot kernels / non-SIMD fallback.
- */
-
-#include "impl.h"
-
-float dot_f32_generic(const float *a, const float *b, int n) {
- float sum = 0.0f;
- for (int i = 0; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-void vec_scale_inplace_generic(float *dst, float scale, int n) {
- for (int i = 0; i < n; i++)
- dst[i] *= scale;
-}
-
-void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
- for (int i = 0; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-void vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
- for (int i = 0; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
diff --git a/libffwd/impl.h b/libffwd/impl.h
deleted file mode 100644
index f216518..0000000
--- a/libffwd/impl.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/* Inline vector operations keep attention rows in registers across reductions. */
-
-#ifndef KERNELS_IMPL_H
-#define KERNELS_IMPL_H
-
-#include <stdint.h>
-
-float dot_f32_generic(const float *a, const float *b, int n);
-void vec_scale_inplace_generic(float *dst, float scale, int n);
-void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
-void vec_scale_add_generic(float *dst, const float *src, float correction, int n);
-
-#ifdef __ARM_NEON
-
-# include <arm_neon.h>
-
-static inline float dot_f32_impl(const float *a, const float *b, int n) {
- int i = 0;
- float32x4_t acc0 = vdupq_n_f32(0.0f);
- float32x4_t acc1 = vdupq_n_f32(0.0f);
- for (; i + 8 <= n; i += 8) {
- float32x4_t a0 = vld1q_f32(a + i);
- float32x4_t b0 = vld1q_f32(b + i);
- float32x4_t a1 = vld1q_f32(a + i + 4);
- float32x4_t b1 = vld1q_f32(b + i + 4);
- acc0 = vfmaq_f32(acc0, a0, b0);
- acc1 = vfmaq_f32(acc1, a1, b1);
- }
- float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
- int i = 0;
- float32x4_t s = vdupq_n_f32(scale);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
- vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-}
-
-static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
- int i = 0;
- float32x4_t a = vdupq_n_f32(alpha);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
- vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
- int i = 0;
- float32x4_t c = vdupq_n_f32(correction);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
- vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
-
-#elif defined(__AVX2__) && defined(__FMA__)
-
-# include <immintrin.h>
-
-static inline float dot_f32_impl(const float *a, const float *b, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 acc0 = _mm512_setzero_ps();
- __m512 acc1 = _mm512_setzero_ps();
- __m512 acc2 = _mm512_setzero_ps();
- __m512 acc3 = _mm512_setzero_ps();
- for (; i + 64 <= n; i += 64) {
- acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
- acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
- acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
- acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
- }
- __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
- for (; i + 16 <= n; i += 16) {
- acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
- }
- float sum = _mm512_reduce_add_ps(acc);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-# else
- int i = 0;
- __m256 acc0 = _mm256_setzero_ps();
- __m256 acc1 = _mm256_setzero_ps();
- __m256 acc2 = _mm256_setzero_ps();
- __m256 acc3 = _mm256_setzero_ps();
- for (; i + 32 <= n; i += 32) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
- acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
- acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
- }
- acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
- for (; i + 8 <= n; i += 8) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- }
- __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
- r = _mm_hadd_ps(r, r);
- r = _mm_hadd_ps(r, r);
- float sum = _mm_cvtss_f32(r);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-# endif
-}
-
-static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 s = _mm512_set1_ps(scale);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
- _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
- _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-# else
- int i = 0;
- __m256 s = _mm256_set1_ps(scale);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
- _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
- _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-# endif
-}
-
-static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 a = _mm512_set1_ps(alpha);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
- _mm512_loadu_ps(dst + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
- _mm512_loadu_ps(dst + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
- _mm512_loadu_ps(dst + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-# else
- int i = 0;
- __m256 a = _mm256_set1_ps(alpha);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
- _mm256_loadu_ps(dst + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
- _mm256_loadu_ps(dst + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
- _mm256_loadu_ps(dst + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-# endif
-}
-
-static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 c = _mm512_set1_ps(correction);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
- _mm512_loadu_ps(src + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
- _mm512_loadu_ps(src + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
- _mm512_loadu_ps(src + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-# else
- int i = 0;
- __m256 c = _mm256_set1_ps(correction);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
- _mm256_loadu_ps(src + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
- _mm256_loadu_ps(src + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
- _mm256_loadu_ps(src + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-# endif
-}
-
-#else /* no SIMD path: dispatch to the scalar references */
-
-# define dot_f32_impl dot_f32_generic
-# define vec_scale_inplace_impl vec_scale_inplace_generic
-# define vec_axpy_inplace_impl vec_axpy_inplace_generic
-# define vec_scale_add_impl vec_scale_add_generic
-
-#endif
-
-#endif /* KERNELS_IMPL_H */
diff --git a/libffwd/kernels.h b/libffwd/kernels.h
deleted file mode 100644
index 1025602..0000000
--- a/libffwd/kernels.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef KERNELS_H
-#define KERNELS_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-void add_inplace(float *a, const float *b, int n);
-
-/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out]. */
-void linear(
- float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
-void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
-
-/* Mean and population variance are reduced over the hidden axis per row. */
-void layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps);
-
-void softmax(float *x, int rows, int cols);
-void gelu_inplace(float *x, int n);
-
-/* Attention is block-diagonal: each token attends only within its offsets span.
- * Missing scratch uses the online-softmax path instead of tiled BLAS. */
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
-
-#endif /* KERNELS_H */
diff --git a/libffwd/model.c b/libffwd/model.c
deleted file mode 100644
index 81073f3..0000000
--- a/libffwd/model.c
+++ /dev/null
@@ -1,159 +0,0 @@
-#include "model_internal.h"
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim) {
- if (t->dtype != DTYPE_F32) {
- fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name);
- return 0;
- }
- if (t->ndim != ndim) {
- fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
- return 0;
- }
-
- size_t numel = 1;
- for (int i = 0; i < ndim; i++) {
- if (t->shape[i] != shape[i]) {
- fprintf(stderr,
- "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n",
- name, i, (long long)t->shape[i], (long long)shape[i]);
- return 0;
- }
- if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
- fprintf(stderr, "ffwd: shape too large or empty for %s\n", name);
- return 0;
- }
- numel *= (size_t)shape[i];
- }
- size_t bytes;
- if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) {
- fprintf(stderr, "ffwd: invalid data size for %s\n", name);
- return 0;
- }
- size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
- t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name);
- return 0;
- }
- return 1;
-}
-
-static const float *load_weight(multi_safetensors_t *ms, const char *name,
- const int64_t *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
- if (!t) {
- fprintf(stderr, "ffwd: tensor not found: %s\n", name);
- return NULL;
- }
- if (!tensor_has_supported_shape(sf, t, name, shape, ndim))
- return NULL;
- return safetensors_data(sf, t);
-}
-
-static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) {
- const ffwd_config_t *cfg = &model->config;
- ffwd_weights_t *w = &model->weights;
- char name[256];
- int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
-
- w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
- (int64_t[]){cfg->vocab_size, hidden}, 2);
- w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
- (int64_t[]){cfg->max_position_embeddings, hidden}, 2);
- w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
- (int64_t[]){cfg->type_vocab_size, hidden}, 2);
- w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
- w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
- if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
- !w->ffwd_ln_w || !w->ffwd_ln_b)
- return -1;
-
- w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t));
- if (!w->layers)
- return -1;
-
- for (int i = 0; i < cfg->n_layers; i++) {
- ffwd_layer_t *l = &w->layers[i];
-
-#define BERT_W(field, suffix, d0, d1) \
- do { \
- const int64_t e[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 2); \
- if (!l->field) \
- return -1; \
- } while (0)
-#define BERT_V(field, suffix, d0) \
- do { \
- const int64_t e[1] = {(d0)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 1); \
- if (!l->field) \
- return -1; \
- } while (0)
-
- BERT_W(wq, "attention.self.query.weight", hidden, hidden);
- BERT_V(q_bias, "attention.self.query.bias", hidden);
- BERT_W(wk, "attention.self.key.weight", hidden, hidden);
- BERT_V(k_bias, "attention.self.key.bias", hidden);
- BERT_W(wv, "attention.self.value.weight", hidden, hidden);
- BERT_V(v_bias, "attention.self.value.bias", hidden);
- BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
- BERT_V(o_bias, "attention.output.dense.bias", hidden);
- BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
- BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
- BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
- BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
- BERT_W(down_proj, "output.dense.weight", hidden, inter);
- BERT_V(ffn_out_bias, "output.dense.bias", hidden);
- BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
- BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
-
-#undef BERT_W
-#undef BERT_V
- }
- return 0;
-}
-
-ffwd_model_t *ffwd_model_load(const char *model_dir) {
- if (!model_dir)
- return NULL;
- ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t));
- if (!model)
- return NULL;
- if (ffwd_config_parse(&model->config, model_dir) != 0)
- goto fail;
- model->safetensors = multi_safetensors_open(model_dir);
- if (!model->safetensors) {
- fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir);
- goto fail;
- }
- if (load_bert_weights(model, model->safetensors) != 0)
- goto fail;
- return model;
-
-fail:
- ffwd_model_free(model);
- return NULL;
-}
-
-void ffwd_model_free(ffwd_model_t *model) {
- if (!model)
- return;
- multi_safetensors_close(model->safetensors);
- free(model->weights.layers);
- free(model);
-}
-
-const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) {
- return model ? &model->config : NULL;
-}
diff --git a/libffwd/model_blas.c b/libffwd/model_blas.c
deleted file mode 100644
index 409ba66..0000000
--- a/libffwd/model_blas.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include "model_internal.h"
-#include "kernels.h"
-
-#include <stdlib.h>
-
-ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) {
- if (!model)
- return NULL;
- ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t));
- if (!ws)
- return NULL;
- ws->model = model;
- return ws;
-}
-
-void ffwd_workspace_free(ffwd_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->x);
- free(ws->q);
- free(ws->k);
- free(ws->v);
- free(ws->attn_out);
- free(ws->proj_out);
- free(ws->ffn_up);
- free(ws->attn_scores);
- free(ws);
-}
-
-int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) {
- if (seq <= ws->buf_seq_cap)
- return 0;
-
- int cap;
- if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
- return -1;
-
-#define R(ptr, n) \
- do { \
- if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
- return -1; \
- } while (0)
-
- R(ws->x, c->hidden_size);
- R(ws->q, c->hidden_size);
- R(ws->k, c->hidden_size);
- R(ws->v, c->hidden_size);
- R(ws->attn_out, c->hidden_size);
- R(ws->proj_out, c->hidden_size);
- R(ws->ffn_up, c->intermediate_size);
-
-#undef R
-
- ws->buf_seq_cap = cap;
- return 0;
-}
-
-void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) {
- size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (bytes <= ws->attn_scores_bytes)
- return;
-
- void *p = realloc(ws->attn_scores, bytes);
- if (!p)
- return;
- ws->attn_scores = (float *)p;
- ws->attn_scores_bytes = bytes;
-}
diff --git a/libffwd/model_internal.h b/libffwd/model_internal.h
deleted file mode 100644
index aeea52e..0000000
--- a/libffwd/model_internal.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef FFWD_MODEL_INTERNAL_H
-#define FFWD_MODEL_INTERNAL_H
-
-#include "model_types.h"
-#include "alloc.h"
-
-/* Validate F32 dtype, exact dimensions, byte count, and mapped-file bounds. */
-int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim);
-
-int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int seq);
-void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch);
-
-#endif /* FFWD_MODEL_INTERNAL_H */
diff --git a/libffwd/model_types.h b/libffwd/model_types.h
deleted file mode 100644
index 447b653..0000000
--- a/libffwd/model_types.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef FFWD_MODEL_TYPES_H
-#define FFWD_MODEL_TYPES_H
-
-#include "ffwd.h"
-#include "safetensors.h"
-
-typedef struct {
- const float *wq;
- const float *wk;
- const float *wv;
- const float *wo;
- const float *q_bias;
- const float *k_bias;
- const float *v_bias;
- const float *o_bias;
- const float *input_norm;
- const float *attn_ln_bias;
- const float *up_proj;
- const float *ffn_inter_bias;
- const float *down_proj;
- const float *ffn_out_bias;
- const float *post_attn_norm;
- const float *ffn_ln_bias;
-} ffwd_layer_t;
-
-typedef struct {
- const float *embed_tokens;
- ffwd_layer_t *layers;
- const float *position_embeddings;
- const float *token_type_embeddings;
- const float *ffwd_ln_w;
- const float *ffwd_ln_b;
-} ffwd_weights_t;
-
-struct ffwd_model {
- ffwd_config_t config;
- ffwd_weights_t weights;
- /* All weight pointers borrow this mapping until model teardown. */
- multi_safetensors_t *safetensors;
-};
-
-struct ffwd_workspace {
- const ffwd_model_t *model;
- int buf_seq_cap;
- float *x;
- float *q;
- float *k;
- float *v;
- float *attn_out;
- float *proj_out;
- float *ffn_up;
- float *attn_scores;
- size_t attn_scores_bytes;
-};
-
-#endif /* FFWD_MODEL_TYPES_H */
diff --git a/libffwd/norm.c b/libffwd/norm.c
deleted file mode 100644
index 59e8bdb..0000000
--- a/libffwd/norm.c
+++ /dev/null
@@ -1,33 +0,0 @@
-#include "kernels.h"
-
-#include <math.h>
-
-/* Safe in place: each row's mean and variance are reduced before it is rewritten. */
-void layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps) {
- if (seq_len <= 0 || hidden <= 0)
- return;
- float inv_h = 1.0f / (float)hidden;
- for (int s = 0; s < seq_len; s++) {
- const float *row = x + (size_t)s * hidden;
- float *dst = out + (size_t)s * hidden;
- float mean = 0.0f;
- for (int d = 0; d < hidden; d++)
- mean += row[d];
- mean *= inv_h;
- float var = 0.0f;
- for (int d = 0; d < hidden; d++) {
- float diff = row[d] - mean;
- var += diff * diff;
- }
- var *= inv_h;
- float inv_std = 1.0f / sqrtf(var + eps);
- for (int d = 0; d < hidden; d++)
- dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
- }
-}
diff --git a/libffwd/safetensors.c b/libffwd/safetensors.c
deleted file mode 100644
index cfba76e..0000000
--- a/libffwd/safetensors.c
+++ /dev/null
@@ -1,483 +0,0 @@
-#include "safetensors.h"
-
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-/* Minimal JSON parser for safetensors header */
-/* TODO: consider replacing with yyjson */
-
-static void skip_whitespace(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
- (*p)++;
-}
-
-static int parse_string(const char **p, char *out, size_t max_len) {
- skip_whitespace(p);
- if (**p != '"')
- return -1;
- (*p)++;
- size_t i = 0;
- while (**p && **p != '"' && i < max_len - 1) {
- if (**p == '\\') {
- (*p)++;
- if (**p == 'n')
- out[i++] = '\n';
- else if (**p == 't')
- out[i++] = '\t';
- else if (**p == '"')
- out[i++] = '"';
- else if (**p == '\\')
- out[i++] = '\\';
- else
- out[i++] = **p;
- } else {
- out[i++] = **p;
- }
- (*p)++;
- }
- out[i] = '\0';
- if (**p != '"')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_int(const char **p, int64_t *out) {
- skip_whitespace(p);
- int64_t val = 0;
- int neg = 0;
- if (**p == '-') {
- neg = 1;
- (*p)++;
- }
- if (**p < '0' || **p > '9')
- return -1;
- while (**p >= '0' && **p <= '9') {
- val = val * 10 + (**p - '0');
- (*p)++;
- }
- *out = neg ? -val : val;
- return 0;
-}
-
-static safetensor_dtype_t parse_dtype(const char *s) {
- if (strcmp(s, "F32") == 0)
- return DTYPE_F32;
- if (strcmp(s, "F16") == 0)
- return DTYPE_F16;
- if (strcmp(s, "BF16") == 0)
- return DTYPE_BF16;
- if (strcmp(s, "I32") == 0)
- return DTYPE_I32;
- if (strcmp(s, "I64") == 0)
- return DTYPE_I64;
- if (strcmp(s, "BOOL") == 0)
- return DTYPE_BOOL;
- return DTYPE_UNKNOWN;
-}
-
-static int parse_tensor_entry(const char **p, safetensor_t *t) {
- skip_whitespace(p);
- if (**p != '{')
- return -1;
- (*p)++;
-
- t->dtype = DTYPE_UNKNOWN;
- t->ndim = 0;
- t->data_offset = 0;
- t->data_size = 0;
-
- while (**p && **p != '}') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
-
- char key[64];
- if (parse_string(p, key, sizeof(key)) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ':')
- return -1;
- (*p)++;
- skip_whitespace(p);
-
- if (strcmp(key, "dtype") == 0) {
- char dtype_str[32];
- if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
- return -1;
- t->dtype = parse_dtype(dtype_str);
- } else if (strcmp(key, "shape") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- t->ndim = 0;
- while (**p && **p != ']') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
- if (t->ndim >= 8)
- return -1;
- if (parse_int(p, &t->shape[t->ndim]) != 0)
- return -1;
- t->ndim++;
- }
- if (**p != ']')
- return -1;
- (*p)++;
- } else if (strcmp(key, "data_offsets") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t start_i;
- if (parse_int(p, &start_i) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ',')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t end_i;
- if (parse_int(p, &end_i) != 0)
- return -1;
- if (start_i < 0 || end_i < start_i)
- return -1;
- t->data_offset = (size_t)start_i;
- t->data_size = (size_t)(end_i - start_i);
- skip_whitespace(p);
- if (**p != ']')
- return -1;
- (*p)++;
- } else {
- /* Skip unknown value */
- if (**p == '"') {
- (*p)++;
- while (**p && **p != '"') {
- if (**p == '\\')
- (*p)++;
- if (**p)
- (*p)++;
- }
- if (**p == '"')
- (*p)++;
- } else if (**p == '[') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '[')
- depth++;
- else if (**p == ']')
- depth--;
- (*p)++;
- }
- } else if (**p == '{') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '{')
- depth++;
- else if (**p == '}')
- depth--;
- (*p)++;
- }
- } else {
- while (**p && **p != ',' && **p != '}')
- (*p)++;
- }
- }
- }
- if (**p != '}')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_header(safetensors_file_t *sf) {
- const char *p = sf->header_json;
- skip_whitespace(&p);
- if (*p != '{')
- return -1;
- p++;
-
- sf->num_tensors = 0;
-
- while (*p && *p != '}') {
- skip_whitespace(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
-
- char name[256];
- if (parse_string(&p, name, sizeof(name)) != 0)
- return -1;
- skip_whitespace(&p);
- if (*p != ':')
- return -1;
- p++;
-
- if (strcmp(name, "__metadata__") == 0) {
- skip_whitespace(&p);
- if (*p == '{') {
- int depth = 1;
- p++;
- while (*p && depth > 0) {
- if (*p == '{')
- depth++;
- else if (*p == '}')
- depth--;
- p++;
- }
- }
- continue;
- }
-
- if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
- fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
- sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
- return -1;
- }
-
- safetensor_t *t = &sf->tensors[sf->num_tensors];
- snprintf(t->name, sizeof(t->name), "%s", name);
- if (parse_tensor_entry(&p, t) != 0)
- return -1;
- sf->num_tensors++;
- }
- if (*p != '}')
- return -1;
- return 0;
-}
-
-/* Single file operations */
-
-safetensors_file_t *safetensors_open(const char *path) {
- int fd = open(path, O_RDONLY);
- if (fd < 0)
- return NULL;
-
- struct stat st;
- if (fstat(fd, &st) < 0) {
- close(fd);
- return NULL;
- }
-
- size_t file_size = (size_t)st.st_size;
- if (file_size < 8) {
- close(fd);
- return NULL;
- }
-
- void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
- close(fd);
- if (data == MAP_FAILED)
- return NULL;
-
- uint64_t header_size = 0;
- memcpy(&header_size, data, 8);
- if (header_size > file_size - 8) {
- munmap(data, file_size);
- return NULL;
- }
-
- safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
- if (!sf) {
- munmap(data, file_size);
- return NULL;
- }
-
- sf->path = strdup(path);
- sf->data = data;
- sf->file_size = file_size;
- sf->header_size = (size_t)header_size;
-
- sf->header_json = malloc(header_size + 1);
- if (!sf->header_json) {
- safetensors_close(sf);
- return NULL;
- }
- memcpy(sf->header_json, (char *)data + 8, header_size);
- sf->header_json[header_size] = '\0';
-
- if (parse_header(sf) != 0) {
- safetensors_close(sf);
- return NULL;
- }
-
- return sf;
-}
-
-void safetensors_close(safetensors_file_t *sf) {
- if (!sf)
- return;
- if (sf->data)
- munmap(sf->data, sf->file_size);
- free(sf->path);
- free(sf->header_json);
- free(sf);
-}
-
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
- return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
-}
-
-/* Multi-shard operations */
-
-static int cmp_shard_names(const void *a, const void *b) {
- const char *sa = (const char *)a;
- const char *sb = (const char *)b;
- return strcmp(sa, sb);
-}
-
-static int parse_shard_name(const char *name, int *idx, int *total) {
- int i = 0, n = 0, consumed = 0;
- if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
- return -1;
- if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
- return -1;
- *idx = i;
- *total = n;
- return 0;
-}
-
-multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
- multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
- if (!ms)
- return NULL;
-
- char path[4096];
-
- /* Try single file first */
- snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
- safetensors_file_t *sf = safetensors_open(path);
- if (sf) {
- ms->shards[0] = sf;
- ms->num_shards = 1;
- return ms;
- }
-
- /* Scan directory for shard files */
- DIR *dir = opendir(model_dir);
- if (!dir) {
- free(ms);
- return NULL;
- }
-
- struct dirent *entry;
- char shard_names[SAFETENSORS_MAX_SHARDS][256];
- int seen[SAFETENSORS_MAX_SHARDS] = {0};
- int n_shards = 0;
- int expected_total = 0;
-
- while ((entry = readdir(dir)) != NULL) {
- if (strncmp(entry->d_name, "model-", 6) == 0 &&
- strstr(entry->d_name, ".safetensors") != NULL) {
- int shard_idx = 0, shard_total = 0;
- if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
- fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (shard_total > SAFETENSORS_MAX_SHARDS) {
- fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
- SAFETENSORS_MAX_SHARDS);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (expected_total == 0) {
- expected_total = shard_total;
- } else if (expected_total != shard_total) {
- fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
- entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (seen[shard_idx - 1]) {
- fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
- closedir(dir);
- free(ms);
- return NULL;
- }
- seen[shard_idx - 1] = 1;
-
- snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
- n_shards++;
- }
- }
- closedir(dir);
-
- if (n_shards == 0) {
- fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
- free(ms);
- return NULL;
- }
- if (expected_total != n_shards) {
- fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
- expected_total, n_shards, model_dir);
- free(ms);
- return NULL;
- }
- for (int i = 0; i < expected_total; i++) {
- if (!seen[i]) {
- fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
- expected_total, model_dir);
- free(ms);
- return NULL;
- }
- }
-
- /* Sort shard names to ensure consistent ordering */
- qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
-
- /* Open each shard */
- for (int i = 0; i < n_shards; i++) {
- snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
- ms->shards[i] = safetensors_open(path);
- if (!ms->shards[i]) {
- fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
- multi_safetensors_close(ms);
- return NULL;
- }
- }
- ms->num_shards = n_shards;
- return ms;
-}
-
-void multi_safetensors_close(multi_safetensors_t *ms) {
- if (!ms)
- return;
- for (int i = 0; i < ms->num_shards; i++) {
- safetensors_close(ms->shards[i]);
- }
- free(ms);
-}
-
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) {
- for (int s = 0; s < ms->num_shards; s++) {
- safetensors_file_t *sf = ms->shards[s];
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0) {
- if (out_sf)
- *out_sf = sf;
- return &sf->tensors[i];
- }
- }
- }
- return NULL;
-}
diff --git a/libffwd/safetensors.h b/libffwd/safetensors.h
deleted file mode 100644
index dca68a7..0000000
--- a/libffwd/safetensors.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Safetensors file format reader with multi-shard support.
- */
-
-#ifndef SAFETENSORS_H
-#define SAFETENSORS_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-#define SAFETENSORS_MAX_TENSORS 1024
-#define SAFETENSORS_MAX_SHARDS 8
-
-typedef enum {
- DTYPE_F32 = 0,
- DTYPE_F16 = 1,
- DTYPE_BF16 = 2,
- DTYPE_I32 = 3,
- DTYPE_I64 = 4,
- DTYPE_BOOL = 5,
- DTYPE_UNKNOWN = -1
-} safetensor_dtype_t;
-
-typedef struct {
- char name[256];
- safetensor_dtype_t dtype;
- int ndim;
- int64_t shape[8];
- size_t data_offset;
- size_t data_size;
-} safetensor_t;
-
-typedef struct {
- char *path;
- void *data;
- size_t file_size;
- size_t header_size;
- char *header_json;
- int num_tensors;
- safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
-} safetensors_file_t;
-
-/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
-typedef struct {
- safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
- int num_shards;
-} multi_safetensors_t;
-
-/* Open a single safetensors file (memory-mapped) */
-safetensors_file_t *safetensors_open(const char *path);
-void safetensors_close(safetensors_file_t *sf);
-
-/* Open model from directory (auto-detects single file or multi-shard) */
-multi_safetensors_t *multi_safetensors_open(const char *model_dir);
-void multi_safetensors_close(multi_safetensors_t *ms);
-
-/* Find a tensor by name across all shards */
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf);
-
-/* Get raw pointer to tensor data (within mmap'd region) */
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
-
-#endif /* SAFETENSORS_H */
diff --git a/libffwd/sentencepiece.c b/libffwd/sentencepiece.c
deleted file mode 100644
index 6c30ba1..0000000
--- a/libffwd/sentencepiece.c
+++ /dev/null
@@ -1,905 +0,0 @@
-/*
- * SentencePiece Unigram tokenizer.
- *
- * The implementation reads the small subset of ModelProto needed at runtime:
- * pieces, scores, piece types, trainer model_type, and normalizer options. The
- * normalizer supports SentencePiece's precompiled charsmap format used by
- * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program
- * over UTF-8 byte offsets.
- */
-
-#include "sentencepiece.h"
-
-#include <errno.h>
-#include <float.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-enum {
- SP_MODEL_UNIGRAM = 1,
- SP_PIECE_NORMAL = 1,
- SP_PIECE_UNKNOWN = 2,
- SP_PIECE_CONTROL = 3,
- SP_PIECE_USER_DEFINED = 4,
- SP_PIECE_UNUSED = 5,
- SP_PIECE_BYTE = 6,
-};
-
-typedef struct {
- const char *key;
- uint16_t len;
- int val;
-} sp_entry_t;
-
-typedef struct {
- int id;
- int prev;
- double score;
-} sp_best_t;
-
-struct tok_spm_workspace {
- char *norm;
- size_t norm_len;
- size_t norm_cap;
- sp_best_t *best;
- int best_cap;
- int *ids;
- int ids_cap;
-};
-
-static uint64_t fnv1a_bytes(const char *s, size_t n) {
- uint64_t h = 1469598103934665603ULL;
- for (size_t i = 0; i < n; i++) {
- h ^= (unsigned char)s[i];
- h *= 1099511628211ULL;
- }
- return h;
-}
-
-static int next_pow2(int x) {
- int p = 1;
- while (p < x && p <= (1 << 29))
- p <<= 1;
- return p;
-}
-
-static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) {
- if (len <= 0 || len > UINT16_MAX)
- return -1;
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
- while (map[pos].key) {
- if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) {
- map[pos].val = val;
- return 0;
- }
- pos = (int)((pos + 1) & (int)mask);
- }
- map[pos].key = key;
- map[pos].len = (uint16_t)len;
- map[pos].val = val;
- return 0;
-}
-
-static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
- if (!map || len <= 0)
- return -1;
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
- while (map[pos].key) {
- if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0)
- return map[pos].val;
- pos = (int)((pos + 1) & (int)mask);
- }
- return -1;
-}
-
-static int is_tokenizable_type(int type) {
- return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
- type == SP_PIECE_BYTE;
-}
-
-static int reserve_chars(char **buf, size_t *cap, size_t need) {
- if (need <= *cap)
- return 0;
- size_t nc = *cap ? *cap : 128;
- while (nc < need) {
- if (nc > SIZE_MAX / 2)
- return -1;
- nc *= 2;
- }
- char *p = (char *)realloc(*buf, nc);
- if (!p)
- return -1;
- *buf = p;
- *cap = nc;
- return 0;
-}
-
-static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) {
- if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0)
- return -1;
- memcpy(*buf + *len, s, n);
- *len += n;
- (*buf)[*len] = '\0';
- return 0;
-}
-
-static int reserve_best(tok_spm_workspace_t *ws, int need) {
- if (need <= ws->best_cap)
- return 0;
- int nc = ws->best_cap ? ws->best_cap : 128;
- while (nc < need) {
- if (nc > INT_MAX / 2)
- return -1;
- nc *= 2;
- }
- sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p));
- if (!p)
- return -1;
- ws->best = p;
- ws->best_cap = nc;
- return 0;
-}
-
-static int reserve_ids(tok_spm_workspace_t *ws, int need) {
- if (need <= ws->ids_cap)
- return 0;
- int nc = ws->ids_cap ? ws->ids_cap : 64;
- while (nc < need) {
- if (nc > INT_MAX / 2)
- return -1;
- nc *= 2;
- }
- int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p));
- if (!p)
- return -1;
- ws->ids = p;
- ws->ids_cap = nc;
- return 0;
-}
-
-static int utf8_len(unsigned char c) {
- if (c < 0x80)
- return 1;
- if ((c & 0xE0) == 0xC0)
- return 2;
- if ((c & 0xF0) == 0xE0)
- return 3;
- if ((c & 0xF8) == 0xF0)
- return 4;
- return 1;
-}
-
-static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) {
- if (off >= n)
- return 0;
- const unsigned char *p = (const unsigned char *)s + off;
- int len = utf8_len(p[0]);
- if (off + (size_t)len > n) {
- *len_out = 1;
- return 0;
- }
- if (len > 1) {
- for (int i = 1; i < len; i++) {
- if ((p[i] & 0xC0) != 0x80) {
- *len_out = 1;
- return 0;
- }
- }
- }
- *len_out = (size_t)len;
- return 1;
-}
-
-static uint32_t read_le32(const unsigned char *p) {
- return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
-}
-
-static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) {
- uint64_t v = 0;
- int shift = 0;
- while (*p < end && shift <= 63) {
- unsigned char c = *(*p)++;
- v |= (uint64_t)(c & 0x7f) << shift;
- if (!(c & 0x80)) {
- *out = v;
- return 0;
- }
- shift += 7;
- }
- return -1;
-}
-
-static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) {
- uint64_t key = 0;
- if (pb_read_varint(p, end, &key) != 0)
- return -1;
- *field = (int)(key >> 3);
- *wire = (int)(key & 7);
- return *field > 0 ? 0 : -1;
-}
-
-static int pb_read_len(const unsigned char **p,
- const unsigned char *end,
- const unsigned char **data,
- size_t *len) {
- uint64_t n = 0;
- if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p))
- return -1;
- *data = *p;
- *len = (size_t)n;
- *p += n;
- return 0;
-}
-
-static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) {
- uint64_t v = 0;
- const unsigned char *data = NULL;
- size_t len = 0;
- switch (wire) {
- case 0:
- return pb_read_varint(p, end, &v);
- case 1:
- if ((size_t)(end - *p) < 8)
- return -1;
- *p += 8;
- return 0;
- case 2:
- return pb_read_len(p, end, &data, &len);
- case 5:
- if ((size_t)(end - *p) < 4)
- return -1;
- *p += 4;
- return 0;
- default:
- return -1;
- }
-}
-
-static char *memdup0(const unsigned char *data, size_t len) {
- char *s = (char *)malloc(len + 1);
- if (!s)
- return NULL;
- memcpy(s, data, len);
- s[len] = '\0';
- return s;
-}
-
-static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- char *piece = NULL;
- float score = 0.0f;
- int type = SP_PIECE_NORMAL;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- goto fail;
- if (field == 1 && wire == 2) {
- const unsigned char *s = NULL;
- size_t n = 0;
- if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX)
- goto fail;
- free(piece);
- piece = memdup0(s, n);
- if (!piece)
- goto fail;
- tok->piece_lens[idx] = (uint16_t)n;
- } else if (field == 2 && wire == 5) {
- if ((size_t)(end - p) < 4)
- goto fail;
- uint32_t bits = read_le32(p);
- memcpy(&score, &bits, sizeof(score));
- p += 4;
- } else if (field == 3 && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX)
- goto fail;
- type = (int)v;
- } else if (pb_skip(&p, end, wire) != 0) {
- goto fail;
- }
- }
- if (!piece)
- goto fail;
- tok->id_to_piece[idx] = piece;
- tok->scores[idx] = score;
- tok->types[idx] = (uint8_t)type;
- if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
- tok->max_piece_len = tok->piece_lens[idx];
- return 0;
-fail:
- free(piece);
- return -1;
-}
-
-static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- int model_type = SP_MODEL_UNIGRAM;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (field == 3 && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX)
- return -1;
- model_type = (int)v;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- if (model_type != SP_MODEL_UNIGRAM) {
- fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n",
- model_type);
- return -1;
- }
- (void)tok;
- return 0;
-}
-
-static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if ((field == 3 || field == 4 || field == 5) && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0)
- return -1;
- if (field == 3)
- tok->add_dummy_prefix = v != 0;
- else if (field == 4)
- tok->remove_extra_whitespaces = v != 0;
- else
- tok->escape_whitespaces = v != 0;
- } else if (field == 2 && wire == 2) {
- const unsigned char *blob = NULL;
- size_t n = 0;
- if (pb_read_len(&p, end, &blob, &n) != 0)
- return -1;
- if (n >= 4) {
- uint32_t xcda_bytes = read_le32(blob);
- if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) {
- tok->xcda_count = xcda_bytes / 4;
- tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda));
- if (!tok->xcda)
- return -1;
- for (size_t i = 0; i < tok->xcda_count; i++)
- tok->xcda[i] = read_le32(blob + 4 + i * 4);
- tok->prefix_replacements_len = n - 4 - xcda_bytes;
- tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1);
- if (!tok->prefix_replacements)
- return -1;
- memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes,
- tok->prefix_replacements_len);
- tok->prefix_replacements[tok->prefix_replacements_len] = '\0';
- }
- }
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- return 0;
-}
-
-static int count_pieces(const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- int n = 0;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (field == 1 && wire == 2) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0)
- return -1;
- n++;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- return n;
-}
-
-static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
- int n_pieces = count_pieces(data, len);
- if (n_pieces <= 0)
- return -1;
- tok->piece_count = n_pieces;
- tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
- tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
- tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
- tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
- if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
- return -1;
-
- const unsigned char *p = data, *end = data + len;
- int piece_idx = 0;
- int saw_trainer = 0;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (wire == 2 && field == 1) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces ||
- parse_piece(tok, piece_idx++, msg, msg_len) != 0)
- return -1;
- } else if (wire == 2 && field == 2) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
- return -1;
- saw_trainer = 1;
- } else if (wire == 2 && field == 3) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0)
- return -1;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- if (piece_idx != n_pieces || !saw_trainer)
- return -1;
- return 0;
-}
-
-static int raw_piece_id(const tok_spm_t *tok, const char *piece) {
- if (!tok || !piece)
- return -1;
- return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece));
-}
-
-static int raw_to_model_id(const tok_spm_t *tok, int raw_id) {
- if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (raw_id == 1)
- return 0; /* <s> */
- if (raw_id == 2)
- return 2; /* </s> */
- if (raw_id == 0)
- return 3; /* <unk> */
- return raw_id + 1;
- }
- return raw_id;
-}
-
-static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return 0;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return 0;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return 0;
- }
- size_t got = fread(buf, 1, (size_t)sz, f);
- fclose(f);
- buf[got] = '\0';
- int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
- strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
- strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
- free(buf);
- return yes;
-}
-
-static int detect_xlm_roberta(const char *model_dir) {
- return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
- file_contains_xlm_roberta(model_dir, "config.json");
-}
-
-static int find_model_file(const char *model_dir, char path[1024]) {
- static const char *names[] = {
- "sentencepiece.bpe.model",
- "tokenizer.model",
- "spiece.model",
- };
- for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
- int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
- if (n < 0 || n >= 1024)
- return -1;
- FILE *f = fopen(path, "rb");
- if (f) {
- fclose(f);
- return 0;
- }
- if (errno != ENOENT)
- return -1;
- }
- return -1;
-}
-
-static int load_file(const char *path, unsigned char **out, size_t *out_len) {
- FILE *f = fopen(path, "rb");
- if (!f)
- return -1;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- unsigned char *buf = (unsigned char *)malloc((size_t)sz);
- if (!buf) {
- fclose(f);
- return -1;
- }
- if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f);
- free(buf);
- return -1;
- }
- fclose(f);
- *out = buf;
- *out_len = (size_t)sz;
- return 0;
-}
-
-static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) {
- uint32_t v = tok->xcda[idx];
- return (v >> 10) << ((v & (1u << 9)) >> 6);
-}
-
-static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) {
- return tok->xcda[idx] & ((1u << 31) | 0xffu);
-}
-
-static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; }
-
-static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) {
- return tok->xcda[idx] & ((1u << 31) - 1u);
-}
-
-typedef struct {
- const char *data;
- size_t len;
- size_t consumed;
-} norm_prefix_t;
-
-static norm_prefix_t
-normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) {
- static const char repl[] = "\xef\xbf\xbd";
- if (off >= input_len)
- return (norm_prefix_t){input + off, 0, 0};
-
- size_t best_len = 0;
- size_t best_repl = 0;
- if (tok->xcda && tok->xcda_count > 0) {
- uint32_t node = xcda_base(tok, 0);
- for (size_t p = off; p < input_len; p++) {
- unsigned char c = (unsigned char)input[p];
- if (c == 0)
- break;
- node ^= c;
- if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c)
- break;
- int leaf = xcda_leaf(tok, node);
- node ^= xcda_base(tok, node);
- if (leaf) {
- best_len = p - off + 1;
- best_repl = xcda_value(tok, node);
- }
- }
- }
- if (best_len > 0 && best_repl < tok->prefix_replacements_len) {
- const char *r = tok->prefix_replacements + best_repl;
- size_t max = tok->prefix_replacements_len - best_repl;
- size_t n = strnlen(r, max);
- return (norm_prefix_t){r, n, best_len};
- }
-
- size_t clen = 1;
- if (valid_utf8_one(input, input_len, off, &clen))
- return (norm_prefix_t){input + off, clen, clen};
- return (norm_prefix_t){repl, 3, 1};
-}
-
-static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) {
- ws->norm_len = 0;
- if (!text)
- return -1;
- size_t input_len = strlen(text);
- const char space[] = "\xe2\x96\x81"; /* U+2581 */
- const char *space_s = tok->escape_whitespaces ? space : " ";
- size_t space_n = tok->escape_whitespaces ? 3 : 1;
- int processing_non_ws = 0;
- int space_prepended = 0;
-
- for (size_t off = 0; off < input_len;) {
- norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
- for (size_t i = 0; i < r.len; i++) {
- char c = r.data[i];
- if (c != ' ') {
- if (!processing_non_ws) {
- processing_non_ws = 1;
- if ((tok->add_dummy_prefix && !space_prepended) ||
- tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
- 0)
- return -1;
- space_prepended = 1;
- }
- }
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
- return -1;
- } else {
- processing_non_ws = 0;
- if (!tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0)
- return -1;
- }
- }
- }
- off += r.consumed ? r.consumed : 1;
- }
- if (ws->norm)
- ws->norm[ws->norm_len] = '\0';
- return 0;
-}
-
-static double compute_unknown_score(const tok_spm_t *tok) {
- float min_score = FLT_MAX;
- for (int i = 0; i < tok->piece_count; i++) {
- if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
- min_score = tok->scores[i];
- }
- if (min_score == FLT_MAX)
- min_score = -10.0f;
- return (double)min_score - 10.0;
-}
-
-static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
- size_t n = ws->norm_len;
- if (n == 0)
- return 0;
- if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 ||
- reserve_ids(ws, (int)n + 1) != 0)
- return -1;
- for (size_t i = 0; i <= n; i++)
- ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
- ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
- double unk_score = tok->unknown_score;
-
- for (size_t off = 0; off < n;) {
- size_t cp_len = 1;
- valid_utf8_one(ws->norm, n, off, &cp_len);
- sp_best_t cur = ws->best[off];
- if (cur.score <= -DBL_MAX / 2.0) {
- off += cp_len;
- continue;
- }
-
- int single_cp_found = 0;
- for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
- int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off,
- (int)(end - off));
- if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
- if (end - off == cp_len)
- single_cp_found = 1;
- double sc =
- cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
- if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
- }
- if (end == n)
- break;
- size_t next_len = 1;
- valid_utf8_one(ws->norm, n, end, &next_len);
- end += next_len;
- }
- if (!single_cp_found) {
- size_t end = off + cp_len;
- double sc = cur.score + unk_score;
- if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
- }
- off += cp_len;
- }
-
- int out_n = 0;
- int prev_unk = 0;
- for (size_t off = n; off > 0;) {
- sp_best_t b = ws->best[off];
- if (b.prev < 0 || b.prev >= (int)off)
- return -1;
- int is_unk = b.id == tok->unk_id;
- if (!(prev_unk && is_unk)) {
- if (out_n >= ws->ids_cap)
- return -1;
- ws->ids[out_n++] = b.id;
- }
- prev_unk = is_unk;
- off = (size_t)b.prev;
- }
- for (int i = 0; i < out_n / 2; i++) {
- int t = ws->ids[i];
- ws->ids[i] = ws->ids[out_n - 1 - i];
- ws->ids[out_n - 1 - i] = t;
- }
- return out_n;
-}
-
-tok_spm_t *tok_spm_load(const char *model_dir) {
- if (!model_dir)
- return NULL;
- char path[1024];
- if (find_model_file(model_dir, path) != 0) {
- fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
- return NULL;
- }
- unsigned char *data = NULL;
- size_t len = 0;
- if (load_file(path, &data, &len) != 0) {
- fprintf(stderr, "sentencepiece: cannot read %s\n", path);
- return NULL;
- }
-
- tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t));
- if (!tok) {
- free(data);
- return NULL;
- }
- tok->add_dummy_prefix = 1;
- tok->remove_extra_whitespaces = 1;
- tok->escape_whitespaces = 1;
- tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY;
- if (parse_model(tok, data, len) != 0) {
- fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
- free(data);
- tok_spm_free(tok);
- return NULL;
- }
- free(data);
-
- int cap = next_pow2(tok->piece_count * 2 + 1);
- tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t));
- if (!tok->piece_map) {
- tok_spm_free(tok);
- return NULL;
- }
- tok->piece_map_cap = cap;
- for (int i = 0; i < tok->piece_count; i++) {
- if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
- tok->piece_lens[i], i) != 0) {
- tok_spm_free(tok);
- return NULL;
- }
- }
-
- tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
- tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
- tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
- tok->pad_id =
- tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
- tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA
- ? tok->piece_count + 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
- tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
- tok->unknown_score = compute_unknown_score(tok);
- if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
- fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
- tok_spm_free(tok);
- return NULL;
- }
- return tok;
-}
-
-int tok_spm_token_id(const tok_spm_t *tok, const char *token) {
- if (!tok || !token)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (!strcmp(token, "<s>"))
- return 0;
- if (!strcmp(token, "<pad>"))
- return 1;
- if (!strcmp(token, "</s>"))
- return 2;
- if (!strcmp(token, "<unk>"))
- return 3;
- if (!strcmp(token, "<mask>"))
- return tok->piece_count + 1;
- }
- return raw_to_model_id(tok, raw_piece_id(tok, token));
-}
-
-static int tok_spm_encode_into(const tok_spm_t *tok,
- tok_spm_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return -1;
- if (normalize_into(tok, ws, text) != 0)
- return -1;
- int n = segment(tok, ws);
- if (n < 0)
- return -1;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return 0;
- if (!out_ids || out_cap < n)
- return -2;
- memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
- return 0;
-}
-
-int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
- tok_spm_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return NULL;
- int n = 0;
- int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n);
- if (rc < 0 && rc != -2)
- return NULL;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return NULL;
- int *ids = (int *)malloc((size_t)n * sizeof(*ids));
- if (!ids)
- return NULL;
- memcpy(ids, ws->ids, (size_t)n * sizeof(*ids));
- return ids;
-}
-
-tok_spm_workspace_t *tok_spm_workspace_new(void) {
- return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t));
-}
-
-void tok_spm_workspace_free(tok_spm_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->norm);
- free(ws->best);
- free(ws->ids);
- free(ws);
-}
-
-void tok_spm_free(tok_spm_t *tok) {
- if (!tok)
- return;
- if (tok->id_to_piece) {
- for (int i = 0; i < tok->piece_count; i++)
- free(tok->id_to_piece[i]);
- }
- free(tok->id_to_piece);
- free(tok->piece_lens);
- free(tok->scores);
- free(tok->types);
- free(tok->piece_map);
- free(tok->xcda);
- free(tok->prefix_replacements);
- free(tok);
-}
diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h
deleted file mode 100644
index 2230538..0000000
--- a/libffwd/sentencepiece.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * SentencePiece tokenizer for XLM-R style models.
- *
- * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model,
- * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R
- * and multilingual E5. It encodes text only; frontends add the model's special
- * token layout, e.g. <s> ... </s> for XLM-R.
- */
-
-#ifndef TOK_SPM_H
-#define TOK_SPM_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-typedef enum {
- TOK_SPM_IDENTITY = 0,
- TOK_SPM_XLM_ROBERTA = 1,
-} tok_spm_id_map_t;
-
-typedef struct {
- char **id_to_piece; /* [piece_count] raw SentencePiece ids */
- uint16_t *piece_lens; /* byte length for id_to_piece entries */
- float *scores; /* SentencePiece score per raw id */
- uint8_t *types; /* SentencePiece piece type per raw id */
- int piece_count; /* raw SentencePiece vocabulary size */
- int vocab_size; /* model embedding vocab size after id mapping */
- int unk_id; /* model-space unknown-token id */
- int bos_id; /* model-space beginning-token id, or -1 */
- int eos_id; /* model-space end-token id, or -1 */
- int pad_id; /* model-space pad-token id, or -1 */
- int mask_id; /* model-space mask-token id, or -1 */
- int max_piece_len; /* max normal/user/unused piece byte length */
- double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
- int add_dummy_prefix;
- int remove_extra_whitespaces;
- int escape_whitespaces;
- tok_spm_id_map_t id_map;
-
- void *piece_map; /* piece bytes -> raw SentencePiece id */
- int piece_map_cap;
-
- uint32_t *xcda;
- size_t xcda_count;
- char *prefix_replacements;
- size_t prefix_replacements_len;
-} tok_spm_t;
-
-typedef struct tok_spm_workspace tok_spm_workspace_t;
-
-/* Load the first supported SentencePiece model found in model_dir. Returns NULL
- * on missing model file, unsupported model type, malformed protobuf, or OOM. */
-tok_spm_t *tok_spm_load(const char *model_dir);
-
-/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
- * model-space id, or -1 when absent. */
-int tok_spm_token_id(const tok_spm_t *tok, const char *token);
-
-tok_spm_workspace_t *tok_spm_workspace_new(void);
-void tok_spm_workspace_free(tok_spm_workspace_t *ws);
-
-/* Return malloc-owned model-space token IDs, reusing workspace scratch.
- * Empty text returns NULL with a zero count. */
-int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
- tok_spm_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-void tok_spm_free(tok_spm_t *tok);
-
-#endif /* TOK_SPM_H */
diff --git a/libffwd/threadpool.c b/libffwd/threadpool.c
deleted file mode 100644
index 6cb4252..0000000
--- a/libffwd/threadpool.c
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * CPU thread pool.
- */
-
-#if defined(__linux__) && !defined(_GNU_SOURCE)
-# define _GNU_SOURCE
-#endif
-
-#include "threadpool.h"
-
-#include <pthread.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifdef __APPLE__
-# include <sys/sysctl.h>
-#else
-# ifdef __linux__
-# include <sched.h>
-# endif
-# include <unistd.h>
-#endif
-
-#if defined(USE_OPENBLAS)
-void openblas_set_num_threads(int num_threads);
-#endif
-
-#define TP_MAX_THREADS 16
-
-static struct {
- pthread_t threads[TP_MAX_THREADS - 1];
- int tids[TP_MAX_THREADS - 1];
- int n_threads;
- int shutdown;
-
- parallel_fn_t fn;
- void *arg;
- uint64_t generation;
-
- pthread_mutex_t mutex;
- pthread_cond_t cond_work;
- pthread_cond_t cond_done;
- int n_done;
-} tp = {
- .n_threads = 1,
- .shutdown = 0,
- .generation = 0,
- .mutex = PTHREAD_MUTEX_INITIALIZER,
- .cond_work = PTHREAD_COND_INITIALIZER,
- .cond_done = PTHREAD_COND_INITIALIZER,
-};
-
-static void *worker_loop(void *arg) {
- int tid = *(int *)arg;
- uint64_t my_gen = 0;
-
- for (;;) {
- pthread_mutex_lock(&tp.mutex);
- while (tp.generation == my_gen && !tp.shutdown)
- pthread_cond_wait(&tp.cond_work, &tp.mutex);
- if (tp.shutdown) {
- pthread_mutex_unlock(&tp.mutex);
- return NULL;
- }
- my_gen = tp.generation;
- parallel_fn_t fn = tp.fn;
- void *a = tp.arg;
- int nt = tp.n_threads;
- pthread_mutex_unlock(&tp.mutex);
-
- fn(tid, nt, a);
-
- pthread_mutex_lock(&tp.mutex);
- if (++tp.n_done >= tp.n_threads - 1)
- pthread_cond_signal(&tp.cond_done);
- pthread_mutex_unlock(&tp.mutex);
- }
-}
-
-void tp_set_threads(int n) {
- if (n < 1)
- n = 1;
- if (n > TP_MAX_THREADS)
- n = TP_MAX_THREADS;
-
-#if defined(USE_OPENBLAS)
- /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the
- * auxiliary pool serial to avoid nested thread contention. */
- int blas_threads = n;
- openblas_set_num_threads(blas_threads);
- n = 1;
-#endif
-
- /* Shutdown existing workers */
- if (tp.n_threads > 1) {
- pthread_mutex_lock(&tp.mutex);
- tp.shutdown = 1;
- pthread_cond_broadcast(&tp.cond_work);
- pthread_mutex_unlock(&tp.mutex);
- for (int i = 0; i < tp.n_threads - 1; i++)
- pthread_join(tp.threads[i], NULL);
- tp.shutdown = 0;
- tp.generation = 0;
- }
-
- tp.n_threads = n;
- if (n <= 1)
- return;
-
- for (int i = 0; i < n - 1; i++) {
- tp.tids[i] = i + 1;
- pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]);
- }
-}
-
-#ifndef __APPLE__
-/* CPU quota available to this process, in whole cores, or 0 if uncapped.
- * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host
- * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process
- * can actually use.
- *
- * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota:
- * threads use the period budget early, then get throttled for the rest of the
- * period, causing large bursty stalls. Example: a 16-thread default against a
- * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool.
- *
- * Floor instead of ceil, so a fully busy pool stays under quota and avoids
- * throttling.
- */
-static int cgroup_cpu_quota(void) {
- /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or
- * "max <period>" when unlimited. */
- FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r");
- if (f) {
- char quota[32];
- long period = 0;
- int got = fscanf(f, "%31s %ld", quota, &period);
- fclose(f);
- if (got == 2 && period > 0 && strcmp(quota, "max") != 0) {
- long q = atol(quota);
- if (q > 0)
- return q / period < 1 ? 1 : (int)(q / period);
- }
- }
- /* cgroup v1: separate files; quota -1 means unlimited. */
- long q = -1, period = 0;
- f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r");
- if (f) {
- if (fscanf(f, "%ld", &q) != 1)
- q = -1;
- fclose(f);
- }
- f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r");
- if (f) {
- if (fscanf(f, "%ld", &period) != 1)
- period = 0;
- fclose(f);
- }
- if (q > 0 && period > 0)
- return q / period < 1 ? 1 : (int)(q / period);
- return 0;
-}
-#endif
-
-int tp_get_num_cpus(void) {
-#ifdef __APPLE__
- int n = 0;
- size_t len = sizeof(n);
- sysctlbyname("hw.ncpu", &n, &len, NULL, 0);
- return n > 0 ? n : 1;
-#else
- int online = (int)sysconf(_SC_NPROCESSORS_ONLN);
- if (online < 1)
- online = 1;
-# ifdef __linux__
- cpu_set_t set;
- if (sched_getaffinity(0, sizeof(set), &set) == 0) {
- int affinity = CPU_COUNT(&set);
- if (affinity >= 1 && affinity < online)
- online = affinity;
- }
-# endif
- /* Honor a cgroup CPU quota when it is tighter than the visible core count,
- * so the default thread pool does not oversubscribe a CPU-limited
- * container and get throttled. */
- int quota = cgroup_cpu_quota();
- if (quota >= 1 && quota < online)
- return quota;
- return online;
-#endif
-}
-
-int tp_num_threads(void) { return tp.n_threads; }
-
-/* Dispatch work to all threads; main thread is tid=0 */
-void parallel_for(parallel_fn_t fn, void *arg) {
- if (tp.n_threads <= 1) {
- fn(0, 1, arg);
- return;
- }
-
- pthread_mutex_lock(&tp.mutex);
- tp.fn = fn;
- tp.arg = arg;
- tp.n_done = 0;
- tp.generation++;
- pthread_cond_broadcast(&tp.cond_work);
- pthread_mutex_unlock(&tp.mutex);
-
- fn(0, tp.n_threads, arg);
-
- pthread_mutex_lock(&tp.mutex);
- while (tp.n_done < tp.n_threads - 1)
- pthread_cond_wait(&tp.cond_done, &tp.mutex);
- pthread_mutex_unlock(&tp.mutex);
-}
diff --git a/libffwd/threadpool.h b/libffwd/threadpool.h
deleted file mode 100644
index f9739a1..0000000
--- a/libffwd/threadpool.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef TP_THREADPOOL_H
-#define TP_THREADPOOL_H
-
-/* Persistent worker pool for CPU kernels.
- * Uses a fixed thread set that sleeps until parallel_for() gives it a callback.
- * Also provides the cgroup-quota-aware CPU-count helper used to size the pool.
- *
- * Kept separate from the kernels so they depend only on parallel_for and
- * tp_num_threads, not pool internals.
- *
- * kernels.h includes this, so kernel callers need no extra include. */
-
-/* Worker callback: (thread index, total threads, user arg). */
-typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg);
-
-/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call
- * before inference. */
-void tp_set_threads(int n);
-
-/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */
-int tp_get_num_cpus(void);
-
-/* Current pool size (1 == serial). Kernels gate parallelism on this. */
-int tp_num_threads(void);
-
-/* Run fn across the pool; the calling thread participates as tid 0. Blocks
- * until every worker has finished. */
-void parallel_for(parallel_fn_t fn, void *arg);
-
-#endif /* TP_THREADPOOL_H */
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
deleted file mode 100644
index 8e1000f..0000000
--- a/libffwd/tokenize.c
+++ /dev/null
@@ -1,85 +0,0 @@
-#include "ffwd.h"
-#include "sentencepiece.h"
-
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-struct ffwd_tok {
- tok_spm_t *spm;
- tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
-};
-
-static void set_err(char *err, size_t errlen, const char *msg) {
- if (err && errlen)
- snprintf(err, errlen, "%s", msg);
-}
-
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) {
- if (!model_dir) {
- set_err(err, errlen, "model_dir is NULL");
- return NULL;
- }
- ffwd_tok_t *t = calloc(1, sizeof(*t));
- if (!t) {
- set_err(err, errlen, "out of memory");
- return NULL;
- }
- t->spm = tok_spm_load(model_dir);
- if (!t->spm) {
- set_err(err, errlen, "failed to load SentencePiece tokenizer");
- goto fail;
- }
- t->workspace = tok_spm_workspace_new();
- if (!t->workspace) {
- set_err(err, errlen, "out of memory");
- goto fail;
- }
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
- return t;
-
-fail:
- ffwd_tok_free(t);
- return NULL;
-}
-
-void ffwd_tok_free(ffwd_tok_t *t) {
- if (!t)
- return;
- tok_spm_workspace_free(t->workspace);
- tok_spm_free(t->spm);
- free(t);
-}
-
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) {
- if (n_out)
- *n_out = 0;
- if (!t || !text)
- return NULL;
- int n = 0;
- int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
- if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
- free(core);
- return NULL;
- }
- /* Empty text still produces the two special tokens. */
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return NULL;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
- if (n_out)
- *n_out = n + 2;
- return ids;
-}