aboutsummaryrefslogtreecommitdiff
path: root/src/embed.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:16:14 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:16:14 +0200
commit3904a33e4416c35a48e8dd7c242a137a2cc17871 (patch)
tree677c7d0a8d272fdc4a089d3bbd2004712938f0ee /src/embed.c
parentcca81e59a9afeebe1237f713e919b5002df3d749 (diff)
downloadffwd-3904a33e4416c35a48e8dd7c242a137a2cc17871.tar.gz
ffwd-3904a33e4416c35a48e8dd7c242a137a2cc17871.zip
cpu: extract late-interaction model into late.c
Diffstat (limited to 'src/embed.c')
-rw-r--r--src/embed.c417
1 files changed, 30 insertions, 387 deletions
diff --git a/src/embed.c b/src/embed.c
index e265427..baa9d11 100644
--- a/src/embed.c
+++ b/src/embed.c
@@ -1,4 +1,5 @@
#include "embed.h"
+#include "engine.h"
#include "alloc.h"
#include "config.h"
#include "internal.h"
@@ -11,21 +12,6 @@
#include <string.h>
#include <math.h>
#include <limits.h>
-#include <float.h>
-
-struct embed_late_model {
- embed_model_t *base;
- safetensors_file_t *dense_sf;
- embed_weight_ref_t projection; /* [token_dim, hidden] */
- int token_dim;
-};
-
-struct embed_late_workspace {
- const embed_late_model_t *model;
- embed_workspace_t *base_ws;
- float *states;
- int states_seq_cap;
-};
/* ========================================================================
* Globals
@@ -52,23 +38,11 @@ static int model_dir_has_file(const char *model_dir, const char *rel) {
return 1;
}
-static int model_dir_has_late_projection(const char *model_dir) {
+int model_dir_has_late_projection(const char *model_dir) {
return model_dir_has_file(model_dir, "1_Dense/config.json") &&
model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
- if (!ws || !ws->model || n_tokens <= 0)
- return -1;
- if (n_tokens <= ws->states_seq_cap)
- return 0;
- int hidden = ws->model->base->config.hidden_size;
- if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
- return -1;
- ws->states_seq_cap = n_tokens;
- return 0;
-}
-
/* ========================================================================
* Weight loading (direct mmap pointers into safetensors)
* ======================================================================== */
@@ -103,12 +77,12 @@ static const char *dtype_name(safetensor_dtype_t dtype) {
}
}
-static int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim,
- int bf16_ok) {
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim,
+ int bf16_ok) {
if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) {
fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
@@ -217,13 +191,13 @@ static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
return 0;
}
-static void linear_nobias_weight(embed_workspace_t *ws,
- float *y,
- const float *x,
- const embed_weight_ref_t *w,
- int seq_len,
- int in_dim,
- int out_dim) {
+void linear_nobias_weight(embed_workspace_t *ws,
+ float *y,
+ const float *x,
+ const embed_weight_ref_t *w,
+ int seq_len,
+ int in_dim,
+ int out_dim) {
if (w->dtype == DTYPE_F32) {
embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
return;
@@ -500,7 +474,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
return 0;
}
-static embed_model_t *
+embed_model_t *
model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
if (!allow_late && model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_model_load: late-interaction models require "
@@ -722,245 +696,6 @@ const embed_config_t *embed_model_config(const embed_model_t *model) {
return model ? &model->config : NULL;
}
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
- if (!sf || !name)
- return NULL;
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0)
- return &sf->tensors[i];
- }
- return NULL;
-}
-
-embed_late_model_t *embed_late_model_load(const char *model_dir) {
- if (!model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base)
- goto fail;
-
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_late_model_load: model path too long\n");
- goto fail;
- }
-
- late->dense_sf = safetensors_open(path);
- if (!late->dense_sf) {
- fprintf(stderr, "embed_late_model_load: cannot open %s\n", path);
- goto fail;
- }
-
- const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
- if (!t) {
- fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
- fprintf(stderr, "embed_late_model_load: bad projection shape\n");
- goto fail;
- }
-
- int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
- goto fail;
-
- late->projection.dtype = t->dtype;
- late->projection.data = safetensors_data(late->dense_sf, t);
- late->token_dim = (int)t->shape[0];
- return late;
-
-fail:
- embed_late_model_free(late);
- return NULL;
-}
-
-void embed_late_model_free(embed_late_model_t *model) {
- if (!model)
- return;
- embed_model_free(model->base);
- safetensors_close(model->dense_sf);
- free(model);
-}
-
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
- if (!model || !model->base)
- return NULL;
- embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws)
- return NULL;
- ws->model = model;
- ws->base_ws = embed_workspace_new(model->base);
- if (!ws->base_ws) {
- free(ws);
- return NULL;
- }
- return ws;
-}
-
-void embed_late_workspace_free(embed_late_workspace_t *ws) {
- if (!ws)
- return;
- embed_workspace_free(ws->base_ws);
- free(ws->states);
- free(ws);
-}
-
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
- return model && model->base ? &model->base->config : NULL;
-}
-
-int embed_late_model_token_dim(const embed_late_model_t *model) {
- return model ? model->token_dim : 0;
-}
-
-/* Internal (embed_internal.h): expose the base model and projection so GPU
- * backends can upload them to device without seeing struct embed_late_model. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
- return model ? model->base : NULL;
-}
-
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
- return model ? &model->projection : NULL;
-}
-
-int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
- model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
- if (ensure_late_state_buffer(ws, n_tokens) != 0)
- return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
- return -1;
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
- dim);
- if (normalize) {
- for (int i = 0; i < n_tokens; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
-float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim) {
- if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
- return 0.0f;
-
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *q = query_vectors + (size_t)qi * dim;
- float best = -FLT_MAX;
- for (int di = 0; di < doc_tokens; di++) {
- const float *d = doc_vectors + (size_t)di * dim;
- float dot = embed_dot_f32(q, d, dim);
- if (dot > best)
- best = dot;
- }
- score += best;
- }
- return score;
-}
-
-int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores) {
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
- docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
- return -1;
-
- for (int i = 0; i < docs; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
- return -1;
- }
-
-#ifdef USE_BLAS
- /* One similarity GEMM per group of whole candidates:
- * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
- * row-max plus sum. The group token budget bounds the scratch matrix;
- * one oversized document still runs alone in a group of its own. */
- enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
- int total = doc_offsets[docs];
- int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
- int largest = 0;
- for (int i = 0; i < docs; i++) {
- int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest)
- largest = len;
- }
- if ((size_t)query_tokens * (size_t)largest > cap)
- cap = (size_t)query_tokens * (size_t)largest;
- float *sim = (float *)malloc(cap * sizeof(float));
- if (sim) {
- int doc = 0;
- while (doc < docs) {
- int first = doc;
- int start = doc_offsets[first];
- doc++;
- while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
- doc++;
- int group_tokens = doc_offsets[doc] - start;
-
- embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
- group_tokens);
-
- for (int i = first; i < doc; i++) {
- int s0 = doc_offsets[i] - start;
- int s1 = doc_offsets[i + 1] - start;
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *row = sim + (size_t)qi * group_tokens;
- float best = row[s0];
- for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best)
- best = row[t];
- score += best;
- }
- scores[i] = score;
- }
- }
- free(sim);
- return 0;
- }
- /* Allocation failure: fall through to the scalar path. */
-#endif
-
- for (int i = 0; i < docs; i++) {
- int start = doc_offsets[i];
- int end = doc_offsets[i + 1];
- scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim, end - start, dim);
- }
- return 0;
-}
-
size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
if (!ws)
return 0;
@@ -1028,12 +763,12 @@ size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
* Packed forward pass
* ======================================================================== */
-static int build_offsets(const embed_input_t *inputs,
- int batch,
- int require_ids,
- int *offsets,
- int *total_out,
- int *max_out) {
+int build_offsets(const embed_input_t *inputs,
+ int batch,
+ int require_ids,
+ int *offsets,
+ int *total_out,
+ int *max_out) {
if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
return -1;
@@ -1176,14 +911,14 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
return 0;
}
-static int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq,
- int apply_final_norm) {
+int forward_packed_inplace(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
+ int apply_final_norm) {
return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL,
0, model->config.n_layers, apply_final_norm);
}
@@ -1533,98 +1268,6 @@ int embed_model_encode_batch(const embed_model_t *model,
return rc;
}
-/* Late-interaction batch encoder. Lives here, beside the packed/ragged batch
- * machinery it reuses, rather than next to embed_late_model_encode_tokens. */
-int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
- if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
- n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
-
- /* Pack all candidates for one block-diagonal forward. inputs/offsets index
- * the full token stream; out_offsets is the prefix sum of kept tokens. */
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
- int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
- if (!inputs || !offsets) {
- free(inputs);
- free(offsets);
- return -1;
- }
-
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- }
- inputs[d].ids = doc_ids[d];
- inputs[d].n_tokens = n_tokens[d];
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- int total_seq = 0, max_seq = 0;
- int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
- if (rc == 0)
- rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
- max_seq, 1);
- free(inputs);
- if (rc != 0) {
- free(offsets);
- return -1;
- }
-
- /* Gather each document's kept hidden rows into ws->states (its natural
- * [seq, hidden] shape), then project the packed [total_keep, hidden] block
- * to [total_keep, dim] in one GEMM, matching the per-document path. */
- if (ensure_late_state_buffer(ws, total_keep) != 0) {
- free(offsets);
- return -1;
- }
- const float *states = ws->base_ws->x;
- int pos = 0;
- for (int d = 0; d < n_docs; d++) {
- int start = offsets[d];
- for (int k = 0; k < n_keep[d]; k++) {
- memcpy(ws->states + (size_t)pos * hidden,
- states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
- pos++;
- }
- }
- free(offsets);
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
- hidden, dim);
- if (normalize) {
- for (int i = 0; i < total_keep; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
int embed_pool_batch(const embed_config_t *cfg,
const float *states,
const int *seq_lengths,