aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/embed.c417
-rw-r--r--src/engine.h60
-rw-r--r--src/late.c377
3 files changed, 467 insertions, 387 deletions
diff --git a/src/embed.c b/src/embed.c
index e265427..baa9d11 100644
--- a/src/embed.c
+++ b/src/embed.c
@@ -1,4 +1,5 @@
#include "embed.h"
+#include "engine.h"
#include "alloc.h"
#include "config.h"
#include "internal.h"
@@ -11,21 +12,6 @@
#include <string.h>
#include <math.h>
#include <limits.h>
-#include <float.h>
-
-struct embed_late_model {
- embed_model_t *base;
- safetensors_file_t *dense_sf;
- embed_weight_ref_t projection; /* [token_dim, hidden] */
- int token_dim;
-};
-
-struct embed_late_workspace {
- const embed_late_model_t *model;
- embed_workspace_t *base_ws;
- float *states;
- int states_seq_cap;
-};
/* ========================================================================
* Globals
@@ -52,23 +38,11 @@ static int model_dir_has_file(const char *model_dir, const char *rel) {
return 1;
}
-static int model_dir_has_late_projection(const char *model_dir) {
+int model_dir_has_late_projection(const char *model_dir) {
return model_dir_has_file(model_dir, "1_Dense/config.json") &&
model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
- if (!ws || !ws->model || n_tokens <= 0)
- return -1;
- if (n_tokens <= ws->states_seq_cap)
- return 0;
- int hidden = ws->model->base->config.hidden_size;
- if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
- return -1;
- ws->states_seq_cap = n_tokens;
- return 0;
-}
-
/* ========================================================================
* Weight loading (direct mmap pointers into safetensors)
* ======================================================================== */
@@ -103,12 +77,12 @@ static const char *dtype_name(safetensor_dtype_t dtype) {
}
}
-static int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim,
- int bf16_ok) {
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim,
+ int bf16_ok) {
if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) {
fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
@@ -217,13 +191,13 @@ static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
return 0;
}
-static void linear_nobias_weight(embed_workspace_t *ws,
- float *y,
- const float *x,
- const embed_weight_ref_t *w,
- int seq_len,
- int in_dim,
- int out_dim) {
+void linear_nobias_weight(embed_workspace_t *ws,
+ float *y,
+ const float *x,
+ const embed_weight_ref_t *w,
+ int seq_len,
+ int in_dim,
+ int out_dim) {
if (w->dtype == DTYPE_F32) {
embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
return;
@@ -500,7 +474,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
return 0;
}
-static embed_model_t *
+embed_model_t *
model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
if (!allow_late && model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_model_load: late-interaction models require "
@@ -722,245 +696,6 @@ const embed_config_t *embed_model_config(const embed_model_t *model) {
return model ? &model->config : NULL;
}
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
- if (!sf || !name)
- return NULL;
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0)
- return &sf->tensors[i];
- }
- return NULL;
-}
-
-embed_late_model_t *embed_late_model_load(const char *model_dir) {
- if (!model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base)
- goto fail;
-
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_late_model_load: model path too long\n");
- goto fail;
- }
-
- late->dense_sf = safetensors_open(path);
- if (!late->dense_sf) {
- fprintf(stderr, "embed_late_model_load: cannot open %s\n", path);
- goto fail;
- }
-
- const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
- if (!t) {
- fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
- fprintf(stderr, "embed_late_model_load: bad projection shape\n");
- goto fail;
- }
-
- int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
- goto fail;
-
- late->projection.dtype = t->dtype;
- late->projection.data = safetensors_data(late->dense_sf, t);
- late->token_dim = (int)t->shape[0];
- return late;
-
-fail:
- embed_late_model_free(late);
- return NULL;
-}
-
-void embed_late_model_free(embed_late_model_t *model) {
- if (!model)
- return;
- embed_model_free(model->base);
- safetensors_close(model->dense_sf);
- free(model);
-}
-
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
- if (!model || !model->base)
- return NULL;
- embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws)
- return NULL;
- ws->model = model;
- ws->base_ws = embed_workspace_new(model->base);
- if (!ws->base_ws) {
- free(ws);
- return NULL;
- }
- return ws;
-}
-
-void embed_late_workspace_free(embed_late_workspace_t *ws) {
- if (!ws)
- return;
- embed_workspace_free(ws->base_ws);
- free(ws->states);
- free(ws);
-}
-
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
- return model && model->base ? &model->base->config : NULL;
-}
-
-int embed_late_model_token_dim(const embed_late_model_t *model) {
- return model ? model->token_dim : 0;
-}
-
-/* Internal (embed_internal.h): expose the base model and projection so GPU
- * backends can upload them to device without seeing struct embed_late_model. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
- return model ? model->base : NULL;
-}
-
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
- return model ? &model->projection : NULL;
-}
-
-int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
- model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
- if (ensure_late_state_buffer(ws, n_tokens) != 0)
- return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
- return -1;
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
- dim);
- if (normalize) {
- for (int i = 0; i < n_tokens; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
-float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim) {
- if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
- return 0.0f;
-
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *q = query_vectors + (size_t)qi * dim;
- float best = -FLT_MAX;
- for (int di = 0; di < doc_tokens; di++) {
- const float *d = doc_vectors + (size_t)di * dim;
- float dot = embed_dot_f32(q, d, dim);
- if (dot > best)
- best = dot;
- }
- score += best;
- }
- return score;
-}
-
-int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores) {
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
- docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
- return -1;
-
- for (int i = 0; i < docs; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
- return -1;
- }
-
-#ifdef USE_BLAS
- /* One similarity GEMM per group of whole candidates:
- * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
- * row-max plus sum. The group token budget bounds the scratch matrix;
- * one oversized document still runs alone in a group of its own. */
- enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
- int total = doc_offsets[docs];
- int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
- int largest = 0;
- for (int i = 0; i < docs; i++) {
- int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest)
- largest = len;
- }
- if ((size_t)query_tokens * (size_t)largest > cap)
- cap = (size_t)query_tokens * (size_t)largest;
- float *sim = (float *)malloc(cap * sizeof(float));
- if (sim) {
- int doc = 0;
- while (doc < docs) {
- int first = doc;
- int start = doc_offsets[first];
- doc++;
- while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
- doc++;
- int group_tokens = doc_offsets[doc] - start;
-
- embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
- group_tokens);
-
- for (int i = first; i < doc; i++) {
- int s0 = doc_offsets[i] - start;
- int s1 = doc_offsets[i + 1] - start;
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *row = sim + (size_t)qi * group_tokens;
- float best = row[s0];
- for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best)
- best = row[t];
- score += best;
- }
- scores[i] = score;
- }
- }
- free(sim);
- return 0;
- }
- /* Allocation failure: fall through to the scalar path. */
-#endif
-
- for (int i = 0; i < docs; i++) {
- int start = doc_offsets[i];
- int end = doc_offsets[i + 1];
- scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim, end - start, dim);
- }
- return 0;
-}
-
size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
if (!ws)
return 0;
@@ -1028,12 +763,12 @@ size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
* Packed forward pass
* ======================================================================== */
-static int build_offsets(const embed_input_t *inputs,
- int batch,
- int require_ids,
- int *offsets,
- int *total_out,
- int *max_out) {
+int build_offsets(const embed_input_t *inputs,
+ int batch,
+ int require_ids,
+ int *offsets,
+ int *total_out,
+ int *max_out) {
if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
return -1;
@@ -1176,14 +911,14 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
return 0;
}
-static int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq,
- int apply_final_norm) {
+int forward_packed_inplace(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
+ int apply_final_norm) {
return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL,
0, model->config.n_layers, apply_final_norm);
}
@@ -1533,98 +1268,6 @@ int embed_model_encode_batch(const embed_model_t *model,
return rc;
}
-/* Late-interaction batch encoder. Lives here, beside the packed/ragged batch
- * machinery it reuses, rather than next to embed_late_model_encode_tokens. */
-int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
- if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
- n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
-
- /* Pack all candidates for one block-diagonal forward. inputs/offsets index
- * the full token stream; out_offsets is the prefix sum of kept tokens. */
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
- int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
- if (!inputs || !offsets) {
- free(inputs);
- free(offsets);
- return -1;
- }
-
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- }
- inputs[d].ids = doc_ids[d];
- inputs[d].n_tokens = n_tokens[d];
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- int total_seq = 0, max_seq = 0;
- int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
- if (rc == 0)
- rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
- max_seq, 1);
- free(inputs);
- if (rc != 0) {
- free(offsets);
- return -1;
- }
-
- /* Gather each document's kept hidden rows into ws->states (its natural
- * [seq, hidden] shape), then project the packed [total_keep, hidden] block
- * to [total_keep, dim] in one GEMM, matching the per-document path. */
- if (ensure_late_state_buffer(ws, total_keep) != 0) {
- free(offsets);
- return -1;
- }
- const float *states = ws->base_ws->x;
- int pos = 0;
- for (int d = 0; d < n_docs; d++) {
- int start = offsets[d];
- for (int k = 0; k < n_keep[d]; k++) {
- memcpy(ws->states + (size_t)pos * hidden,
- states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
- pos++;
- }
- }
- free(offsets);
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
- hidden, dim);
- if (normalize) {
- for (int i = 0; i < total_keep; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
int embed_pool_batch(const embed_config_t *cfg,
const float *states,
const int *seq_lengths,
diff --git a/src/engine.h b/src/engine.h
new file mode 100644
index 0000000..abdb0c7
--- /dev/null
+++ b/src/engine.h
@@ -0,0 +1,60 @@
+#ifndef EMBED_ENGINE_H
+#define EMBED_ENGINE_H
+
+#include "internal.h"
+#include "alloc.h"
+
+/* Cross-file declarations internal to the CPU engine. These functions are not
+ * part of the public API (embed.h) and carry no EMBED_API visibility; they are
+ * shared between the engine translation units (model loading, the forward pass,
+ * and late interaction) that used to live in one file. The frontends and the
+ * GPU backends do not include this header. */
+
+/* Validate that a safetensor matches an expected dtype/rank/shape and that its
+ * bytes lie within the mapped file. bf16_ok permits BF16 in addition to F32. */
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim,
+ int bf16_ok);
+
+/* True if model_dir carries a 1_Dense projection (a late-interaction model). */
+int model_dir_has_late_projection(const char *model_dir);
+
+/* Load (a layer range of) a model. allow_late lets the late loader accept a
+ * 1_Dense snapshot that the pooled loader rejects. layer_end == -1 means all. */
+embed_model_t *
+model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late);
+
+/* y[seq,out] = x[seq,in] @ w^T, no bias; dispatches F32/BF16 paths. Shared by
+ * the forward pass and the late projection. */
+void linear_nobias_weight(embed_workspace_t *ws,
+ float *y,
+ const float *x,
+ const embed_weight_ref_t *w,
+ int seq_len,
+ int in_dim,
+ int out_dim);
+
+/* Compute packed sequence offsets [batch+1] plus total and max sequence length
+ * over a batch of inputs. require_ids rejects inputs with a NULL id pointer. */
+int build_offsets(const embed_input_t *inputs,
+ int batch,
+ int require_ids,
+ int *offsets,
+ int *total_out,
+ int *max_out);
+
+/* Run the full transformer forward over a packed batch in ws, leaving hidden
+ * states in ws->x. Used by pooled encoding and by late-interaction encoding. */
+int forward_packed_inplace(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
+ int apply_final_norm);
+
+#endif /* EMBED_ENGINE_H */
diff --git a/src/late.c b/src/late.c
new file mode 100644
index 0000000..c3d9d3f
--- /dev/null
+++ b/src/late.c
@@ -0,0 +1,377 @@
+#include "embed.h"
+#include "engine.h"
+#include "kernels.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <float.h>
+
+/* ========================================================================
+ * Late-interaction model
+ *
+ * A late-interaction model is a pooled base model plus a 1_Dense linear
+ * projection from hidden states to token vectors. Instead of pooling to one
+ * vector, it keeps one projected vector per kept token and scores query/doc
+ * pairs with MaxSim. It reuses the base model's packed forward pass and
+ * workspace; only the projection, the token-vector buffer, and the scoring
+ * live here.
+ * ======================================================================== */
+
+struct embed_late_model {
+ embed_model_t *base;
+ safetensors_file_t *dense_sf;
+ embed_weight_ref_t projection; /* [token_dim, hidden] */
+ int token_dim;
+};
+
+struct embed_late_workspace {
+ const embed_late_model_t *model;
+ embed_workspace_t *base_ws;
+ float *states;
+ int states_seq_cap;
+};
+
+static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
+ if (!ws || !ws->model || n_tokens <= 0)
+ return -1;
+ if (n_tokens <= ws->states_seq_cap)
+ return 0;
+ int hidden = ws->model->base->config.hidden_size;
+ if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
+ return -1;
+ ws->states_seq_cap = n_tokens;
+ return 0;
+}
+
+static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
+ if (!sf || !name)
+ return NULL;
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0)
+ return &sf->tensors[i];
+ }
+ return NULL;
+}
+
+embed_late_model_t *embed_late_model_load(const char *model_dir) {
+ if (!model_dir_has_late_projection(model_dir)) {
+ fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
+ return NULL;
+ }
+
+ embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
+ if (!late)
+ return NULL;
+
+ late->base = model_load_range_ex(model_dir, 0, -1, 1);
+ if (!late->base)
+ goto fail;
+
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ fprintf(stderr, "embed_late_model_load: model path too long\n");
+ goto fail;
+ }
+
+ late->dense_sf = safetensors_open(path);
+ if (!late->dense_sf) {
+ fprintf(stderr, "embed_late_model_load: cannot open %s\n", path);
+ goto fail;
+ }
+
+ const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
+ if (!t) {
+ fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
+ goto fail;
+ }
+ if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
+ fprintf(stderr, "embed_late_model_load: bad projection shape\n");
+ goto fail;
+ }
+
+ int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
+ if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
+ goto fail;
+
+ late->projection.dtype = t->dtype;
+ late->projection.data = safetensors_data(late->dense_sf, t);
+ late->token_dim = (int)t->shape[0];
+ return late;
+
+fail:
+ embed_late_model_free(late);
+ return NULL;
+}
+
+void embed_late_model_free(embed_late_model_t *model) {
+ if (!model)
+ return;
+ embed_model_free(model->base);
+ safetensors_close(model->dense_sf);
+ free(model);
+}
+
+embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
+ if (!model || !model->base)
+ return NULL;
+ embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
+ if (!ws)
+ return NULL;
+ ws->model = model;
+ ws->base_ws = embed_workspace_new(model->base);
+ if (!ws->base_ws) {
+ free(ws);
+ return NULL;
+ }
+ return ws;
+}
+
+void embed_late_workspace_free(embed_late_workspace_t *ws) {
+ if (!ws)
+ return;
+ embed_workspace_free(ws->base_ws);
+ free(ws->states);
+ free(ws);
+}
+
+const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
+ return model && model->base ? &model->base->config : NULL;
+}
+
+int embed_late_model_token_dim(const embed_late_model_t *model) {
+ return model ? model->token_dim : 0;
+}
+
+/* Internal (embed_internal.h): expose the base model and projection so GPU
+ * backends can upload them to device without seeing struct embed_late_model. */
+embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
+ return model ? model->base : NULL;
+}
+
+const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
+ return model ? &model->projection : NULL;
+}
+
+int embed_late_model_encode_tokens(const embed_late_model_t *model,
+ embed_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
+ model->token_dim <= 0)
+ return -1;
+
+ int hidden = model->base->config.hidden_size;
+ int dim = model->token_dim;
+ if (ensure_late_state_buffer(ws, n_tokens) != 0)
+ return -1;
+ if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
+ return -1;
+
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
+ dim);
+ if (normalize) {
+ for (int i = 0; i < n_tokens; i++) {
+ if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
+float embed_late_maxsim(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ int doc_tokens,
+ int dim) {
+ if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
+ return 0.0f;
+
+ float score = 0.0f;
+ for (int qi = 0; qi < query_tokens; qi++) {
+ const float *q = query_vectors + (size_t)qi * dim;
+ float best = -FLT_MAX;
+ for (int di = 0; di < doc_tokens; di++) {
+ const float *d = doc_vectors + (size_t)di * dim;
+ float dot = embed_dot_f32(q, d, dim);
+ if (dot > best)
+ best = dot;
+ }
+ score += best;
+ }
+ return score;
+}
+
+int embed_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores) {
+ if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
+ docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
+ return -1;
+
+ for (int i = 0; i < docs; i++) {
+ if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
+ return -1;
+ }
+
+#ifdef USE_BLAS
+ /* One similarity GEMM per group of whole candidates:
+ * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
+ * row-max plus sum. The group token budget bounds the scratch matrix;
+ * one oversized document still runs alone in a group of its own. */
+ enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
+ int total = doc_offsets[docs];
+ int budget = LATE_GROUP_TOKEN_BUDGET;
+ size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
+ int largest = 0;
+ for (int i = 0; i < docs; i++) {
+ int len = doc_offsets[i + 1] - doc_offsets[i];
+ if (len > largest)
+ largest = len;
+ }
+ if ((size_t)query_tokens * (size_t)largest > cap)
+ cap = (size_t)query_tokens * (size_t)largest;
+ float *sim = (float *)malloc(cap * sizeof(float));
+ if (sim) {
+ int doc = 0;
+ while (doc < docs) {
+ int first = doc;
+ int start = doc_offsets[first];
+ doc++;
+ while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
+ doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
+ doc++;
+ int group_tokens = doc_offsets[doc] - start;
+
+ embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
+ group_tokens);
+
+ for (int i = first; i < doc; i++) {
+ int s0 = doc_offsets[i] - start;
+ int s1 = doc_offsets[i + 1] - start;
+ float score = 0.0f;
+ for (int qi = 0; qi < query_tokens; qi++) {
+ const float *row = sim + (size_t)qi * group_tokens;
+ float best = row[s0];
+ for (int t = s0 + 1; t < s1; t++)
+ if (row[t] > best)
+ best = row[t];
+ score += best;
+ }
+ scores[i] = score;
+ }
+ }
+ free(sim);
+ return 0;
+ }
+ /* Allocation failure: fall through to the scalar path. */
+#endif
+
+ for (int i = 0; i < docs; i++) {
+ int start = doc_offsets[i];
+ int end = doc_offsets[i + 1];
+ scores[i] = embed_late_maxsim(query_vectors, query_tokens,
+ doc_vectors + (size_t)start * dim, end - start, dim);
+ }
+ return 0;
+}
+
+/* Late-interaction batch encoder. Lives here, beside the per-document encoder,
+ * and reuses the base model's packed/ragged batch forward pass. */
+int embed_late_model_encode_docs(const embed_late_model_t *model,
+ embed_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
+ if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
+ n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
+ return -1;
+
+ int hidden = model->base->config.hidden_size;
+ int dim = model->token_dim;
+
+ /* Pack all candidates for one block-diagonal forward. inputs/offsets index
+ * the full token stream; out_offsets is the prefix sum of kept tokens. */
+ embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
+ if (!inputs || !offsets) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+
+ int total_keep = 0;
+ out_offsets[0] = 0;
+ for (int d = 0; d < n_docs; d++) {
+ if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
+ n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ for (int k = 0; k < n_keep[d]; k++) {
+ if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ }
+ inputs[d].ids = doc_ids[d];
+ inputs[d].n_tokens = n_tokens[d];
+ total_keep += n_keep[d];
+ out_offsets[d + 1] = total_keep;
+ }
+
+ int total_seq = 0, max_seq = 0;
+ int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
+ if (rc == 0)
+ rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
+ max_seq, 1);
+ free(inputs);
+ if (rc != 0) {
+ free(offsets);
+ return -1;
+ }
+
+ /* Gather each document's kept hidden rows into ws->states (its natural
+ * [seq, hidden] shape), then project the packed [total_keep, hidden] block
+ * to [total_keep, dim] in one GEMM, matching the per-document path. */
+ if (ensure_late_state_buffer(ws, total_keep) != 0) {
+ free(offsets);
+ return -1;
+ }
+ const float *states = ws->base_ws->x;
+ int pos = 0;
+ for (int d = 0; d < n_docs; d++) {
+ int start = offsets[d];
+ for (int k = 0; k < n_keep[d]; k++) {
+ memcpy(ws->states + (size_t)pos * hidden,
+ states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
+ pos++;
+ }
+ }
+ free(offsets);
+
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
+ hidden, dim);
+ if (normalize) {
+ for (int i = 0; i < total_keep; i++) {
+ if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}