aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 09:45:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 09:45:22 +0200
commit43fdb308879fb0c61cb71553d9ac885c011d3a66 (patch)
treef33b036fa5ef8d8c069a5787cac684e85d54c17f
parent16d2bd83d24d45828fe039d593f5560a0eb8329f (diff)
downloadffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.tar.gz
ffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.zip
late: batch CPU rerank candidate encoding in one packed forward
-rw-r--r--embed.c92
-rw-r--r--embed.h19
-rw-r--r--embed_server.c43
-rw-r--r--tests/test_late.c86
4 files changed, 219 insertions, 21 deletions
diff --git a/embed.c b/embed.c
index d9e7936..56a9381 100644
--- a/embed.c
+++ b/embed.c
@@ -1494,6 +1494,98 @@ int embed_model_encode_batch(const embed_model_t *model,
return rc;
}
+/* Late-interaction batch encoder. Lives here, beside the packed/ragged batch
+ * machinery it reuses, rather than next to embed_late_model_encode_tokens. */
+int embed_late_model_encode_docs(const embed_late_model_t *model,
+ embed_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
+ if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
+ n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
+ return -1;
+
+ int hidden = model->base->config.hidden_size;
+ int dim = model->token_dim;
+
+ /* Pack all candidates for one block-diagonal forward. inputs/offsets index
+ * the full token stream; out_offsets is the prefix sum of kept tokens. */
+ embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
+ if (!inputs || !offsets) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+
+ int total_keep = 0;
+ out_offsets[0] = 0;
+ for (int d = 0; d < n_docs; d++) {
+ if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
+ n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ for (int k = 0; k < n_keep[d]; k++) {
+ if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ }
+ inputs[d].ids = doc_ids[d];
+ inputs[d].n_tokens = n_tokens[d];
+ total_keep += n_keep[d];
+ out_offsets[d + 1] = total_keep;
+ }
+
+ int total_seq = 0, max_seq = 0;
+ int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
+ if (rc == 0)
+ rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
+ max_seq, 1);
+ free(inputs);
+ if (rc != 0) {
+ free(offsets);
+ return -1;
+ }
+
+ /* Gather each document's kept hidden rows into ws->states (its natural
+ * [seq, hidden] shape), then project the packed [total_keep, hidden] block
+ * to [total_keep, dim] in one GEMM, matching the per-document path. */
+ if (ensure_late_state_buffer(ws, total_keep) != 0) {
+ free(offsets);
+ return -1;
+ }
+ const float *states = ws->base_ws->x;
+ int pos = 0;
+ for (int d = 0; d < n_docs; d++) {
+ int start = offsets[d];
+ for (int k = 0; k < n_keep[d]; k++) {
+ memcpy(ws->states + (size_t)pos * hidden,
+ states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
+ pos++;
+ }
+ }
+ free(offsets);
+
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
+ hidden, dim);
+ if (normalize) {
+ for (int i = 0; i < total_keep; i++) {
+ if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
int embed_pool_batch(const embed_config_t *cfg,
const float *states,
const int *seq_lengths,
diff --git a/embed.h b/embed.h
index 24e53e1..b84f71c 100644
--- a/embed.h
+++ b/embed.h
@@ -287,6 +287,25 @@ EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model,
float *out_vectors);
/*
+ * Encode n_docs documents in one block-diagonal forward and pack each
+ * document's kept token vectors back-to-back into out_vectors[total_keep,
+ * token_dim] in document order, where total_keep is the sum of n_keep.
+ * out_offsets[n_docs + 1] receives the prefix sum of kept counts, the layout
+ * embed_late_maxsim_batch consumes. keep[d] lists the n_keep[d] token indices to
+ * retain from document d. Returns 0 on success, -1 on error.
+ */
+EMBED_API int embed_late_model_encode_docs(const embed_late_model_t *model,
+ embed_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
+
+/*
* Sum over query tokens of max dot product against document tokens.
*/
EMBED_API float embed_late_maxsim(const float *query_vectors,
diff --git a/embed_server.c b/embed_server.c
index bedd7b2..15dd335 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -2360,37 +2360,40 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) {
loaded_model *m = r->model;
int dim = m->info->token_dim;
int total_doc_vecs = 0;
- int max_doc_tokens = 0;
- for (int i = 0; i < r->n_documents; i++) {
+ for (int i = 0; i < r->n_documents; i++)
total_doc_vecs += r->documents[i].n_keep;
- if (r->documents[i].n_tokens > max_doc_tokens)
- max_doc_tokens = r->documents[i].n_tokens;
- }
float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
- float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all));
int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+
+ /* Encode every candidate in one block-diagonal forward. The encoder packs
+ * each document's kept token vectors back-to-back and fills offsets, so the
+ * cost of N short documents collapses to a single packed forward. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+
int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
r->query.n_tokens, 1, query);
- int pos = 0;
- offsets[0] = 0;
- for (int i = 0; rc == 0 && i < r->n_documents; i++) {
- late_tokens *doc = &r->documents[i];
- rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids,
- doc->n_tokens, 1, all);
- for (int k = 0; rc == 0 && k < doc->n_keep; k++) {
- memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim,
- (size_t)dim * sizeof(*docs));
- pos++;
- }
- offsets[i + 1] = pos;
- }
+ if (rc == 0)
+ rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens,
+ keep, n_keep, r->n_documents, 1, docs, offsets);
if (rc == 0)
rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
scores);
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
free(offsets);
- free(all);
free(docs);
free(query);
return rc;
diff --git a/tests/test_late.c b/tests/test_late.c
index e491596..661661c 100644
--- a/tests/test_late.c
+++ b/tests/test_late.c
@@ -71,8 +71,10 @@ int main(void) {
int *ids = NULL;
float *states = NULL, *raw = NULL, *vecs = NULL, *expected = NULL;
float *doc_vecs = NULL;
+ float *batched = NULL;
float *big_q = NULL, *big_d = NULL;
int *doc_ids[3] = {0};
+ int *keep_arrs[3] = {0};
char dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-late-test-XXXXXX",
@@ -276,6 +278,85 @@ int main(void) {
fprintf(stderr, "maxsim_batch accepted invalid arguments\n");
goto fail;
}
+
+ /* Batched encoder parity: encoding all three docs in one packed forward
+ * (keep = every token) must reproduce the per-document vectors and the
+ * same offsets. */
+ int batch_offsets[4] = {0};
+ batched = (float *)calloc((size_t)offsets[3] * TOKEN_DIM, sizeof(float));
+ if (!batched) {
+ fprintf(stderr, "allocation failure\n");
+ goto fail;
+ }
+ for (int i = 0; i < 3; i++) {
+ keep_arrs[i] = (int *)malloc((size_t)lens[i] * sizeof(int));
+ if (!keep_arrs[i]) {
+ fprintf(stderr, "allocation failure\n");
+ goto fail;
+ }
+ for (int t = 0; t < lens[i]; t++)
+ keep_arrs[i][t] = t;
+ }
+ if (embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
+ (const int *const *)keep_arrs, lens, 3, 1, batched,
+ batch_offsets) != 0) {
+ fprintf(stderr, "encode_docs failed\n");
+ goto fail;
+ }
+ for (int i = 0; i <= 3; i++) {
+ if (batch_offsets[i] != offsets[i]) {
+ fprintf(stderr, "encode_docs offsets[%d]: %d vs %d\n", i, batch_offsets[i],
+ offsets[i]);
+ goto fail;
+ }
+ }
+ if (max_abs_diff(batched, doc_vecs, (size_t)offsets[3] * TOKEN_DIM) > 0.0001f) {
+ fprintf(stderr, "encode_docs vs per-doc mismatch: %g\n",
+ max_abs_diff(batched, doc_vecs, (size_t)offsets[3] * TOKEN_DIM));
+ goto fail;
+ }
+
+ /* Keep filtering: drop doc 0's first token; the packed result must equal
+ * doc 0's per-document rows gathered from the kept indices. */
+ {
+ int keep0[8];
+ int nk0 = lens[0] - 1;
+ if (nk0 <= 0 || nk0 > (int)(sizeof(keep0) / sizeof(keep0[0])))
+ goto fail;
+ for (int t = 0; t < nk0; t++)
+ keep0[t] = t + 1;
+ const int *one_ids[1] = {doc_ids[0]};
+ const int *one_keep[1] = {keep0};
+ int one_len[1] = {lens[0]};
+ int one_nkeep[1] = {nk0};
+ int one_off[2] = {0};
+ float *filtered = (float *)calloc((size_t)nk0 * TOKEN_DIM, sizeof(float));
+ if (!filtered)
+ goto fail;
+ if (embed_late_model_encode_docs(late, ws, one_ids, one_len, one_keep, one_nkeep, 1, 1,
+ filtered, one_off) != 0 ||
+ one_off[1] != nk0 ||
+ max_abs_diff(filtered, doc_vecs + TOKEN_DIM, (size_t)nk0 * TOKEN_DIM) > 0.0001f) {
+ fprintf(stderr, "encode_docs keep-filter mismatch\n");
+ free(filtered);
+ goto fail;
+ }
+ free(filtered);
+ }
+
+ /* Invalid arguments must be rejected. */
+ if (embed_late_model_encode_docs(NULL, ws, (const int *const *)doc_ids, lens,
+ (const int *const *)keep_arrs, lens, 3, 1, batched,
+ batch_offsets) == 0 ||
+ embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
+ (const int *const *)keep_arrs, lens, 0, 1, batched,
+ batch_offsets) == 0 ||
+ embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
+ (const int *const *)keep_arrs, lens, 3, 1, NULL,
+ batch_offsets) == 0) {
+ fprintf(stderr, "encode_docs accepted invalid arguments\n");
+ goto fail;
+ }
}
/* One document larger than the similarity-GEMM token budget must score
@@ -309,9 +390,12 @@ int main(void) {
fail:
free(big_d);
free(big_q);
+ free(batched);
free(doc_vecs);
- for (int i = 0; i < 3; i++)
+ for (int i = 0; i < 3; i++) {
free(doc_ids[i]);
+ free(keep_arrs[i]);
+ }
free(expected);
free(vecs);
free(raw);