diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:45:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:45:22 +0200 |
| commit | 43fdb308879fb0c61cb71553d9ac885c011d3a66 (patch) | |
| tree | f33b036fa5ef8d8c069a5787cac684e85d54c17f | |
| parent | 16d2bd83d24d45828fe039d593f5560a0eb8329f (diff) | |
| download | ffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.tar.gz ffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.zip | |
late: batch CPU rerank candidate encoding in one packed forward
| -rw-r--r-- | embed.c | 92 | ||||
| -rw-r--r-- | embed.h | 19 | ||||
| -rw-r--r-- | embed_server.c | 43 | ||||
| -rw-r--r-- | tests/test_late.c | 86 |
4 files changed, 219 insertions, 21 deletions
@@ -1494,6 +1494,98 @@ int embed_model_encode_batch(const embed_model_t *model, return rc; } +/* Late-interaction batch encoder. Lives here, beside the packed/ragged batch + * machinery it reuses, rather than next to embed_late_model_encode_tokens. */ +int embed_late_model_encode_docs(const embed_late_model_t *model, + embed_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { + if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep || + n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0) + return -1; + + int hidden = model->base->config.hidden_size; + int dim = model->token_dim; + + /* Pack all candidates for one block-diagonal forward. inputs/offsets index + * the full token stream; out_offsets is the prefix sum of kept tokens. */ + embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets)); + if (!inputs || !offsets) { + free(inputs); + free(offsets); + return -1; + } + + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { + free(inputs); + free(offsets); + return -1; + } + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { + free(inputs); + free(offsets); + return -1; + } + } + inputs[d].ids = doc_ids[d]; + inputs[d].n_tokens = n_tokens[d]; + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + int total_seq = 0, max_seq = 0; + int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq); + if (rc == 0) + rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq, + max_seq, 1); + free(inputs); + if (rc != 0) { + free(offsets); + return -1; + } + + /* Gather each document's kept hidden rows into ws->states (its natural + * [seq, hidden] shape), then project the packed [total_keep, hidden] block + * to [total_keep, dim] in one GEMM, matching the per-document path. */ + if (ensure_late_state_buffer(ws, total_keep) != 0) { + free(offsets); + return -1; + } + const float *states = ws->base_ws->x; + int pos = 0; + for (int d = 0; d < n_docs; d++) { + int start = offsets[d]; + for (int k = 0; k < n_keep[d]; k++) { + memcpy(ws->states + (size_t)pos * hidden, + states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float)); + pos++; + } + } + free(offsets); + + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep, + hidden, dim); + if (normalize) { + for (int i = 0; i < total_keep; i++) { + if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} + int embed_pool_batch(const embed_config_t *cfg, const float *states, const int *seq_lengths, @@ -287,6 +287,25 @@ EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model, float *out_vectors); /* + * Encode n_docs documents in one block-diagonal forward and pack each + * document's kept token vectors back-to-back into out_vectors[total_keep, + * token_dim] in document order, where total_keep is the sum of n_keep. + * out_offsets[n_docs + 1] receives the prefix sum of kept counts, the layout + * embed_late_maxsim_batch consumes. keep[d] lists the n_keep[d] token indices to + * retain from document d. Returns 0 on success, -1 on error. + */ +EMBED_API int embed_late_model_encode_docs(const embed_late_model_t *model, + embed_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets); + +/* * Sum over query tokens of max dot product against document tokens. */ EMBED_API float embed_late_maxsim(const float *query_vectors, diff --git a/embed_server.c b/embed_server.c index bedd7b2..15dd335 100644 --- a/embed_server.c +++ b/embed_server.c @@ -2360,37 +2360,40 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { loaded_model *m = r->model; int dim = m->info->token_dim; int total_doc_vecs = 0; - int max_doc_tokens = 0; - for (int i = 0; i < r->n_documents; i++) { + for (int i = 0; i < r->n_documents; i++) total_doc_vecs += r->documents[i].n_keep; - if (r->documents[i].n_tokens > max_doc_tokens) - max_doc_tokens = r->documents[i].n_tokens; - } float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + + /* Encode every candidate in one block-diagonal forward. The encoder packs + * each document's kept token vectors back-to-back and fills offsets, so the + * cost of N short documents collapses to a single packed forward. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens, 1, query); - int pos = 0; - offsets[0] = 0; - for (int i = 0; rc == 0 && i < r->n_documents; i++) { - late_tokens *doc = &r->documents[i]; - rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids, - doc->n_tokens, 1, all); - for (int k = 0; rc == 0 && k < doc->n_keep; k++) { - memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim, - (size_t)dim * sizeof(*docs)); - pos++; - } - offsets[i + 1] = pos; - } + if (rc == 0) + rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, + keep, n_keep, r->n_documents, 1, docs, offsets); if (rc == 0) rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, scores); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); free(offsets); - free(all); free(docs); free(query); return rc; diff --git a/tests/test_late.c b/tests/test_late.c index e491596..661661c 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -71,8 +71,10 @@ int main(void) { int *ids = NULL; float *states = NULL, *raw = NULL, *vecs = NULL, *expected = NULL; float *doc_vecs = NULL; + float *batched = NULL; float *big_q = NULL, *big_d = NULL; int *doc_ids[3] = {0}; + int *keep_arrs[3] = {0}; char dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-late-test-XXXXXX", @@ -276,6 +278,85 @@ int main(void) { fprintf(stderr, "maxsim_batch accepted invalid arguments\n"); goto fail; } + + /* Batched encoder parity: encoding all three docs in one packed forward + * (keep = every token) must reproduce the per-document vectors and the + * same offsets. */ + int batch_offsets[4] = {0}; + batched = (float *)calloc((size_t)offsets[3] * TOKEN_DIM, sizeof(float)); + if (!batched) { + fprintf(stderr, "allocation failure\n"); + goto fail; + } + for (int i = 0; i < 3; i++) { + keep_arrs[i] = (int *)malloc((size_t)lens[i] * sizeof(int)); + if (!keep_arrs[i]) { + fprintf(stderr, "allocation failure\n"); + goto fail; + } + for (int t = 0; t < lens[i]; t++) + keep_arrs[i][t] = t; + } + if (embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens, + (const int *const *)keep_arrs, lens, 3, 1, batched, + batch_offsets) != 0) { + fprintf(stderr, "encode_docs failed\n"); + goto fail; + } + for (int i = 0; i <= 3; i++) { + if (batch_offsets[i] != offsets[i]) { + fprintf(stderr, "encode_docs offsets[%d]: %d vs %d\n", i, batch_offsets[i], + offsets[i]); + goto fail; + } + } + if (max_abs_diff(batched, doc_vecs, (size_t)offsets[3] * TOKEN_DIM) > 0.0001f) { + fprintf(stderr, "encode_docs vs per-doc mismatch: %g\n", + max_abs_diff(batched, doc_vecs, (size_t)offsets[3] * TOKEN_DIM)); + goto fail; + } + + /* Keep filtering: drop doc 0's first token; the packed result must equal + * doc 0's per-document rows gathered from the kept indices. */ + { + int keep0[8]; + int nk0 = lens[0] - 1; + if (nk0 <= 0 || nk0 > (int)(sizeof(keep0) / sizeof(keep0[0]))) + goto fail; + for (int t = 0; t < nk0; t++) + keep0[t] = t + 1; + const int *one_ids[1] = {doc_ids[0]}; + const int *one_keep[1] = {keep0}; + int one_len[1] = {lens[0]}; + int one_nkeep[1] = {nk0}; + int one_off[2] = {0}; + float *filtered = (float *)calloc((size_t)nk0 * TOKEN_DIM, sizeof(float)); + if (!filtered) + goto fail; + if (embed_late_model_encode_docs(late, ws, one_ids, one_len, one_keep, one_nkeep, 1, 1, + filtered, one_off) != 0 || + one_off[1] != nk0 || + max_abs_diff(filtered, doc_vecs + TOKEN_DIM, (size_t)nk0 * TOKEN_DIM) > 0.0001f) { + fprintf(stderr, "encode_docs keep-filter mismatch\n"); + free(filtered); + goto fail; + } + free(filtered); + } + + /* Invalid arguments must be rejected. */ + if (embed_late_model_encode_docs(NULL, ws, (const int *const *)doc_ids, lens, + (const int *const *)keep_arrs, lens, 3, 1, batched, + batch_offsets) == 0 || + embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens, + (const int *const *)keep_arrs, lens, 0, 1, batched, + batch_offsets) == 0 || + embed_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens, + (const int *const *)keep_arrs, lens, 3, 1, NULL, + batch_offsets) == 0) { + fprintf(stderr, "encode_docs accepted invalid arguments\n"); + goto fail; + } } /* One document larger than the similarity-GEMM token budget must score @@ -309,9 +390,12 @@ int main(void) { fail: free(big_d); free(big_q); + free(batched); free(doc_vecs); - for (int i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) { free(doc_ids[i]); + free(keep_arrs[i]); + } free(expected); free(vecs); free(raw); |
