diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:45:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:45:22 +0200 |
| commit | 43fdb308879fb0c61cb71553d9ac885c011d3a66 (patch) | |
| tree | f33b036fa5ef8d8c069a5787cac684e85d54c17f /embed_server.c | |
| parent | 16d2bd83d24d45828fe039d593f5560a0eb8329f (diff) | |
| download | ffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.tar.gz ffwd-43fdb308879fb0c61cb71553d9ac885c011d3a66.zip | |
late: batch CPU rerank candidate encoding in one packed forward
Diffstat (limited to 'embed_server.c')
| -rw-r--r-- | embed_server.c | 43 |
1 files changed, 23 insertions, 20 deletions
diff --git a/embed_server.c b/embed_server.c index bedd7b2..15dd335 100644 --- a/embed_server.c +++ b/embed_server.c @@ -2360,37 +2360,40 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { loaded_model *m = r->model; int dim = m->info->token_dim; int total_doc_vecs = 0; - int max_doc_tokens = 0; - for (int i = 0; i < r->n_documents; i++) { + for (int i = 0; i < r->n_documents; i++) total_doc_vecs += r->documents[i].n_keep; - if (r->documents[i].n_tokens > max_doc_tokens) - max_doc_tokens = r->documents[i].n_tokens; - } float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + + /* Encode every candidate in one block-diagonal forward. The encoder packs + * each document's kept token vectors back-to-back and fills offsets, so the + * cost of N short documents collapses to a single packed forward. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens, 1, query); - int pos = 0; - offsets[0] = 0; - for (int i = 0; rc == 0 && i < r->n_documents; i++) { - late_tokens *doc = &r->documents[i]; - rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids, - doc->n_tokens, 1, all); - for (int k = 0; rc == 0 && k < doc->n_keep; k++) { - memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim, - (size_t)dim * sizeof(*docs)); - pos++; - } - offsets[i + 1] = pos; - } + if (rc == 0) + rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, + keep, n_keep, r->n_documents, 1, docs, offsets); if (rc == 0) rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, scores); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); free(offsets); - free(all); free(docs); free(query); return rc; |
