diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:23:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 09:23:02 +0200 |
| commit | 16d2bd83d24d45828fe039d593f5560a0eb8329f (patch) | |
| tree | a798598402969593daafb1d3f550145727209cb7 /embed_server.c | |
| parent | c4a1c97a09da6d8d88c2ac834bf6234506fc41bc (diff) | |
| download | ffwd-16d2bd83d24d45828fe039d593f5560a0eb8329f.tar.gz ffwd-16d2bd83d24d45828fe039d593f5560a0eb8329f.zip | |
late: batch MLX rerank candidate encoding in one padded pass
Diffstat (limited to 'embed_server.c')
| -rw-r--r-- | embed_server.c | 53 |
1 files changed, 23 insertions, 30 deletions
diff --git a/embed_server.c b/embed_server.c index 38da125..bedd7b2 100644 --- a/embed_server.c +++ b/embed_server.c @@ -2401,42 +2401,35 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { loaded_model *m = r->model; embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); - embed_mlx_late_vectors_t **docs = xcalloc((size_t)r->n_documents, sizeof(*docs)); - const embed_mlx_late_vectors_t **doc_refs = xcalloc((size_t)r->n_documents, sizeof(*doc_refs)); + + /* Encode every candidate in one padded transformer pass. The encoder packs + * each document's kept token vectors back-to-back and fills offsets, so the + * cost of N short documents collapses to a single batched forward. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - int rc = query ? 0 : -1; - offsets[0] = 0; - for (int i = 0; rc == 0 && i < r->n_documents; i++) { - late_tokens *doc = &r->documents[i]; - embed_mlx_late_vectors_t *all = - embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, doc->ids, doc->n_tokens, 1); - if (!all) { - rc = -1; - break; - } - docs[i] = embed_mlx_late_vectors_select(m->mlx_late_ctx, all, doc->keep, doc->n_keep); - embed_mlx_late_vectors_free(all); - if (!docs[i]) { - rc = -1; - break; - } - doc_refs[i] = docs[i]; - offsets[i + 1] = offsets[i] + doc->n_keep; + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; } embed_mlx_late_vectors_t *packed = - rc == 0 ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, r->n_documents) : NULL; - if (!packed) - rc = -1; - if (rc == 0) - rc = embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, - r->n_documents, scores); + query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep, + r->n_documents, 1, offsets) + : NULL; + int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, + r->n_documents, scores) + : -1; embed_mlx_late_vectors_free(packed); - for (int i = 0; i < r->n_documents; i++) - embed_mlx_late_vectors_free(docs[i]); embed_mlx_late_vectors_free(query); free(offsets); - free(doc_refs); - free(docs); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); return rc; } #endif |
