aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 10:40:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 10:40:44 +0200
commit6fd890a6ba42f9bd4683fc2ccd6243872b163e67 (patch)
tree035881de454f05ab86e1859364d16290e9f791e8
parent07d3b036de430fb86935e029ca03bcf6011a02e1 (diff)
downloadffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.tar.gz
ffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.zip
late: add CUDA late-interaction rerank path (GPU encode, host MaxSim)
-rw-r--r--embed.c10
-rw-r--r--embed_cuda.cu223
-rw-r--r--embed_cuda.h35
-rw-r--r--embed_internal.h7
-rw-r--r--embed_server.c84
5 files changed, 346 insertions, 13 deletions
diff --git a/embed.c b/embed.c
index 56a9381..645660c 100644
--- a/embed.c
+++ b/embed.c
@@ -906,6 +906,16 @@ int embed_late_model_token_dim(const embed_late_model_t *model) {
return model ? model->token_dim : 0;
}
+/* Internal (embed_internal.h): expose the base model and projection so GPU
+ * backends can upload them to device without seeing struct embed_late_model. */
+embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
+ return model ? model->base : NULL;
+}
+
+const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
+ return model ? &model->projection : NULL;
+}
+
int embed_late_model_encode_tokens(const embed_late_model_t *model,
embed_late_workspace_t *ws,
const int *token_ids,
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 3955907..ba78f09 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -32,6 +32,7 @@ typedef struct {
struct embed_cuda_ctx {
embed_model_t *cpu;
+ int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it
embed_config_t config;
cudaStream_t stream;
cublasHandle_t blas;
@@ -1099,17 +1100,22 @@ static void free_layer(cuda_layer_t *l) {
cuda_matrix_free(&l->down_proj);
}
-embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
- embed_model_t *cpu = embed_model_load(model_dir);
+/* Build a device context from an already-loaded CPU model. own_cpu records
+ * whether embed_cuda_free should release that CPU model: embed_cuda_load owns
+ * the model it loaded; the late path passes a model owned by its CPU late
+ * model and sets own_cpu = 0. */
+static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
if (!cpu)
return NULL;
embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx));
if (!ctx) {
- embed_model_free(cpu);
+ if (own_cpu)
+ embed_model_free(cpu);
return NULL;
}
ctx->cpu = cpu;
+ ctx->own_cpu = own_cpu;
ctx->config = cpu->config;
/* Without an explicit --cuda-weight-dtype, store weights in the model
@@ -1152,6 +1158,10 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
return ctx;
}
+embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
+ return cuda_ctx_from_model(embed_model_load(model_dir), 1);
+}
+
void embed_cuda_free(embed_cuda_ctx_t *ctx) {
if (!ctx)
return;
@@ -1187,7 +1197,8 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) {
cublasDestroy(ctx->blas);
if (ctx->stream)
cudaStreamDestroy(ctx->stream);
- embed_model_free(ctx->cpu);
+ if (ctx->own_cpu)
+ embed_model_free(ctx->cpu);
free(ctx);
}
@@ -1501,3 +1512,207 @@ float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_toke
}
return out;
}
+
+/* ========================================================================
+ * Late-interaction CUDA path
+ *
+ * The transformer forward and the 1_Dense projection run on the GPU; MaxSim
+ * scoring stays on the host, where the grouped-GEMM scorer is far faster than a
+ * device graph for this small op. So the encoders copy the projected token
+ * vectors back to host memory and the caller runs embed_late_maxsim_batch there.
+ * ======================================================================== */
+
+struct embed_cuda_late_ctx {
+ embed_cuda_ctx_t *base; // device base model; does not own its CPU model
+ embed_late_model_t *cpu_late; // owns the CPU base model + projection host data
+ cuda_matrix_t projection; // device [token_dim, hidden]
+ float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand
+ size_t proj_cap; // capacity of proj_dev in floats
+ int token_dim;
+};
+
+embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) {
+ embed_late_model_t *cpu_late = embed_late_model_load(model_dir);
+ if (!cpu_late)
+ return NULL;
+
+ embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ embed_late_model_free(cpu_late);
+ return NULL;
+ }
+ ctx->cpu_late = cpu_late;
+ ctx->token_dim = embed_late_model_token_dim(cpu_late);
+
+ // Upload the base transformer weights; the CPU late model keeps ownership.
+ ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0);
+ if (!ctx->base) {
+ embed_cuda_late_free(ctx);
+ return NULL;
+ }
+
+ // Upload the 1_Dense projection [token_dim, hidden].
+ const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late);
+ if (!proj || ctx->token_dim <= 0 ||
+ load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) {
+ embed_cuda_late_free(ctx);
+ return NULL;
+ }
+ return ctx;
+}
+
+void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) {
+ if (!ctx)
+ return;
+ cuda_matrix_free(&ctx->projection);
+ cudaFree(ctx->proj_dev);
+ embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model
+ embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection
+ free(ctx);
+}
+
+const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) {
+ return ctx ? &ctx->base->config : NULL;
+}
+
+int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
+
+static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) {
+ size_t need = (size_t)total * (size_t)ctx->token_dim;
+ if (need <= ctx->proj_cap)
+ return 0;
+ cudaFree(ctx->proj_dev);
+ ctx->proj_dev = NULL;
+ ctx->proj_cap = 0;
+ if (cudaMalloc((void **)&ctx->proj_dev, need * sizeof(float)) != cudaSuccess)
+ return -1;
+ ctx->proj_cap = need;
+ return 0;
+}
+
+/* Forward all inputs, project every token to token_dim, and copy the packed
+ * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees).
+ * On success *proj_host and *total_out are set; base->offsets_host holds the
+ * packed per-document token offsets. */
+static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int n_docs,
+ float **proj_host,
+ int *total_out) {
+ embed_cuda_ctx_t *base = ctx->base;
+ const embed_config_t *c = &base->config;
+ if (cuda_forward_batch(base, inputs, n_docs) != 0)
+ return -1;
+ int total = base->offsets_host[n_docs];
+ if (total <= 0 || ensure_late_proj(ctx, total) != 0)
+ return -1;
+ if (linear(base, &ctx->projection, base->x, ctx->proj_dev, total, c->hidden_size,
+ ctx->token_dim) != 0)
+ return -1;
+ float *host = (float *)malloc((size_t)total * (size_t)ctx->token_dim * sizeof(float));
+ if (!host)
+ return -1;
+ cudaError_t ce =
+ cudaMemcpyAsync(host, ctx->proj_dev, (size_t)total * (size_t)ctx->token_dim * sizeof(float),
+ cudaMemcpyDeviceToHost, base->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(base->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda late output copy failed: %s\n", cudaGetErrorString(ce));
+ free(host);
+ return -1;
+ }
+ *proj_host = host;
+ *total_out = total;
+ return 0;
+}
+
+int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0)
+ return -1;
+ embed_input_t input = {token_ids, n_tokens};
+ float *proj_host = NULL;
+ int total = 0;
+ if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0)
+ return -1;
+ memcpy(out_vectors, proj_host, (size_t)total * (size_t)ctx->token_dim * sizeof(float));
+ free(proj_host);
+ if (normalize) {
+ for (int i = 0; i < total; i++)
+ if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0)
+ return -1;
+ }
+ return 0;
+}
+
+int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
+ if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors ||
+ !out_offsets || ctx->token_dim <= 0)
+ return -1;
+ int dim = ctx->token_dim;
+
+ embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ if (!inputs)
+ return -1;
+
+ int total_keep = 0;
+ out_offsets[0] = 0;
+ for (int d = 0; d < n_docs; d++) {
+ if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
+ n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
+ free(inputs);
+ return -1;
+ }
+ for (int k = 0; k < n_keep[d]; k++) {
+ if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
+ free(inputs);
+ return -1;
+ }
+ }
+ inputs[d].ids = doc_ids[d];
+ inputs[d].n_tokens = n_tokens[d];
+ total_keep += n_keep[d];
+ out_offsets[d + 1] = total_keep;
+ }
+
+ float *proj_host = NULL;
+ int total = 0;
+ if (cuda_late_forward_project(ctx, inputs, n_docs, &proj_host, &total) != 0) {
+ free(inputs);
+ return -1;
+ }
+
+ /* Gather each document's kept token rows (in packed forward order) into the
+ * output, the layout MaxSim consumes - matching the CPU/MLX paths. */
+ const int *off = ctx->base->offsets_host;
+ int pos = 0;
+ for (int d = 0; d < n_docs; d++) {
+ int start = off[d];
+ for (int k = 0; k < n_keep[d]; k++) {
+ memcpy(out_vectors + (size_t)pos * dim, proj_host + (size_t)(start + keep[d][k]) * dim,
+ (size_t)dim * sizeof(float));
+ pos++;
+ }
+ }
+ free(proj_host);
+ free(inputs);
+
+ if (normalize) {
+ for (int i = 0; i < total_keep; i++)
+ if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ return 0;
+}
diff --git a/embed_cuda.h b/embed_cuda.h
index b3c9582..aad20ab 100644
--- a/embed_cuda.h
+++ b/embed_cuda.h
@@ -19,6 +19,7 @@
#include "embed.h"
typedef struct embed_cuda_ctx embed_cuda_ctx_t;
+typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t;
EMBED_API int embed_cuda_set_fast_gemm(const char *mode);
EMBED_API int embed_cuda_set_weights_bf16(int on);
@@ -26,6 +27,40 @@ EMBED_API int embed_cuda_set_weights_bf16(int on);
EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir);
EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
+/*
+ * Late-interaction CUDA path. Encoding (transformer forward + projection) runs
+ * on the GPU; MaxSim scoring stays on the host, where the grouped-GEMM scorer is
+ * much faster than a device graph for this small op. The encoders therefore copy
+ * the projected token vectors back to host memory.
+ */
+EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir);
+EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx);
+EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx);
+EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx);
+
+/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */
+EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
+
+/*
+ * Encode n_docs documents in one packed forward and pack each document's kept
+ * token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in
+ * document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts
+ * (the layout embed_late_maxsim_batch consumes). Returns 0 on success.
+ */
+EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
+
EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx);
EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
diff --git a/embed_internal.h b/embed_internal.h
index d2c1a30..7d6977c 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -81,4 +81,11 @@ struct embed_workspace {
size_t bf16_widen_count;
};
+/* Internal accessors for the late-interaction model. GPU backends build a device
+ * context from an already-loaded CPU late model because the model dir is a late
+ * snapshot that the pooled loader rejects; these expose the base model and the
+ * projection weight without making struct embed_late_model public. */
+embed_model_t *embed_late_model_base(const embed_late_model_t *model);
+const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model);
+
#endif /* EMBED_INTERNAL_H */
diff --git a/embed_server.c b/embed_server.c
index 15dd335..5224282 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -306,6 +306,7 @@ typedef struct {
#endif
#ifdef USE_CUDA
embed_cuda_ctx_t *cuda_ctx;
+ embed_cuda_late_ctx_t *cuda_late_ctx;
#endif
} loaded_model;
@@ -2437,6 +2438,50 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) {
}
#endif
+#ifdef USE_CUDA
+static int execute_rerank_cuda(rerank_request *r, float *scores) {
+ loaded_model *m = r->model;
+ int dim = m->info->token_dim;
+ int total_doc_vecs = 0;
+ for (int i = 0; i < r->n_documents; i++)
+ total_doc_vecs += r->documents[i].n_keep;
+
+ float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
+ float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
+ int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+
+ /* GPU encodes the query and every candidate (one packed forward); MaxSim
+ * runs on the host, where the grouped-GEMM scorer beats a device graph. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+
+ int rc =
+ embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query);
+ if (rc == 0)
+ rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep,
+ r->n_documents, 1, docs, offsets);
+ if (rc == 0)
+ rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
+ scores);
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
+ free(offsets);
+ free(docs);
+ free(query);
+ return rc;
+}
+#endif
+
static void execute_rerank_request(rerank_request *r) {
if (!r || !r->model) {
if (r && r->j)
@@ -2445,8 +2490,11 @@ static void execute_rerank_request(rerank_request *r) {
}
float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
uint64_t t0 = nstime();
-#ifdef USE_MLX
+#if defined(USE_MLX)
int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
+#elif defined(USE_CUDA)
+ int rc =
+ r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores);
#else
int rc = execute_rerank_cpu(r, scores);
#endif
@@ -2721,13 +2769,26 @@ static void *worker_main(void *arg) {
loaded_model *m = &s->models[i];
if (!m->info)
continue;
- m->cuda_ctx = embed_cuda_load(m->path);
- if (!m->cuda_ctx) {
+ if (m->info->kind == MODEL_KIND_LATE)
+ m->cuda_late_ctx = embed_cuda_late_load(m->path);
+ else
+ m->cuda_ctx = embed_cuda_load(m->path);
+ if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) ||
+ (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
server_log("embed-server: failed to load CUDA model on worker: %s", m->path);
rc = 1;
break;
}
- if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
+ const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
+ ? embed_cuda_late_config(m->cuda_late_ctx)
+ : embed_cuda_config(m->cuda_ctx);
+ int token_dim =
+ m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0;
+ if (configure_loaded_model(m, config, token_dim) != 0 ||
+ (m->info->kind == MODEL_KIND_LATE &&
+ (m->late_mask_id >= config->vocab_size ||
+ m->late_query_prefix_id >= config->vocab_size ||
+ m->late_document_prefix_id >= config->vocab_size))) {
rc = 1;
break;
}
@@ -2746,6 +2807,10 @@ static void *worker_main(void *arg) {
embed_cuda_free(m->cuda_ctx);
m->cuda_ctx = NULL;
}
+ if (m->cuda_late_ctx) {
+ embed_cuda_late_free(m->cuda_late_ctx);
+ m->cuda_late_ctx = NULL;
+ }
}
return NULL;
}
@@ -2788,6 +2853,10 @@ static void *worker_main(void *arg) {
embed_cuda_free(m->cuda_ctx);
m->cuda_ctx = NULL;
}
+ if (m->cuda_late_ctx) {
+ embed_cuda_late_free(m->cuda_late_ctx);
+ m->cuda_late_ctx = NULL;
+ }
}
}
#endif
@@ -2934,6 +3003,8 @@ static void free_models(http_server *s) {
#ifdef USE_CUDA
if (m->cuda_ctx)
embed_cuda_free(m->cuda_ctx);
+ if (m->cuda_late_ctx)
+ embed_cuda_late_free(m->cuda_late_ctx);
#endif
}
}
@@ -2969,11 +3040,6 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
fprintf(stderr, "embed-server: duplicate model id: %s\n", spec->id);
return -1;
}
- if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) {
- fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n",
- spec->id);
- return -1;
- }
seen[slot] = 1;
}
return 0;