diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 10:40:44 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 10:40:44 +0200 |
| commit | 6fd890a6ba42f9bd4683fc2ccd6243872b163e67 (patch) | |
| tree | 035881de454f05ab86e1859364d16290e9f791e8 | |
| parent | 07d3b036de430fb86935e029ca03bcf6011a02e1 (diff) | |
| download | ffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.tar.gz ffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.zip | |
late: add CUDA late-interaction rerank path (GPU encode, host MaxSim)
| -rw-r--r-- | embed.c | 10 | ||||
| -rw-r--r-- | embed_cuda.cu | 223 | ||||
| -rw-r--r-- | embed_cuda.h | 35 | ||||
| -rw-r--r-- | embed_internal.h | 7 | ||||
| -rw-r--r-- | embed_server.c | 84 |
5 files changed, 346 insertions, 13 deletions
@@ -906,6 +906,16 @@ int embed_late_model_token_dim(const embed_late_model_t *model) { return model ? model->token_dim : 0; } +/* Internal (embed_internal.h): expose the base model and projection so GPU + * backends can upload them to device without seeing struct embed_late_model. */ +embed_model_t *embed_late_model_base(const embed_late_model_t *model) { + return model ? model->base : NULL; +} + +const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) { + return model ? &model->projection : NULL; +} + int embed_late_model_encode_tokens(const embed_late_model_t *model, embed_late_workspace_t *ws, const int *token_ids, diff --git a/embed_cuda.cu b/embed_cuda.cu index 3955907..ba78f09 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -32,6 +32,7 @@ typedef struct { struct embed_cuda_ctx { embed_model_t *cpu; + int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it embed_config_t config; cudaStream_t stream; cublasHandle_t blas; @@ -1099,17 +1100,22 @@ static void free_layer(cuda_layer_t *l) { cuda_matrix_free(&l->down_proj); } -embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { - embed_model_t *cpu = embed_model_load(model_dir); +/* Build a device context from an already-loaded CPU model. own_cpu records + * whether embed_cuda_free should release that CPU model: embed_cuda_load owns + * the model it loaded; the late path passes a model owned by its CPU late + * model and sets own_cpu = 0. */ +static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) { if (!cpu) return NULL; embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx)); if (!ctx) { - embed_model_free(cpu); + if (own_cpu) + embed_model_free(cpu); return NULL; } ctx->cpu = cpu; + ctx->own_cpu = own_cpu; ctx->config = cpu->config; /* Without an explicit --cuda-weight-dtype, store weights in the model @@ -1152,6 +1158,10 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { return ctx; } +embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { + return cuda_ctx_from_model(embed_model_load(model_dir), 1); +} + void embed_cuda_free(embed_cuda_ctx_t *ctx) { if (!ctx) return; @@ -1187,7 +1197,8 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) { cublasDestroy(ctx->blas); if (ctx->stream) cudaStreamDestroy(ctx->stream); - embed_model_free(ctx->cpu); + if (ctx->own_cpu) + embed_model_free(ctx->cpu); free(ctx); } @@ -1501,3 +1512,207 @@ float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_toke } return out; } + +/* ======================================================================== + * Late-interaction CUDA path + * + * The transformer forward and the 1_Dense projection run on the GPU; MaxSim + * scoring stays on the host, where the grouped-GEMM scorer is far faster than a + * device graph for this small op. So the encoders copy the projected token + * vectors back to host memory and the caller runs embed_late_maxsim_batch there. + * ======================================================================== */ + +struct embed_cuda_late_ctx { + embed_cuda_ctx_t *base; // device base model; does not own its CPU model + embed_late_model_t *cpu_late; // owns the CPU base model + projection host data + cuda_matrix_t projection; // device [token_dim, hidden] + float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand + size_t proj_cap; // capacity of proj_dev in floats + int token_dim; +}; + +embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) { + embed_late_model_t *cpu_late = embed_late_model_load(model_dir); + if (!cpu_late) + return NULL; + + embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx)); + if (!ctx) { + embed_late_model_free(cpu_late); + return NULL; + } + ctx->cpu_late = cpu_late; + ctx->token_dim = embed_late_model_token_dim(cpu_late); + + // Upload the base transformer weights; the CPU late model keeps ownership. + ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0); + if (!ctx->base) { + embed_cuda_late_free(ctx); + return NULL; + } + + // Upload the 1_Dense projection [token_dim, hidden]. + const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late); + if (!proj || ctx->token_dim <= 0 || + load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) { + embed_cuda_late_free(ctx); + return NULL; + } + return ctx; +} + +void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) { + if (!ctx) + return; + cuda_matrix_free(&ctx->projection); + cudaFree(ctx->proj_dev); + embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model + embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection + free(ctx); +} + +const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) { + return ctx ? &ctx->base->config : NULL; +} + +int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } + +static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) { + size_t need = (size_t)total * (size_t)ctx->token_dim; + if (need <= ctx->proj_cap) + return 0; + cudaFree(ctx->proj_dev); + ctx->proj_dev = NULL; + ctx->proj_cap = 0; + if (cudaMalloc((void **)&ctx->proj_dev, need * sizeof(float)) != cudaSuccess) + return -1; + ctx->proj_cap = need; + return 0; +} + +/* Forward all inputs, project every token to token_dim, and copy the packed + * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees). + * On success *proj_host and *total_out are set; base->offsets_host holds the + * packed per-document token offsets. */ +static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx, + const embed_input_t *inputs, + int n_docs, + float **proj_host, + int *total_out) { + embed_cuda_ctx_t *base = ctx->base; + const embed_config_t *c = &base->config; + if (cuda_forward_batch(base, inputs, n_docs) != 0) + return -1; + int total = base->offsets_host[n_docs]; + if (total <= 0 || ensure_late_proj(ctx, total) != 0) + return -1; + if (linear(base, &ctx->projection, base->x, ctx->proj_dev, total, c->hidden_size, + ctx->token_dim) != 0) + return -1; + float *host = (float *)malloc((size_t)total * (size_t)ctx->token_dim * sizeof(float)); + if (!host) + return -1; + cudaError_t ce = + cudaMemcpyAsync(host, ctx->proj_dev, (size_t)total * (size_t)ctx->token_dim * sizeof(float), + cudaMemcpyDeviceToHost, base->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(base->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda late output copy failed: %s\n", cudaGetErrorString(ce)); + free(host); + return -1; + } + *proj_host = host; + *total_out = total; + return 0; +} + +int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0) + return -1; + embed_input_t input = {token_ids, n_tokens}; + float *proj_host = NULL; + int total = 0; + if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0) + return -1; + memcpy(out_vectors, proj_host, (size_t)total * (size_t)ctx->token_dim * sizeof(float)); + free(proj_host); + if (normalize) { + for (int i = 0; i < total; i++) + if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0) + return -1; + } + return 0; +} + +int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { + if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors || + !out_offsets || ctx->token_dim <= 0) + return -1; + int dim = ctx->token_dim; + + embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + if (!inputs) + return -1; + + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { + free(inputs); + return -1; + } + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { + free(inputs); + return -1; + } + } + inputs[d].ids = doc_ids[d]; + inputs[d].n_tokens = n_tokens[d]; + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + float *proj_host = NULL; + int total = 0; + if (cuda_late_forward_project(ctx, inputs, n_docs, &proj_host, &total) != 0) { + free(inputs); + return -1; + } + + /* Gather each document's kept token rows (in packed forward order) into the + * output, the layout MaxSim consumes - matching the CPU/MLX paths. */ + const int *off = ctx->base->offsets_host; + int pos = 0; + for (int d = 0; d < n_docs; d++) { + int start = off[d]; + for (int k = 0; k < n_keep[d]; k++) { + memcpy(out_vectors + (size_t)pos * dim, proj_host + (size_t)(start + keep[d][k]) * dim, + (size_t)dim * sizeof(float)); + pos++; + } + } + free(proj_host); + free(inputs); + + if (normalize) { + for (int i = 0; i < total_keep; i++) + if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + return 0; +} diff --git a/embed_cuda.h b/embed_cuda.h index b3c9582..aad20ab 100644 --- a/embed_cuda.h +++ b/embed_cuda.h @@ -19,6 +19,7 @@ #include "embed.h" typedef struct embed_cuda_ctx embed_cuda_ctx_t; +typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t; EMBED_API int embed_cuda_set_fast_gemm(const char *mode); EMBED_API int embed_cuda_set_weights_bf16(int on); @@ -26,6 +27,40 @@ EMBED_API int embed_cuda_set_weights_bf16(int on); EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir); EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx); +/* + * Late-interaction CUDA path. Encoding (transformer forward + projection) runs + * on the GPU; MaxSim scoring stays on the host, where the grouped-GEMM scorer is + * much faster than a device graph for this small op. The encoders therefore copy + * the projected token vectors back to host memory. + */ +EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir); +EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx); +EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx); +EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx); + +/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */ +EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors); + +/* + * Encode n_docs documents in one packed forward and pack each document's kept + * token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in + * document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts + * (the layout embed_late_maxsim_batch consumes). Returns 0 on success. + */ +EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets); + EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx); EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, diff --git a/embed_internal.h b/embed_internal.h index d2c1a30..7d6977c 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -81,4 +81,11 @@ struct embed_workspace { size_t bf16_widen_count; }; +/* Internal accessors for the late-interaction model. GPU backends build a device + * context from an already-loaded CPU late model because the model dir is a late + * snapshot that the pooled loader rejects; these expose the base model and the + * projection weight without making struct embed_late_model public. */ +embed_model_t *embed_late_model_base(const embed_late_model_t *model); +const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model); + #endif /* EMBED_INTERNAL_H */ diff --git a/embed_server.c b/embed_server.c index 15dd335..5224282 100644 --- a/embed_server.c +++ b/embed_server.c @@ -306,6 +306,7 @@ typedef struct { #endif #ifdef USE_CUDA embed_cuda_ctx_t *cuda_ctx; + embed_cuda_late_ctx_t *cuda_late_ctx; #endif } loaded_model; @@ -2437,6 +2438,50 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { } #endif +#ifdef USE_CUDA +static int execute_rerank_cuda(rerank_request *r, float *scores) { + loaded_model *m = r->model; + int dim = m->info->token_dim; + int total_doc_vecs = 0; + for (int i = 0; i < r->n_documents; i++) + total_doc_vecs += r->documents[i].n_keep; + + float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); + float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + + /* GPU encodes the query and every candidate (one packed forward); MaxSim + * runs on the host, where the grouped-GEMM scorer beats a device graph. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + + int rc = + embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query); + if (rc == 0) + rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep, + r->n_documents, 1, docs, offsets); + if (rc == 0) + rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, + scores); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); + free(offsets); + free(docs); + free(query); + return rc; +} +#endif + static void execute_rerank_request(rerank_request *r) { if (!r || !r->model) { if (r && r->j) @@ -2445,8 +2490,11 @@ static void execute_rerank_request(rerank_request *r) { } float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); uint64_t t0 = nstime(); -#ifdef USE_MLX +#if defined(USE_MLX) int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); +#elif defined(USE_CUDA) + int rc = + r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores); #else int rc = execute_rerank_cpu(r, scores); #endif @@ -2721,13 +2769,26 @@ static void *worker_main(void *arg) { loaded_model *m = &s->models[i]; if (!m->info) continue; - m->cuda_ctx = embed_cuda_load(m->path); - if (!m->cuda_ctx) { + if (m->info->kind == MODEL_KIND_LATE) + m->cuda_late_ctx = embed_cuda_late_load(m->path); + else + m->cuda_ctx = embed_cuda_load(m->path); + if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) || + (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) { server_log("embed-server: failed to load CUDA model on worker: %s", m->path); rc = 1; break; } - if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) { + const embed_config_t *config = m->info->kind == MODEL_KIND_LATE + ? embed_cuda_late_config(m->cuda_late_ctx) + : embed_cuda_config(m->cuda_ctx); + int token_dim = + m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0; + if (configure_loaded_model(m, config, token_dim) != 0 || + (m->info->kind == MODEL_KIND_LATE && + (m->late_mask_id >= config->vocab_size || + m->late_query_prefix_id >= config->vocab_size || + m->late_document_prefix_id >= config->vocab_size))) { rc = 1; break; } @@ -2746,6 +2807,10 @@ static void *worker_main(void *arg) { embed_cuda_free(m->cuda_ctx); m->cuda_ctx = NULL; } + if (m->cuda_late_ctx) { + embed_cuda_late_free(m->cuda_late_ctx); + m->cuda_late_ctx = NULL; + } } return NULL; } @@ -2788,6 +2853,10 @@ static void *worker_main(void *arg) { embed_cuda_free(m->cuda_ctx); m->cuda_ctx = NULL; } + if (m->cuda_late_ctx) { + embed_cuda_late_free(m->cuda_late_ctx); + m->cuda_late_ctx = NULL; + } } } #endif @@ -2934,6 +3003,8 @@ static void free_models(http_server *s) { #ifdef USE_CUDA if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx); + if (m->cuda_late_ctx) + embed_cuda_late_free(m->cuda_late_ctx); #endif } } @@ -2969,11 +3040,6 @@ static int validate_model_specs(const embed_server_config_t *cfg) { fprintf(stderr, "embed-server: duplicate model id: %s\n", spec->id); return -1; } - if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) { - fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n", - spec->id); - return -1; - } seen[slot] = 1; } return 0; |
