aboutsummaryrefslogtreecommitdiff
path: root/embed_cuda.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 10:40:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 10:40:44 +0200
commit6fd890a6ba42f9bd4683fc2ccd6243872b163e67 (patch)
tree035881de454f05ab86e1859364d16290e9f791e8 /embed_cuda.h
parent07d3b036de430fb86935e029ca03bcf6011a02e1 (diff)
downloadffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.tar.gz
ffwd-6fd890a6ba42f9bd4683fc2ccd6243872b163e67.zip
late: add CUDA late-interaction rerank path (GPU encode, host MaxSim)
Diffstat (limited to 'embed_cuda.h')
-rw-r--r--embed_cuda.h35
1 files changed, 35 insertions, 0 deletions
diff --git a/embed_cuda.h b/embed_cuda.h
index b3c9582..aad20ab 100644
--- a/embed_cuda.h
+++ b/embed_cuda.h
@@ -19,6 +19,7 @@
#include "embed.h"
typedef struct embed_cuda_ctx embed_cuda_ctx_t;
+typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t;
EMBED_API int embed_cuda_set_fast_gemm(const char *mode);
EMBED_API int embed_cuda_set_weights_bf16(int on);
@@ -26,6 +27,40 @@ EMBED_API int embed_cuda_set_weights_bf16(int on);
EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir);
EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
+/*
+ * Late-interaction CUDA path. Encoding (transformer forward + projection) runs
+ * on the GPU; MaxSim scoring stays on the host, where the grouped-GEMM scorer is
+ * much faster than a device graph for this small op. The encoders therefore copy
+ * the projected token vectors back to host memory.
+ */
+EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir);
+EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx);
+EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx);
+EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx);
+
+/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */
+EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
+
+/*
+ * Encode n_docs documents in one packed forward and pack each document's kept
+ * token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in
+ * document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts
+ * (the layout embed_late_maxsim_batch consumes). Returns 0 on success.
+ */
+EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
+
EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx);
EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,