diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /libffwd/backend_cuda.c | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'libffwd/backend_cuda.c')
| -rw-r--r-- | libffwd/backend_cuda.c | 172 |
1 files changed, 172 insertions, 0 deletions
diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c new file mode 100644 index 0000000..db39345 --- /dev/null +++ b/libffwd/backend_cuda.c @@ -0,0 +1,172 @@ +/* backend_cuda.c - NVIDIA GPU backend (CUDA), implements the engine API in ffwd.h. + * Linked only into CUDA builds. The context is owned by the inference thread: + * open records what to load, activate creates the context, worker_free destroys + * it. CUDA encodes on the device and scores MaxSim on the host. */ +#include "internal.h" +#include "cuda.h" +#include "platform.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct embed { + int is_late; + char *dir; + ffwd_cuda_ctx_t *ctx; + ffwd_cuda_late_ctx_t *late_ctx; +}; + +const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; } + +void ffwd_cli_help(FILE *f) { + fputs(" --gpu-gemm-mode MODE\n" + " GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" + " --gpu-weight-dtype DTYPE\n" + " GPU weight storage: f32 or bf16 (default: f32). bf16 halves\n" + " weight memory and uses BF16 tensor cores\n", + f); +} + +void ffwd_server_help(FILE *f) { + fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --gpu-weight-dtype DTYPE GPU weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n", + f); +} + +int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { + if (opts->gpu_gemm_mode && ffwd_cuda_set_fast_gemm(opts->gpu_gemm_mode) != 0) { + snprintf(err, errlen, "--gpu-gemm-mode must be f32, tf32, bf16, or 16f"); + return -1; + } + if (opts->gpu_weight_dtype) { + if (!strcmp(opts->gpu_weight_dtype, "bf16")) + ffwd_cuda_set_weights_bf16(1); + else if (!strcmp(opts->gpu_weight_dtype, "f32")) + ffwd_cuda_set_weights_bf16(0); + else { + snprintf(err, errlen, "--gpu-weight-dtype must be f32 or bf16"); + return -1; + } + } + if (ffwd_verbose >= 1) + fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL); + return 0; +} + +ffwd_t *ffwd_open( + const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)opts; + ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); + if (!b || !(b->dir = strdup(model_dir))) { + snprintf(err, errlen, "out of memory"); + free(b); + return NULL; + } + b->is_late = is_late; + return b; +} + +int ffwd_activate(ffwd_t *b, char *err, size_t errlen) { + if (b->is_late) + b->late_ctx = ffwd_cuda_late_load(b->dir); + else + b->ctx = ffwd_cuda_load(b->dir); + if (!b->ctx && !b->late_ctx) { + snprintf(err, errlen, "failed to load model"); + return -1; + } + return 0; +} + +void ffwd_worker_free(ffwd_t *b) { + if (!b) + return; + if (b->ctx) { + ffwd_cuda_free(b->ctx); + b->ctx = NULL; + } + if (b->late_ctx) { + ffwd_cuda_late_free(b->late_ctx); + b->late_ctx = NULL; + } +} + +void ffwd_free(ffwd_t *b) { + if (!b) + return; + ffwd_worker_free(b); + free(b->dir); + free(b); +} + +const ffwd_config_t *ffwd_config(const ffwd_t *b) { + return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx); +} + +int ffwd_token_dim(const ffwd_t *b) { + return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; +} + +int ffwd_uses_dense_batches(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out); +} + +int ffwd_encode_spans_batch(ffwd_t *b, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out); +} + +int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { + int dim = ffwd_cuda_late_token_dim(b->late_ctx); + int total_doc_vecs = 0; + for (int i = 0; i < in->n_docs; i++) + total_doc_vecs += in->doc_n_keep[i]; + + float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query)); + float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets)); + int rc = (query && docs && offsets) ? 0 : -1; + + /* GPU encodes the query and every candidate in one packed forward; MaxSim + * runs on the host, where the grouped-GEMM scorer beats a device graph. */ + if (rc == 0) + rc = + ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query); + if (rc == 0) + rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep, + in->doc_n_keep, in->n_docs, 1, docs, offsets); + if (rc == 0) + rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, + scores); + free(offsets); + free(docs); + free(query); + return rc; +} + +size_t ffwd_scratch_nbytes(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_default_batch_wait_us(void) { return 1000; } + +int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)paths; + (void)n_paths; + (void)opts; + (void)err; + (void)errlen; + return 0; +} |
