diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-08 17:57:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f (patch) | |
| tree | e007eff496ba92ca8a62890ea4843bd793341be3 | |
| parent | 884ef3007382451de2cb8801d33f7d0efc05f5b6 (diff) | |
| download | ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.tar.gz ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.zip | |
cuda: add server contextual path
| -rw-r--r-- | main.c | 8 | ||||
| -rw-r--r-- | pplx_embed_cuda.cu | 173 | ||||
| -rw-r--r-- | pplx_embed_cuda.h | 3 | ||||
| -rw-r--r-- | pplx_server.c | 75 | ||||
| -rw-r--r-- | pplx_server.h | 1 |
5 files changed, 239 insertions, 21 deletions
@@ -697,11 +697,6 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } - if (use_cuda) { - fprintf(stderr, "--backend cuda is not available in --serve mode yet\n"); - free_model_specs(&model_specs); - return 1; - } if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); free_model_specs(&model_specs); @@ -712,7 +707,7 @@ int main(int argc, char *argv[]) qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } else if (verbose >= 1) { - fprintf(stderr, "Using MLX GPU backend\n"); + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); } pplx_server_config_t scfg = { .models = model_specs.v, @@ -723,6 +718,7 @@ int main(int argc, char *argv[]) .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, + .use_cuda = use_cuda, .mlx_quantize_bits = mlx_quantize_bits, .mlx_quantize_group_size = mlx_quantize_group_size, .allow_memory_overcommit = allow_memory_overcommit, diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index 30fdca3..5a6d821 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -6,6 +6,7 @@ extern "C" { #include <cuda_runtime.h> #include <cublas_v2.h> +#include <limits.h> #include <math.h> #include <stdio.h> #include <stdlib.h> @@ -49,7 +50,10 @@ struct pplx_cuda_ctx { int *token_ids; int *offsets; int *positions; - int pooled_batch_cap; + int *span_starts; + int *span_lens; + int pooled_rows_cap; + int span_cap; }; #define CUDA_CHECK(expr) do { \ @@ -412,6 +416,22 @@ __global__ static void mean_pool_kernel(float *out, const float *x, } } +__global__ static void span_pool_kernel(float *out, const float *x, + const int *starts, const int *lens, + int n_spans, int hidden) +{ + int s = blockIdx.x; + int tid = threadIdx.x; + int start = starts[s]; + int len = lens[s]; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = 0; t < len; t++) + sum += x[(size_t)(start + t) * hidden + d]; + out[(size_t)s * hidden + d] = sum / (float)len; + } +} + static int launch_check(void) { cudaError_t e = cudaGetLastError(); @@ -489,13 +509,6 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); ctx->batch_cap = batch + 1; } - if (batch > ctx->pooled_batch_cap) { - cudaFree(ctx->pooled_out); - ctx->pooled_out = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, - (size_t)batch * c->hidden_size * sizeof(float))); - ctx->pooled_batch_cap = batch; - } if (max_seq > ctx->max_seq_cap) { cudaFree(ctx->rope_cos); cudaFree(ctx->rope_sin); @@ -537,6 +550,33 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se return 0; } +static int ensure_pooled_rows(pplx_cuda_ctx_t *ctx, int rows) +{ + if (rows <= ctx->pooled_rows_cap) return 0; + const pplx_config_t *c = &ctx->config; + cudaFree(ctx->pooled_out); + ctx->pooled_out = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, + (size_t)rows * c->hidden_size * sizeof(float))); + ctx->pooled_rows_cap = rows; + return 0; +} + +static int ensure_span_buffers(pplx_cuda_ctx_t *ctx, int n_spans) +{ + if (n_spans <= ctx->span_cap) return 0; + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); + ctx->span_starts = NULL; + ctx->span_lens = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, + (size_t)n_spans * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, + (size_t)n_spans * sizeof(int))); + ctx->span_cap = n_spans; + return 0; +} + static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src, const pplx_config_t *c) { @@ -628,6 +668,8 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx) cudaFree(ctx->token_ids); cudaFree(ctx->offsets); cudaFree(ctx->positions); + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); if (ctx->blas) cublasDestroy(ctx->blas); if (ctx->stream) cudaStreamDestroy(ctx->stream); pplx_model_free(ctx->cpu); @@ -639,10 +681,10 @@ const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx) return ctx ? &ctx->config : NULL; } -int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings) +static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch) { - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + if (!ctx || !inputs || batch <= 0) return -1; const pplx_config_t *c = &ctx->config; int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); if (!h_offsets) return -1; @@ -759,12 +801,24 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps); if (launch_check() != 0) return -1; + return 0; +} + +int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const pplx_config_t *c = &ctx->config; + if (cuda_forward_batch(ctx, inputs, batch) != 0) + return -1; + if (ensure_pooled_rows(ctx, batch) != 0) + return -1; mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>( ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); if (launch_check() != 0) return -1; - ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, - (size_t)batch * c->hidden_size * sizeof(float), - cudaMemcpyDeviceToHost, ctx->stream); + cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)batch * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); if (ce == cudaSuccess) ce = cudaStreamSynchronize(ctx->stream); if (ce != cudaSuccess) { @@ -774,6 +828,97 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, return 0; } +int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, + const pplx_context_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const pplx_config_t *c = &ctx->config; + int total_spans = 0; + int total_tokens = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans || + total_tokens > INT_MAX - n_tokens) + return -1; + for (int s = 0; s < input->n_spans; s++) { + int start = input->spans[s].start; + int len = input->spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || + len > n_tokens - start) + return -1; + } + total_spans += input->n_spans; + total_tokens += n_tokens; + } + + pplx_input_t *packed = (pplx_input_t *)malloc((size_t)batch * sizeof(*packed)); + int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int)); + int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int)); + if (!packed || !h_starts || !h_lens) { + free(packed); + free(h_starts); + free(h_lens); + return -1; + } + + int token_offset = 0; + int span_offset = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + packed[b] = input->input; + for (int s = 0; s < input->n_spans; s++) { + h_starts[span_offset] = token_offset + input->spans[s].start; + h_lens[span_offset] = input->spans[s].n_tokens; + span_offset++; + } + token_offset += input->input.n_tokens; + } + + int rc = -1; + cudaError_t ce; + if (cuda_forward_batch(ctx, packed, batch) != 0) + goto cleanup; + if (ensure_pooled_rows(ctx, total_spans) != 0 || + ensure_span_buffers(ctx, total_spans) != 0) + goto cleanup; + ce = cudaMemcpyAsync(ctx->span_starts, h_starts, + (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->span_lens, h_lens, + (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce)); + goto cleanup; + } + span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>( + ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, + total_spans, c->hidden_size); + if (launch_check() != 0) + goto cleanup; + ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)total_spans * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span output copy failed: %s\n", + cudaGetErrorString(ce)); + goto cleanup; + } + rc = 0; + +cleanup: + free(h_lens); + free(h_starts); + free(packed); + return rc; +} + int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding) { diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h index af90230..ec57c8e 100644 --- a/pplx_embed_cuda.h +++ b/pplx_embed_cuda.h @@ -20,5 +20,8 @@ float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, int n_tokens); int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, int batch, float *out_embeddings); +int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, + const pplx_context_input_t *inputs, + int batch, float *out_embeddings); #endif /* PPLX_EMBED_CUDA_H */ diff --git a/pplx_server.c b/pplx_server.c index 85df0e5..9df4d5b 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -10,6 +10,9 @@ #ifdef USE_MLX #include "pplx_embed_mlx.h" #endif +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif #include "deps/ae/ae.h" #include "deps/ae/anet.h" @@ -230,6 +233,9 @@ typedef struct { #ifdef USE_MLX pplx_mlx_ctx_t *mlx_ctx; #endif +#ifdef USE_CUDA + pplx_cuda_ctx_t *cuda_ctx; +#endif } loaded_model; typedef struct client client; @@ -249,6 +255,7 @@ typedef struct { char *api_key; loaded_model models[4]; int use_mlx; + int use_cuda; int mlx_quantize_bits; int mlx_quantize_group_size; @@ -1152,6 +1159,10 @@ static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, if (m->mlx_ctx) return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_batch(m->cuda_ctx, inputs, batch, out); +#endif return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } @@ -1162,6 +1173,10 @@ static int model_embed_spans_batch(loaded_model *m, if (m->mlx_ctx) return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_spans_batch(m->cuda_ctx, inputs, batch, out); +#endif return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } @@ -1966,9 +1981,9 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { static void *worker_main(void *arg) { http_server *s = arg; + int rc = 0; #ifdef USE_MLX if (s->use_mlx) { - int rc = 0; for (int i = 0; i < 4; i++) { loaded_model *m = &s->models[i]; if (!m->info) continue; @@ -2009,6 +2024,44 @@ static void *worker_main(void *arg) { } } else #endif +#ifdef USE_CUDA + if (s->use_cuda) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (!m->info) continue; + m->cuda_ctx = pplx_cuda_load(m->path); + if (!m->cuda_ctx) { + server_log("pplx-serve: failed to load CUDA model on worker: %s", + m->path); + rc = 1; + break; + } + if (pplx_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, + pplx_cuda_config(m->cuda_ctx)->hidden_size); + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + return NULL; + } + } else +#endif { pthread_mutex_lock(&s->mu); s->worker_ready = 1; @@ -2045,6 +2098,17 @@ static void *worker_main(void *arg) { } } #endif +#ifdef USE_CUDA + if (s->use_cuda) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + } +#endif return NULL; } @@ -2088,6 +2152,11 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { return 0; } #endif +#ifdef USE_CUDA + if (s->use_cuda) { + return 0; + } +#endif m->cpu_model = pplx_model_load(path); if (!m->cpu_model) { server_log("pplx-serve: failed to load model: %s", path); @@ -2117,6 +2186,9 @@ static void free_models(http_server *s) { #ifdef USE_MLX if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) pplx_cuda_free(m->cuda_ctx); +#endif } } @@ -2239,6 +2311,7 @@ int pplx_run_server(const pplx_server_config_t *cfg) { ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; s.enable_cors = cfg->enable_cors; s.use_mlx = cfg->use_mlx; + s.use_cuda = cfg->use_cuda; s.mlx_quantize_bits = cfg->mlx_quantize_bits; s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; diff --git a/pplx_server.h b/pplx_server.h index 72d4a6c..225a7ac 100644 --- a/pplx_server.h +++ b/pplx_server.h @@ -17,6 +17,7 @@ typedef struct { int max_batch_tokens; int batch_wait_us; int use_mlx; + int use_cuda; int mlx_quantize_bits; int mlx_quantize_group_size; int allow_memory_overcommit; |
