diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-08 17:57:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f (patch) | |
| tree | e007eff496ba92ca8a62890ea4843bd793341be3 /pplx_server.c | |
| parent | 884ef3007382451de2cb8801d33f7d0efc05f5b6 (diff) | |
| download | ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.tar.gz ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.zip | |
cuda: add server contextual path
Diffstat (limited to 'pplx_server.c')
| -rw-r--r-- | pplx_server.c | 75 |
1 files changed, 74 insertions, 1 deletions
diff --git a/pplx_server.c b/pplx_server.c index 85df0e5..9df4d5b 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -10,6 +10,9 @@ #ifdef USE_MLX #include "pplx_embed_mlx.h" #endif +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif #include "deps/ae/ae.h" #include "deps/ae/anet.h" @@ -230,6 +233,9 @@ typedef struct { #ifdef USE_MLX pplx_mlx_ctx_t *mlx_ctx; #endif +#ifdef USE_CUDA + pplx_cuda_ctx_t *cuda_ctx; +#endif } loaded_model; typedef struct client client; @@ -249,6 +255,7 @@ typedef struct { char *api_key; loaded_model models[4]; int use_mlx; + int use_cuda; int mlx_quantize_bits; int mlx_quantize_group_size; @@ -1152,6 +1159,10 @@ static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, if (m->mlx_ctx) return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_batch(m->cuda_ctx, inputs, batch, out); +#endif return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } @@ -1162,6 +1173,10 @@ static int model_embed_spans_batch(loaded_model *m, if (m->mlx_ctx) return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_spans_batch(m->cuda_ctx, inputs, batch, out); +#endif return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } @@ -1966,9 +1981,9 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { static void *worker_main(void *arg) { http_server *s = arg; + int rc = 0; #ifdef USE_MLX if (s->use_mlx) { - int rc = 0; for (int i = 0; i < 4; i++) { loaded_model *m = &s->models[i]; if (!m->info) continue; @@ -2009,6 +2024,44 @@ static void *worker_main(void *arg) { } } else #endif +#ifdef USE_CUDA + if (s->use_cuda) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (!m->info) continue; + m->cuda_ctx = pplx_cuda_load(m->path); + if (!m->cuda_ctx) { + server_log("pplx-serve: failed to load CUDA model on worker: %s", + m->path); + rc = 1; + break; + } + if (pplx_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, + pplx_cuda_config(m->cuda_ctx)->hidden_size); + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + return NULL; + } + } else +#endif { pthread_mutex_lock(&s->mu); s->worker_ready = 1; @@ -2045,6 +2098,17 @@ static void *worker_main(void *arg) { } } #endif +#ifdef USE_CUDA + if (s->use_cuda) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + } +#endif return NULL; } @@ -2088,6 +2152,11 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { return 0; } #endif +#ifdef USE_CUDA + if (s->use_cuda) { + return 0; + } +#endif m->cpu_model = pplx_model_load(path); if (!m->cpu_model) { server_log("pplx-serve: failed to load model: %s", path); @@ -2117,6 +2186,9 @@ static void free_models(http_server *s) { #ifdef USE_MLX if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx); #endif +#ifdef USE_CUDA + if (m->cuda_ctx) pplx_cuda_free(m->cuda_ctx); +#endif } } @@ -2239,6 +2311,7 @@ int pplx_run_server(const pplx_server_config_t *cfg) { ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; s.enable_cors = cfg->enable_cors; s.use_mlx = cfg->use_mlx; + s.use_cuda = cfg->use_cuda; s.mlx_quantize_bits = cfg->mlx_quantize_bits; s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; |
