aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-08 17:57:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitd9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f (patch)
treee007eff496ba92ca8a62890ea4843bd793341be3
parent884ef3007382451de2cb8801d33f7d0efc05f5b6 (diff)
downloadffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.tar.gz
ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.zip
cuda: add server contextual path
-rw-r--r--main.c8
-rw-r--r--pplx_embed_cuda.cu173
-rw-r--r--pplx_embed_cuda.h3
-rw-r--r--pplx_server.c75
-rw-r--r--pplx_server.h1
5 files changed, 239 insertions, 21 deletions
diff --git a/main.c b/main.c
index 7952538..f7711da 100644
--- a/main.c
+++ b/main.c
@@ -697,11 +697,6 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
- if (use_cuda) {
- fprintf(stderr, "--backend cuda is not available in --serve mode yet\n");
- free_model_specs(&model_specs);
- return 1;
- }
if (mlx_quantize_bits && !use_mlx) {
fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n");
free_model_specs(&model_specs);
@@ -712,7 +707,7 @@ int main(int argc, char *argv[])
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
} else if (verbose >= 1) {
- fprintf(stderr, "Using MLX GPU backend\n");
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
}
pplx_server_config_t scfg = {
.models = model_specs.v,
@@ -723,6 +718,7 @@ int main(int argc, char *argv[])
.max_batch_tokens = max_batch_tokens,
.batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
+ .use_cuda = use_cuda,
.mlx_quantize_bits = mlx_quantize_bits,
.mlx_quantize_group_size = mlx_quantize_group_size,
.allow_memory_overcommit = allow_memory_overcommit,
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu
index 30fdca3..5a6d821 100644
--- a/pplx_embed_cuda.cu
+++ b/pplx_embed_cuda.cu
@@ -6,6 +6,7 @@ extern "C" {
#include <cuda_runtime.h>
#include <cublas_v2.h>
+#include <limits.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
@@ -49,7 +50,10 @@ struct pplx_cuda_ctx {
int *token_ids;
int *offsets;
int *positions;
- int pooled_batch_cap;
+ int *span_starts;
+ int *span_lens;
+ int pooled_rows_cap;
+ int span_cap;
};
#define CUDA_CHECK(expr) do { \
@@ -412,6 +416,22 @@ __global__ static void mean_pool_kernel(float *out, const float *x,
}
}
+__global__ static void span_pool_kernel(float *out, const float *x,
+ const int *starts, const int *lens,
+ int n_spans, int hidden)
+{
+ int s = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = starts[s];
+ int len = lens[s];
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = 0; t < len; t++)
+ sum += x[(size_t)(start + t) * hidden + d];
+ out[(size_t)s * hidden + d] = sum / (float)len;
+ }
+}
+
static int launch_check(void)
{
cudaError_t e = cudaGetLastError();
@@ -489,13 +509,6 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
ctx->batch_cap = batch + 1;
}
- if (batch > ctx->pooled_batch_cap) {
- cudaFree(ctx->pooled_out);
- ctx->pooled_out = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out,
- (size_t)batch * c->hidden_size * sizeof(float)));
- ctx->pooled_batch_cap = batch;
- }
if (max_seq > ctx->max_seq_cap) {
cudaFree(ctx->rope_cos);
cudaFree(ctx->rope_sin);
@@ -537,6 +550,33 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
return 0;
}
+static int ensure_pooled_rows(pplx_cuda_ctx_t *ctx, int rows)
+{
+ if (rows <= ctx->pooled_rows_cap) return 0;
+ const pplx_config_t *c = &ctx->config;
+ cudaFree(ctx->pooled_out);
+ ctx->pooled_out = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out,
+ (size_t)rows * c->hidden_size * sizeof(float)));
+ ctx->pooled_rows_cap = rows;
+ return 0;
+}
+
+static int ensure_span_buffers(pplx_cuda_ctx_t *ctx, int n_spans)
+{
+ if (n_spans <= ctx->span_cap) return 0;
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
+ ctx->span_starts = NULL;
+ ctx->span_lens = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts,
+ (size_t)n_spans * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens,
+ (size_t)n_spans * sizeof(int)));
+ ctx->span_cap = n_spans;
+ return 0;
+}
+
static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src,
const pplx_config_t *c)
{
@@ -628,6 +668,8 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
cudaFree(ctx->token_ids);
cudaFree(ctx->offsets);
cudaFree(ctx->positions);
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
if (ctx->blas) cublasDestroy(ctx->blas);
if (ctx->stream) cudaStreamDestroy(ctx->stream);
pplx_model_free(ctx->cpu);
@@ -639,10 +681,10 @@ const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx)
return ctx ? &ctx->config : NULL;
}
-int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
- int batch, float *out_embeddings)
+static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch)
{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ if (!ctx || !inputs || batch <= 0) return -1;
const pplx_config_t *c = &ctx->config;
int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
if (!h_offsets) return -1;
@@ -759,12 +801,24 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps);
if (launch_check() != 0) return -1;
+ return 0;
+}
+
+int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const pplx_config_t *c = &ctx->config;
+ if (cuda_forward_batch(ctx, inputs, batch) != 0)
+ return -1;
+ if (ensure_pooled_rows(ctx, batch) != 0)
+ return -1;
mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
if (launch_check() != 0) return -1;
- ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
- (size_t)batch * c->hidden_size * sizeof(float),
- cudaMemcpyDeviceToHost, ctx->stream);
+ cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)batch * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
if (ce == cudaSuccess)
ce = cudaStreamSynchronize(ctx->stream);
if (ce != cudaSuccess) {
@@ -774,6 +828,97 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
return 0;
}
+int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx,
+ const pplx_context_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const pplx_config_t *c = &ctx->config;
+ int total_spans = 0;
+ int total_tokens = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans ||
+ input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans ||
+ total_tokens > INT_MAX - n_tokens)
+ return -1;
+ for (int s = 0; s < input->n_spans; s++) {
+ int start = input->spans[s].start;
+ int len = input->spans[s].n_tokens;
+ if (start < 0 || len <= 0 || start > n_tokens ||
+ len > n_tokens - start)
+ return -1;
+ }
+ total_spans += input->n_spans;
+ total_tokens += n_tokens;
+ }
+
+ pplx_input_t *packed = (pplx_input_t *)malloc((size_t)batch * sizeof(*packed));
+ int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int));
+ int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int));
+ if (!packed || !h_starts || !h_lens) {
+ free(packed);
+ free(h_starts);
+ free(h_lens);
+ return -1;
+ }
+
+ int token_offset = 0;
+ int span_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ packed[b] = input->input;
+ for (int s = 0; s < input->n_spans; s++) {
+ h_starts[span_offset] = token_offset + input->spans[s].start;
+ h_lens[span_offset] = input->spans[s].n_tokens;
+ span_offset++;
+ }
+ token_offset += input->input.n_tokens;
+ }
+
+ int rc = -1;
+ cudaError_t ce;
+ if (cuda_forward_batch(ctx, packed, batch) != 0)
+ goto cleanup;
+ if (ensure_pooled_rows(ctx, total_spans) != 0 ||
+ ensure_span_buffers(ctx, total_spans) != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(ctx->span_starts, h_starts,
+ (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->span_lens, h_lens,
+ (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens,
+ total_spans, c->hidden_size);
+ if (launch_check() != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)total_spans * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span output copy failed: %s\n",
+ cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ rc = 0;
+
+cleanup:
+ free(h_lens);
+ free(h_starts);
+ free(packed);
+ return rc;
+}
+
int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids,
int n_tokens, float *out_embedding)
{
diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h
index af90230..ec57c8e 100644
--- a/pplx_embed_cuda.h
+++ b/pplx_embed_cuda.h
@@ -20,5 +20,8 @@ float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids,
int n_tokens);
int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
int batch, float *out_embeddings);
+int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx,
+ const pplx_context_input_t *inputs,
+ int batch, float *out_embeddings);
#endif /* PPLX_EMBED_CUDA_H */
diff --git a/pplx_server.c b/pplx_server.c
index 85df0e5..9df4d5b 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -10,6 +10,9 @@
#ifdef USE_MLX
#include "pplx_embed_mlx.h"
#endif
+#ifdef USE_CUDA
+#include "pplx_embed_cuda.h"
+#endif
#include "deps/ae/ae.h"
#include "deps/ae/anet.h"
@@ -230,6 +233,9 @@ typedef struct {
#ifdef USE_MLX
pplx_mlx_ctx_t *mlx_ctx;
#endif
+#ifdef USE_CUDA
+ pplx_cuda_ctx_t *cuda_ctx;
+#endif
} loaded_model;
typedef struct client client;
@@ -249,6 +255,7 @@ typedef struct {
char *api_key;
loaded_model models[4];
int use_mlx;
+ int use_cuda;
int mlx_quantize_bits;
int mlx_quantize_group_size;
@@ -1152,6 +1159,10 @@ static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs,
if (m->mlx_ctx)
return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out);
#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx)
+ return pplx_cuda_embed_batch(m->cuda_ctx, inputs, batch, out);
+#endif
return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
}
@@ -1162,6 +1173,10 @@ static int model_embed_spans_batch(loaded_model *m,
if (m->mlx_ctx)
return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out);
#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx)
+ return pplx_cuda_embed_spans_batch(m->cuda_ctx, inputs, batch, out);
+#endif
return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch,
out);
}
@@ -1966,9 +1981,9 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
static void *worker_main(void *arg) {
http_server *s = arg;
+ int rc = 0;
#ifdef USE_MLX
if (s->use_mlx) {
- int rc = 0;
for (int i = 0; i < 4; i++) {
loaded_model *m = &s->models[i];
if (!m->info) continue;
@@ -2009,6 +2024,44 @@ static void *worker_main(void *arg) {
}
} else
#endif
+#ifdef USE_CUDA
+ if (s->use_cuda) {
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (!m->info) continue;
+ m->cuda_ctx = pplx_cuda_load(m->path);
+ if (!m->cuda_ctx) {
+ server_log("pplx-serve: failed to load CUDA model on worker: %s",
+ m->path);
+ rc = 1;
+ break;
+ }
+ if (pplx_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) {
+ server_log("pplx-serve: model %s has unexpected dimension %d",
+ m->info->id,
+ pplx_cuda_config(m->cuda_ctx)->hidden_size);
+ rc = 1;
+ break;
+ }
+ }
+ pthread_mutex_lock(&s->mu);
+ s->worker_init_rc = rc;
+ s->worker_ready = 1;
+ if (rc) s->stopping = 1;
+ pthread_cond_broadcast(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+ if (rc) {
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (m->cuda_ctx) {
+ pplx_cuda_free(m->cuda_ctx);
+ m->cuda_ctx = NULL;
+ }
+ }
+ return NULL;
+ }
+ } else
+#endif
{
pthread_mutex_lock(&s->mu);
s->worker_ready = 1;
@@ -2045,6 +2098,17 @@ static void *worker_main(void *arg) {
}
}
#endif
+#ifdef USE_CUDA
+ if (s->use_cuda) {
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (m->cuda_ctx) {
+ pplx_cuda_free(m->cuda_ctx);
+ m->cuda_ctx = NULL;
+ }
+ }
+ }
+#endif
return NULL;
}
@@ -2088,6 +2152,11 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
return 0;
}
#endif
+#ifdef USE_CUDA
+ if (s->use_cuda) {
+ return 0;
+ }
+#endif
m->cpu_model = pplx_model_load(path);
if (!m->cpu_model) {
server_log("pplx-serve: failed to load model: %s", path);
@@ -2117,6 +2186,9 @@ static void free_models(http_server *s) {
#ifdef USE_MLX
if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx);
#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx) pplx_cuda_free(m->cuda_ctx);
+#endif
}
}
@@ -2239,6 +2311,7 @@ int pplx_run_server(const pplx_server_config_t *cfg) {
? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US;
s.enable_cors = cfg->enable_cors;
s.use_mlx = cfg->use_mlx;
+ s.use_cuda = cfg->use_cuda;
s.mlx_quantize_bits = cfg->mlx_quantize_bits;
s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0
? cfg->mlx_quantize_group_size : 64;
diff --git a/pplx_server.h b/pplx_server.h
index 72d4a6c..225a7ac 100644
--- a/pplx_server.h
+++ b/pplx_server.h
@@ -17,6 +17,7 @@ typedef struct {
int max_batch_tokens;
int batch_wait_us;
int use_mlx;
+ int use_cuda;
int mlx_quantize_bits;
int mlx_quantize_group_size;
int allow_memory_overcommit;