From d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Mon, 8 Jun 2026 17:57:20 +0200 Subject: cuda: add server contextual path --- main.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'main.c') diff --git a/main.c b/main.c index 7952538..f7711da 100644 --- a/main.c +++ b/main.c @@ -697,11 +697,6 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } - if (use_cuda) { - fprintf(stderr, "--backend cuda is not available in --serve mode yet\n"); - free_model_specs(&model_specs); - return 1; - } if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); free_model_specs(&model_specs); @@ -712,7 +707,7 @@ int main(int argc, char *argv[]) qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } else if (verbose >= 1) { - fprintf(stderr, "Using MLX GPU backend\n"); + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); } pplx_server_config_t scfg = { .models = model_specs.v, @@ -723,6 +718,7 @@ int main(int argc, char *argv[]) .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, + .use_cuda = use_cuda, .mlx_quantize_bits = mlx_quantize_bits, .mlx_quantize_group_size = mlx_quantize_group_size, .allow_memory_overcommit = allow_memory_overcommit, -- cgit v1.3.1