aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-08 17:57:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitd9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f (patch)
treee007eff496ba92ca8a62890ea4843bd793341be3 /main.c
parent884ef3007382451de2cb8801d33f7d0efc05f5b6 (diff)
downloadffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.tar.gz
ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.zip
cuda: add server contextual path
Diffstat (limited to 'main.c')
-rw-r--r--main.c8
1 files changed, 2 insertions, 6 deletions
diff --git a/main.c b/main.c
index 7952538..f7711da 100644
--- a/main.c
+++ b/main.c
@@ -697,11 +697,6 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
- if (use_cuda) {
- fprintf(stderr, "--backend cuda is not available in --serve mode yet\n");
- free_model_specs(&model_specs);
- return 1;
- }
if (mlx_quantize_bits && !use_mlx) {
fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n");
free_model_specs(&model_specs);
@@ -712,7 +707,7 @@ int main(int argc, char *argv[])
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
} else if (verbose >= 1) {
- fprintf(stderr, "Using MLX GPU backend\n");
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
}
pplx_server_config_t scfg = {
.models = model_specs.v,
@@ -723,6 +718,7 @@ int main(int argc, char *argv[])
.max_batch_tokens = max_batch_tokens,
.batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
+ .use_cuda = use_cuda,
.mlx_quantize_bits = mlx_quantize_bits,
.mlx_quantize_group_size = mlx_quantize_group_size,
.allow_memory_overcommit = allow_memory_overcommit,