diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-08 17:57:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f (patch) | |
| tree | e007eff496ba92ca8a62890ea4843bd793341be3 /main.c | |
| parent | 884ef3007382451de2cb8801d33f7d0efc05f5b6 (diff) | |
| download | ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.tar.gz ffwd-d9e4761f63d5e6a6c08aab9e1d104b4a11ebdc8f.zip | |
cuda: add server contextual path
Diffstat (limited to 'main.c')
| -rw-r--r-- | main.c | 8 |
1 files changed, 2 insertions, 6 deletions
@@ -697,11 +697,6 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } - if (use_cuda) { - fprintf(stderr, "--backend cuda is not available in --serve mode yet\n"); - free_model_specs(&model_specs); - return 1; - } if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); free_model_specs(&model_specs); @@ -712,7 +707,7 @@ int main(int argc, char *argv[]) qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } else if (verbose >= 1) { - fprintf(stderr, "Using MLX GPU backend\n"); + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); } pplx_server_config_t scfg = { .models = model_specs.v, @@ -723,6 +718,7 @@ int main(int argc, char *argv[]) .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, + .use_cuda = use_cuda, .mlx_quantize_bits = mlx_quantize_bits, .mlx_quantize_group_size = mlx_quantize_group_size, .allow_memory_overcommit = allow_memory_overcommit, |
