diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-09 01:05:01 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 3613657e678c54500f122c5b305f9cc9f5032d21 (patch) | |
| tree | e475fa24255958e9d636eedb8c634b00f650fd81 /main.c | |
| parent | f4bc5ad86de55257c1f2a3a9de0a0e9fc91bfd3e (diff) | |
| download | ffwd-3613657e678c54500f122c5b305f9cc9f5032d21.tar.gz ffwd-3613657e678c54500f122c5b305f9cc9f5032d21.zip | |
cuda: add optional BF16 CUDA weight storage (--cuda-weights bf16)
Diffstat (limited to 'main.c')
| -rw-r--r-- | main.c | 30 |
1 files changed, 30 insertions, 0 deletions
@@ -54,6 +54,9 @@ static void print_usage(const char *prog) " MLX quantization group size (default: 64)\n" " --cuda-fast-gemm MODE\n" " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" + " --cuda-weights DTYPE\n" + " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" + " weight memory and uses BF16 tensor cores\n" " --allow-memory-overcommit\n" " Allow an MLX server model set above the host-memory safety budget\n" " --host HOST Server bind host (default: 127.0.0.1)\n" @@ -540,6 +543,7 @@ int main(int argc, char *argv[]) int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; const char *cuda_fast_gemm = NULL; + const char *cuda_weights = NULL; int dist_activation_bits = 32; const char *host = "127.0.0.1"; const char *api_key = NULL; @@ -625,6 +629,9 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "--cuda-fast-gemm")) { cuda_fast_gemm = argv[++arg_start]; } + else if (!strcmp(f, "--cuda-weights")) { + cuda_weights = argv[++arg_start]; + } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } else if (!strcmp(f, "--allow-memory-overcommit")) { @@ -705,6 +712,29 @@ int main(int argc, char *argv[]) #endif } + if (cuda_weights) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-weights requires --cuda or --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (!strcmp(cuda_weights, "bf16")) { + pplx_cuda_set_weights_bf16(1); + } else if (!strcmp(cuda_weights, "f32")) { + pplx_cuda_set_weights_bf16(0); + } else { + fprintf(stderr, "--cuda-weights must be f32 or bf16\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-weights requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + if (serve_mode) { if (stdin_mode || dist_worker_mode || dist_remote) { fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n"); |
