From ccceb5d0594e6879d4e2e4236980d233ea76c7ed Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Mon, 8 Jun 2026 23:27:02 +0200 Subject: cuda: add fast GEMM mode (--cuda-fast-gemm mode) --- main.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'main.c') diff --git a/main.c b/main.c index f7711da..9472edf 100644 --- a/main.c +++ b/main.c @@ -52,6 +52,8 @@ static void print_usage(const char *prog) " Quantize MLX linear weights to 8 bits at load time\n" " --mlx-quantize-group-size N\n" " MLX quantization group size (default: 64)\n" + " --cuda-fast-gemm MODE\n" + " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" " --allow-memory-overcommit\n" " Allow an MLX server model set above the host-memory safety budget\n" " --host HOST Server bind host (default: 127.0.0.1)\n" @@ -537,6 +539,7 @@ int main(int argc, char *argv[]) int batch_wait_us = -1; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; + const char *cuda_fast_gemm = NULL; int dist_activation_bits = 32; const char *host = "127.0.0.1"; const char *api_key = NULL; @@ -619,6 +622,9 @@ int main(int argc, char *argv[]) return 1; } } + else if (!strcmp(f, "--cuda-fast-gemm")) { + cuda_fast_gemm = argv[++arg_start]; + } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } else if (!strcmp(f, "--allow-memory-overcommit")) { @@ -680,6 +686,25 @@ int main(int argc, char *argv[]) pplx_verbose = verbose; qwen_verbose = verbose; + if (cuda_fast_gemm) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { + fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + if (serve_mode) { if (stdin_mode || dist_worker_mode || dist_remote) { fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n"); -- cgit v1.3.1