aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-08 23:27:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitccceb5d0594e6879d4e2e4236980d233ea76c7ed (patch)
treedf9031a6690ad3778712290f5d6fa3219c1a0c20 /main.c
parent940303fe2fa3d55746e86fc199d8900f0634568a (diff)
downloadffwd-ccceb5d0594e6879d4e2e4236980d233ea76c7ed.tar.gz
ffwd-ccceb5d0594e6879d4e2e4236980d233ea76c7ed.zip
cuda: add fast GEMM mode (--cuda-fast-gemm mode)
Diffstat (limited to 'main.c')
-rw-r--r--main.c25
1 files changed, 25 insertions, 0 deletions
diff --git a/main.c b/main.c
index f7711da..9472edf 100644
--- a/main.c
+++ b/main.c
@@ -52,6 +52,8 @@ static void print_usage(const char *prog)
" Quantize MLX linear weights to 8 bits at load time\n"
" --mlx-quantize-group-size N\n"
" MLX quantization group size (default: 64)\n"
+ " --cuda-fast-gemm MODE\n"
+ " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
" --allow-memory-overcommit\n"
" Allow an MLX server model set above the host-memory safety budget\n"
" --host HOST Server bind host (default: 127.0.0.1)\n"
@@ -537,6 +539,7 @@ int main(int argc, char *argv[])
int batch_wait_us = -1;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
+ const char *cuda_fast_gemm = NULL;
int dist_activation_bits = 32;
const char *host = "127.0.0.1";
const char *api_key = NULL;
@@ -619,6 +622,9 @@ int main(int argc, char *argv[])
return 1;
}
}
+ else if (!strcmp(f, "--cuda-fast-gemm")) {
+ cuda_fast_gemm = argv[++arg_start];
+ }
else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
else if (!strcmp(f, "--allow-memory-overcommit")) {
@@ -680,6 +686,25 @@ int main(int argc, char *argv[])
pplx_verbose = verbose;
qwen_verbose = verbose;
+ if (cuda_fast_gemm) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
+ fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
if (serve_mode) {
if (stdin_mode || dist_worker_mode || dist_remote) {
fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n");