From ccceb5d0594e6879d4e2e4236980d233ea76c7ed Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Mon, 8 Jun 2026 23:27:02 +0200 Subject: cuda: add fast GEMM mode (--cuda-fast-gemm mode) --- main.c | 25 +++++++++++++++ pplx_embed_cuda.cu | 92 ++++++++++++++++++++++++++++++++++++++++++++---------- pplx_embed_cuda.h | 2 ++ 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/main.c b/main.c index f7711da..9472edf 100644 --- a/main.c +++ b/main.c @@ -52,6 +52,8 @@ static void print_usage(const char *prog) " Quantize MLX linear weights to 8 bits at load time\n" " --mlx-quantize-group-size N\n" " MLX quantization group size (default: 64)\n" + " --cuda-fast-gemm MODE\n" + " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" " --allow-memory-overcommit\n" " Allow an MLX server model set above the host-memory safety budget\n" " --host HOST Server bind host (default: 127.0.0.1)\n" @@ -537,6 +539,7 @@ int main(int argc, char *argv[]) int batch_wait_us = -1; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; + const char *cuda_fast_gemm = NULL; int dist_activation_bits = 32; const char *host = "127.0.0.1"; const char *api_key = NULL; @@ -619,6 +622,9 @@ int main(int argc, char *argv[]) return 1; } } + else if (!strcmp(f, "--cuda-fast-gemm")) { + cuda_fast_gemm = argv[++arg_start]; + } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } else if (!strcmp(f, "--allow-memory-overcommit")) { @@ -680,6 +686,25 @@ int main(int argc, char *argv[]) pplx_verbose = verbose; qwen_verbose = verbose; + if (cuda_fast_gemm) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { + fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + if (serve_mode) { if (stdin_mode || dist_worker_mode || dist_remote) { fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n"); diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index ea04a10..276f1ea 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -78,6 +78,40 @@ struct pplx_cuda_ctx { } \ } while (0) +static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) +{ + if (!mode || !mode[0] || !strcmp(mode, "f32")) { + *out = CUBLAS_COMPUTE_32F; + return 0; + } + if (!strcmp(mode, "tf32")) { + *out = CUBLAS_COMPUTE_32F_FAST_TF32; + return 0; + } + if (!strcmp(mode, "bf16")) { + *out = CUBLAS_COMPUTE_32F_FAST_16BF; + return 0; + } + if (!strcmp(mode, "16f")) { + *out = CUBLAS_COMPUTE_32F_FAST_16F; + return 0; + } + return -1; +} + +static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; +static int g_gemm_compute_set = 0; + +int pplx_cuda_set_fast_gemm(const char *mode) +{ + cublasComputeType_t ct; + if (parse_gemm_compute(mode, &ct) != 0) + return -1; + g_gemm_compute = ct; + g_gemm_compute_set = 1; + return 0; +} + static void cuda_matrix_free(cuda_matrix_t *m) { if (!m) return; @@ -498,22 +532,20 @@ static int launch_check(void) return 0; } -// Compute type for the projection GEMMs. Default is exact F32 (== cublasSgemm). +// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). // PPLX_CUDA_FAST_GEMM={bf16,tf32,16f} selects a reduced-precision tensor-core // matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands // internally. static cublasComputeType_t gemm_compute(void) { - static int init = 0; - static cublasComputeType_t ct = CUBLAS_COMPUTE_32F; - if (!init) { + static int env_init = 0; + if (!g_gemm_compute_set && !env_init) { const char *e = getenv("PPLX_CUDA_FAST_GEMM"); - if (e && !strcmp(e, "bf16")) ct = CUBLAS_COMPUTE_32F_FAST_16BF; - else if (e && !strcmp(e, "tf32")) ct = CUBLAS_COMPUTE_32F_FAST_TF32; - else if (e && !strcmp(e, "16f")) ct = CUBLAS_COMPUTE_32F_FAST_16F; - init = 1; + if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) + g_gemm_compute_set = 1; + env_init = 1; } - return ct; + return g_gemm_compute; } static int linear_ex(cublasHandle_t blas, const float *w, const float *x, @@ -550,6 +582,30 @@ static int linear_accum_strided(cublasHandle_t blas, const cuda_matrix_t *w, return linear_ex(blas, w->d, x, y, rows, in_dim, out_dim, x_stride, 1.0f); } +static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, + cublasOperation_t transb, int m, int n, int k, + const float *alpha, const float *A, int lda, + long long strideA, const float *B, int ldb, + long long strideB, const float *beta, + float *C, int ldc, long long strideC, + int batch_count) +{ + cublasComputeType_t ct = gemm_compute(); + if (ct == CUBLAS_COMPUTE_32F) { + CUBLAS_CHECK(cublasSgemmStridedBatched( + blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, + strideB, beta, C, ldc, strideC, batch_count)); + } else { + CUBLAS_CHECK(cublasGemmStridedBatchedEx( + blas, transa, transb, m, n, k, + alpha, A, CUDA_R_32F, lda, strideA, + B, CUDA_R_32F, ldb, strideB, + beta, C, CUDA_R_32F, ldc, strideC, + batch_count, ct, CUBLAS_GEMM_DEFAULT)); + } + return 0; +} + // GEMM-based attention for one micro-batch. Expands K/V to the per-query-head // layout, then for each sequence runs batched-over-heads Q@K^T (scaled) -> // row softmax -> @V via cuBLAS. Scales far better than the streaming warp @@ -584,18 +640,20 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, const float *V = ctx->vexp + (size_t)start * q_dim; float *O = ctx->attn_out + (size_t)start * q_dim; /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ - CUBLAS_CHECK(cublasSgemmStridedBatched( - ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, - &alpha, K, q_dim, hd, Q, qkv_dim, hd, - &beta, ctx->attn_scores, L, (long long)L * L, H)); + if (gemm_strided_batched( + ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, + &alpha, K, q_dim, hd, Q, qkv_dim, hd, + &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) + return -1; attn_softmax_kernel<<stream>>>( ctx->attn_scores, L); if (launch_check() != 0) return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ - CUBLAS_CHECK(cublasSgemmStridedBatched( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L, - &beta, O, q_dim, hd, H)); + if (gemm_strided_batched( + ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, + &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L, + &beta, O, q_dim, hd, H) != 0) + return -1; } return 0; } diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h index ec57c8e..c0fe010 100644 --- a/pplx_embed_cuda.h +++ b/pplx_embed_cuda.h @@ -9,6 +9,8 @@ typedef struct pplx_cuda_ctx pplx_cuda_ctx_t; +int pplx_cuda_set_fast_gemm(const char *mode); + pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir); void pplx_cuda_free(pplx_cuda_ctx_t *ctx); -- cgit v1.3.1