aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--main.c25
-rw-r--r--pplx_embed_cuda.cu92
-rw-r--r--pplx_embed_cuda.h2
3 files changed, 102 insertions, 17 deletions
diff --git a/main.c b/main.c
index f7711da..9472edf 100644
--- a/main.c
+++ b/main.c
@@ -52,6 +52,8 @@ static void print_usage(const char *prog)
" Quantize MLX linear weights to 8 bits at load time\n"
" --mlx-quantize-group-size N\n"
" MLX quantization group size (default: 64)\n"
+ " --cuda-fast-gemm MODE\n"
+ " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
" --allow-memory-overcommit\n"
" Allow an MLX server model set above the host-memory safety budget\n"
" --host HOST Server bind host (default: 127.0.0.1)\n"
@@ -537,6 +539,7 @@ int main(int argc, char *argv[])
int batch_wait_us = -1;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
+ const char *cuda_fast_gemm = NULL;
int dist_activation_bits = 32;
const char *host = "127.0.0.1";
const char *api_key = NULL;
@@ -619,6 +622,9 @@ int main(int argc, char *argv[])
return 1;
}
}
+ else if (!strcmp(f, "--cuda-fast-gemm")) {
+ cuda_fast_gemm = argv[++arg_start];
+ }
else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
else if (!strcmp(f, "--allow-memory-overcommit")) {
@@ -680,6 +686,25 @@ int main(int argc, char *argv[])
pplx_verbose = verbose;
qwen_verbose = verbose;
+ if (cuda_fast_gemm) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
+ fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
if (serve_mode) {
if (stdin_mode || dist_worker_mode || dist_remote) {
fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n");
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu
index ea04a10..276f1ea 100644
--- a/pplx_embed_cuda.cu
+++ b/pplx_embed_cuda.cu
@@ -78,6 +78,40 @@ struct pplx_cuda_ctx {
} \
} while (0)
+static int parse_gemm_compute(const char *mode, cublasComputeType_t *out)
+{
+ if (!mode || !mode[0] || !strcmp(mode, "f32")) {
+ *out = CUBLAS_COMPUTE_32F;
+ return 0;
+ }
+ if (!strcmp(mode, "tf32")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_TF32;
+ return 0;
+ }
+ if (!strcmp(mode, "bf16")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16BF;
+ return 0;
+ }
+ if (!strcmp(mode, "16f")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16F;
+ return 0;
+ }
+ return -1;
+}
+
+static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
+static int g_gemm_compute_set = 0;
+
+int pplx_cuda_set_fast_gemm(const char *mode)
+{
+ cublasComputeType_t ct;
+ if (parse_gemm_compute(mode, &ct) != 0)
+ return -1;
+ g_gemm_compute = ct;
+ g_gemm_compute_set = 1;
+ return 0;
+}
+
static void cuda_matrix_free(cuda_matrix_t *m)
{
if (!m) return;
@@ -498,22 +532,20 @@ static int launch_check(void)
return 0;
}
-// Compute type for the projection GEMMs. Default is exact F32 (== cublasSgemm).
+// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
// PPLX_CUDA_FAST_GEMM={bf16,tf32,16f} selects a reduced-precision tensor-core
// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
// internally.
static cublasComputeType_t gemm_compute(void)
{
- static int init = 0;
- static cublasComputeType_t ct = CUBLAS_COMPUTE_32F;
- if (!init) {
+ static int env_init = 0;
+ if (!g_gemm_compute_set && !env_init) {
const char *e = getenv("PPLX_CUDA_FAST_GEMM");
- if (e && !strcmp(e, "bf16")) ct = CUBLAS_COMPUTE_32F_FAST_16BF;
- else if (e && !strcmp(e, "tf32")) ct = CUBLAS_COMPUTE_32F_FAST_TF32;
- else if (e && !strcmp(e, "16f")) ct = CUBLAS_COMPUTE_32F_FAST_16F;
- init = 1;
+ if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
+ g_gemm_compute_set = 1;
+ env_init = 1;
}
- return ct;
+ return g_gemm_compute;
}
static int linear_ex(cublasHandle_t blas, const float *w, const float *x,
@@ -550,6 +582,30 @@ static int linear_accum_strided(cublasHandle_t blas, const cuda_matrix_t *w,
return linear_ex(blas, w->d, x, y, rows, in_dim, out_dim, x_stride, 1.0f);
}
+static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa,
+ cublasOperation_t transb, int m, int n, int k,
+ const float *alpha, const float *A, int lda,
+ long long strideA, const float *B, int ldb,
+ long long strideB, const float *beta,
+ float *C, int ldc, long long strideC,
+ int batch_count)
+{
+ cublasComputeType_t ct = gemm_compute();
+ if (ct == CUBLAS_COMPUTE_32F) {
+ CUBLAS_CHECK(cublasSgemmStridedBatched(
+ blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb,
+ strideB, beta, C, ldc, strideC, batch_count));
+ } else {
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(
+ blas, transa, transb, m, n, k,
+ alpha, A, CUDA_R_32F, lda, strideA,
+ B, CUDA_R_32F, ldb, strideB,
+ beta, C, CUDA_R_32F, ldc, strideC,
+ batch_count, ct, CUBLAS_GEMM_DEFAULT));
+ }
+ return 0;
+}
+
// GEMM-based attention for one micro-batch. Expands K/V to the per-query-head
// layout, then for each sequence runs batched-over-heads Q@K^T (scaled) ->
// row softmax -> @V via cuBLAS. Scales far better than the streaming warp
@@ -584,18 +640,20 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
const float *V = ctx->vexp + (size_t)start * q_dim;
float *O = ctx->attn_out + (size_t)start * q_dim;
/* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
- CUBLAS_CHECK(cublasSgemmStridedBatched(
- ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
- &alpha, K, q_dim, hd, Q, qkv_dim, hd,
- &beta, ctx->attn_scores, L, (long long)L * L, H));
+ if (gemm_strided_batched(
+ ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
+ &alpha, K, q_dim, hd, Q, qkv_dim, hd,
+ &beta, ctx->attn_scores, L, (long long)L * L, H) != 0)
+ return -1;
attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
ctx->attn_scores, L);
if (launch_check() != 0) return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- CUBLAS_CHECK(cublasSgemmStridedBatched(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L,
- &beta, O, q_dim, hd, H));
+ if (gemm_strided_batched(
+ ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
+ &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L,
+ &beta, O, q_dim, hd, H) != 0)
+ return -1;
}
return 0;
}
diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h
index ec57c8e..c0fe010 100644
--- a/pplx_embed_cuda.h
+++ b/pplx_embed_cuda.h
@@ -9,6 +9,8 @@
typedef struct pplx_cuda_ctx pplx_cuda_ctx_t;
+int pplx_cuda_set_fast_gemm(const char *mode);
+
pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir);
void pplx_cuda_free(pplx_cuda_ctx_t *ctx);