diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-08 22:41:06 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 26aa937f023652648f201ad849d8dd644ca0261c (patch) | |
| tree | fc5d2f06a06250bfa0f9a71658a5df2190065336 | |
| parent | 0b8fb1d850ffb0b1002767dbc5b533158c318e4f (diff) | |
| download | ffwd-26aa937f023652648f201ad849d8dd644ca0261c.tar.gz ffwd-26aa937f023652648f201ad849d8dd644ca0261c.zip | |
cuda: add opt-in BF16 tensor-core GEMM mode
| -rw-r--r-- | pplx_embed_cuda.cu | 55 |
1 files changed, 34 insertions, 21 deletions
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index 8c335d2..4c192be 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -498,42 +498,55 @@ static int launch_check(void) return 0; } -static int linear(cublasHandle_t blas, const cuda_matrix_t *w, - const float *x, float *y, int rows, int in_dim, int out_dim) +// Compute type for the projection GEMMs. Default is exact F32 (== cublasSgemm). +// PPLX_CUDA_FAST_GEMM={bf16,tf32,16f} opts into tensor-core matmul that keeps +// F32 inputs/outputs and F32 accumulation but rounds operands internally. +static cublasComputeType_t gemm_compute(void) +{ + static int init = 0; + static cublasComputeType_t ct = CUBLAS_COMPUTE_32F; + if (!init) { + const char *e = getenv("PPLX_CUDA_FAST_GEMM"); + if (e && !strcmp(e, "bf16")) ct = CUBLAS_COMPUTE_32F_FAST_16BF; + else if (e && !strcmp(e, "tf32")) ct = CUBLAS_COMPUTE_32F_FAST_TF32; + else if (e && !strcmp(e, "16f")) ct = CUBLAS_COMPUTE_32F_FAST_16F; + init = 1; + } + return ct; +} + +static int linear_ex(cublasHandle_t blas, const float *w, const float *x, + float *y, int rows, int in_dim, int out_dim, int x_stride, + float beta) { const float alpha = 1.0f; - const float beta = 0.0f; - CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, in_dim, x, in_dim, - &beta, y, out_dim)); + CUBLAS_CHECK(cublasGemmEx(blas, CUBLAS_OP_T, CUBLAS_OP_N, + out_dim, rows, in_dim, + &alpha, w, CUDA_R_32F, in_dim, + x, CUDA_R_32F, x_stride, + &beta, y, CUDA_R_32F, out_dim, + gemm_compute(), CUBLAS_GEMM_DEFAULT)); return 0; } +static int linear(cublasHandle_t blas, const cuda_matrix_t *w, + const float *x, float *y, int rows, int in_dim, int out_dim) +{ + return linear_ex(blas, w->d, x, y, rows, in_dim, out_dim, in_dim, 0.0f); +} + static int linear_accum(cublasHandle_t blas, const cuda_matrix_t *w, const float *x, float *y, int rows, int in_dim, int out_dim) { - const float alpha = 1.0f; - const float beta = 1.0f; - CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, in_dim, x, in_dim, - &beta, y, out_dim)); - return 0; + return linear_ex(blas, w->d, x, y, rows, in_dim, out_dim, in_dim, 1.0f); } static int linear_accum_strided(cublasHandle_t blas, const cuda_matrix_t *w, const float *x, int x_stride, float *y, int rows, int in_dim, int out_dim) { - const float alpha = 1.0f; - const float beta = 1.0f; - CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, in_dim, x, x_stride, - &beta, y, out_dim)); - return 0; + return linear_ex(blas, w->d, x, y, rows, in_dim, out_dim, x_stride, 1.0f); } // GEMM-based attention for one micro-batch. Expands K/V to the per-query-head |
