aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-23 04:07:58 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-23 04:07:58 +0200
commitc075bbd6a5091a51dddb8141005aa75f41ed4e2c (patch)
treedb9ebdeaa9cbf40bd0fc01ce552c79698f338335
parentaeb9212bc5e0d977ca31d7524ea55a9434686a40 (diff)
downloadffwd-c075bbd6a5091a51dddb8141005aa75f41ed4e2c.tar.gz
ffwd-c075bbd6a5091a51dddb8141005aa75f41ed4e2c.zip
cuda: cache the cublaslt gate-up matmul plan per shape
-rw-r--r--libffwd/cuda.cu191
1 files changed, 114 insertions, 77 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 7b129cb..a59c22e 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -62,6 +62,12 @@ struct ffwd_cuda_ctx {
void *lt_workspace; // device workspace for cublasLtMatmul
size_t lt_ws_bytes;
void *lt_bias16; // 16-bit cast of an epilogue bias (cuBLASLt rejects F32 bias on 16-bit out)
+ // Cached cuBLASLt plan for the repeated gate-up shape (rebuilt only when the
+ // shape changes, e.g. a new total-token count); skips per-layer heuristics.
+ cublasLtMatmulDesc_t lt_desc;
+ cublasLtMatrixLayout_t lt_A, lt_B, lt_D;
+ cublasLtMatmulHeuristicResult_t lt_heur;
+ int lt_have, lt_k_rows, lt_k_in, lt_k_out, lt_k_xs, lt_k_epi, lt_k_io, lt_k_od;
cuda_matrix_t embed_tokens;
cuda_layer_t *layers;
float *norm;
@@ -2153,12 +2159,26 @@ static int linear_accum(ffwd_cuda_ctx_t *ctx,
return linear_ex(ctx, w, x, CUDA_DTYPE_F32, y, rows, in_dim, out_dim, in_dim, 1.0f);
}
+static void linear_lt_plan_free(ffwd_cuda_ctx_t *ctx) {
+ if (!ctx->lt_have)
+ return;
+ cublasLtMatrixLayoutDestroy(ctx->lt_D);
+ cublasLtMatrixLayoutDestroy(ctx->lt_B);
+ cublasLtMatrixLayoutDestroy(ctx->lt_A);
+ cublasLtMatmulDescDestroy(ctx->lt_desc);
+ ctx->lt_have = 0;
+}
+
// Fused projection via cuBLASLt: y = epilogue(W^T @ x). The epilogue applies the
// row bias and, for CUBLASLT_EPILOGUE_GELU_BIAS, the exact erf GeLU inside the
// GEMM, so a separate bias/activation kernel and its [out_dim x rows] HBM
-// round-trip are removed. Operands are the weight's 16-bit dtype (also x's), the
-// output is `out_dtype`, accumulation is F32. The bias is F32. Layouts mirror the
-// cublasGemmEx call (OP_T on the [in_dim x out_dim] weight, OP_N on x).
+// round-trip are removed. Operands and output are 16-bit (bf16/f16), accumulation
+// is F32, the F32 bias is cast to the output dtype (cuBLASLt rejects an F32 bias
+// on 16-bit output). The matmul plan (descriptor + layouts + algo) is cached on
+// the context keyed by shape: the BERT gate-up shape repeats across every layer
+// and across same-size requests, so the heuristic runs ~once per request rather
+// than once per layer. The cached descriptor points its bias at lt_bias16, so the
+// per-call cast refreshes the values the matmul reads.
static int linear_lt(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const void *x,
@@ -2174,88 +2194,104 @@ static int linear_lt(ffwd_cuda_ctx_t *ctx,
fprintf(stderr, "cuda: linear_lt requires 16-bit weights\n");
return -1;
}
+ if (out_dtype != CUDA_DTYPE_BF16 && out_dtype != CUDA_DTYPE_F16) {
+ fprintf(stderr, "cuda: linear_lt requires 16-bit output\n");
+ return -1;
+ }
cudaDataType io = w->dtype == CUDA_DTYPE_BF16 ? CUDA_R_16BF : CUDA_R_16F;
- cudaDataType od = out_dtype == CUDA_DTYPE_F16
- ? CUDA_R_16F
- : (out_dtype == CUDA_DTYPE_BF16 ? CUDA_R_16BF : CUDA_R_32F);
+ cudaDataType od = out_dtype == CUDA_DTYPE_BF16 ? CUDA_R_16BF : CUDA_R_16F;
const float alpha = 1.0f, beta = 0.0f;
- // cuBLASLt's heuristic rejects an F32 bias when the output is 16-bit
- // (INVALID_VALUE), so for a 16-bit output cast the F32 bias to the output
- // dtype into the small lt_bias16 scratch and declare that dtype.
- const void *bias_dev = bias;
- cudaDataType bias_t = CUDA_R_32F;
- if (od == CUDA_R_16BF) {
+
+ // Refresh the bias each call: cast the F32 bias to the output dtype into the
+ // fixed lt_bias16 scratch that the cached descriptor's BIAS_POINTER targets.
+ if (od == CUDA_R_16BF)
cast_f32_to_bf16_kernel<<<(out_dim + 255) / 256, 256, 0, ctx->stream>>>(
(__nv_bfloat16 *)ctx->lt_bias16, bias, out_dim);
- bias_dev = ctx->lt_bias16;
- bias_t = CUDA_R_16BF;
- } else if (od == CUDA_R_16F) {
+ else
cast_f32_to_f16_kernel<<<(out_dim + 255) / 256, 256, 0, ctx->stream>>>(
(__half *)ctx->lt_bias16, bias, out_dim);
- bias_dev = ctx->lt_bias16;
- bias_t = CUDA_R_16F;
- }
- cublasLtMatmulDesc_t op = NULL;
- cublasLtMatrixLayout_t A = NULL, B = NULL, D = NULL;
- cublasLtMatmulPreference_t pref = NULL;
- int rc = -1;
- do {
- if (cublasLtMatmulDescCreate(&op, CUBLAS_COMPUTE_32F, CUDA_R_32F) != CUBLAS_STATUS_SUCCESS)
- break;
- cublasOperation_t ta = CUBLAS_OP_T, tb = CUBLAS_OP_N;
- cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSA, &ta, sizeof(ta));
- cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSB, &tb, sizeof(tb));
- cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue,
- sizeof(epilogue));
- cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_dev,
- sizeof(bias_dev));
- cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, &bias_t,
- sizeof(bias_t));
- // A = weight stored [in_dim x out_dim] col-major (ld in_dim), transposed by op;
- // B = x stored [in_dim x rows] (ld x_stride); D = y stored [out_dim x rows].
- if (cublasLtMatrixLayoutCreate(&A, io, in_dim, out_dim, in_dim) != CUBLAS_STATUS_SUCCESS)
- break;
- if (cublasLtMatrixLayoutCreate(&B, io, in_dim, rows, x_stride) != CUBLAS_STATUS_SUCCESS)
- break;
- if (cublasLtMatrixLayoutCreate(&D, od, out_dim, rows, out_dim) != CUBLAS_STATUS_SUCCESS)
- break;
- if (cublasLtMatmulPreferenceCreate(&pref) != CUBLAS_STATUS_SUCCESS)
- break;
- cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,
- &ctx->lt_ws_bytes, sizeof(ctx->lt_ws_bytes));
- cublasLtMatmulHeuristicResult_t heur;
- int got = 0;
- cublasStatus_t hst =
- cublasLtMatmulAlgoGetHeuristic(ctx->lt, op, A, B, D, D, pref, 1, &heur, &got);
- if (hst != CUBLAS_STATUS_SUCCESS || got == 0) {
- fprintf(stderr,
- "cuda: linear_lt heuristic st=%d got=%d (m=%d n=%d k=%d epi=%d io=%d od=%d)\n",
- (int)hst, got, out_dim, rows, in_dim, (int)epilogue, (int)io, (int)od);
- break;
- }
- cublasStatus_t mst =
- cublasLtMatmul(ctx->lt, op, &alpha, w->d, A, x, B, &beta, y, D, y, D, &heur.algo,
- ctx->lt_workspace, ctx->lt_ws_bytes, ctx->stream);
- if (mst != CUBLAS_STATUS_SUCCESS) {
- fprintf(stderr, "cuda: linear_lt matmul st=%d\n", (int)mst);
- break;
+ int key_ok = ctx->lt_have && ctx->lt_k_rows == rows && ctx->lt_k_in == in_dim &&
+ ctx->lt_k_out == out_dim && ctx->lt_k_xs == x_stride &&
+ ctx->lt_k_epi == (int)epilogue && ctx->lt_k_io == (int)io && ctx->lt_k_od == (int)od;
+ if (!key_ok) {
+ linear_lt_plan_free(ctx);
+ cublasLtMatmulDesc_t op = NULL;
+ cublasLtMatrixLayout_t A = NULL, B = NULL, D = NULL;
+ cublasLtMatmulPreference_t pref = NULL;
+ const void *bias_dev = ctx->lt_bias16;
+ cudaDataType bias_t = od;
+ int ok = 0;
+ do {
+ if (cublasLtMatmulDescCreate(&op, CUBLAS_COMPUTE_32F, CUDA_R_32F) !=
+ CUBLAS_STATUS_SUCCESS)
+ break;
+ cublasOperation_t ta = CUBLAS_OP_T, tb = CUBLAS_OP_N;
+ cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSA, &ta, sizeof(ta));
+ cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSB, &tb, sizeof(tb));
+ cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue,
+ sizeof(epilogue));
+ cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias_dev,
+ sizeof(bias_dev));
+ cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, &bias_t,
+ sizeof(bias_t));
+ // A = weight stored [in_dim x out_dim] col-major (ld in_dim), op transposes it;
+ // B = x stored [in_dim x rows] (ld x_stride); D = y stored [out_dim x rows].
+ if (cublasLtMatrixLayoutCreate(&A, io, in_dim, out_dim, in_dim) != CUBLAS_STATUS_SUCCESS)
+ break;
+ if (cublasLtMatrixLayoutCreate(&B, io, in_dim, rows, x_stride) != CUBLAS_STATUS_SUCCESS)
+ break;
+ if (cublasLtMatrixLayoutCreate(&D, od, out_dim, rows, out_dim) != CUBLAS_STATUS_SUCCESS)
+ break;
+ if (cublasLtMatmulPreferenceCreate(&pref) != CUBLAS_STATUS_SUCCESS)
+ break;
+ cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,
+ &ctx->lt_ws_bytes, sizeof(ctx->lt_ws_bytes));
+ int got = 0;
+ cublasStatus_t hst =
+ cublasLtMatmulAlgoGetHeuristic(ctx->lt, op, A, B, D, D, pref, 1, &ctx->lt_heur, &got);
+ if (hst != CUBLAS_STATUS_SUCCESS || got == 0) {
+ fprintf(stderr, "cuda: linear_lt heuristic st=%d got=%d (m=%d n=%d k=%d epi=%d)\n",
+ (int)hst, got, out_dim, rows, in_dim, (int)epilogue);
+ break;
+ }
+ ok = 1;
+ } while (0);
+ if (pref)
+ cublasLtMatmulPreferenceDestroy(pref);
+ if (!ok) {
+ if (D)
+ cublasLtMatrixLayoutDestroy(D);
+ if (B)
+ cublasLtMatrixLayoutDestroy(B);
+ if (A)
+ cublasLtMatrixLayoutDestroy(A);
+ if (op)
+ cublasLtMatmulDescDestroy(op);
+ fprintf(stderr, "cuda: linear_lt plan build failed\n");
+ return -1;
}
- rc = 0;
- } while (0);
- if (pref)
- cublasLtMatmulPreferenceDestroy(pref);
- if (D)
- cublasLtMatrixLayoutDestroy(D);
- if (B)
- cublasLtMatrixLayoutDestroy(B);
- if (A)
- cublasLtMatrixLayoutDestroy(A);
- if (op)
- cublasLtMatmulDescDestroy(op);
- if (rc != 0)
- fprintf(stderr, "cuda: linear_lt fused matmul failed\n");
- return rc;
+ ctx->lt_desc = op;
+ ctx->lt_A = A;
+ ctx->lt_B = B;
+ ctx->lt_D = D;
+ ctx->lt_have = 1;
+ ctx->lt_k_rows = rows;
+ ctx->lt_k_in = in_dim;
+ ctx->lt_k_out = out_dim;
+ ctx->lt_k_xs = x_stride;
+ ctx->lt_k_epi = (int)epilogue;
+ ctx->lt_k_io = (int)io;
+ ctx->lt_k_od = (int)od;
+ }
+ cublasStatus_t mst = cublasLtMatmul(ctx->lt, ctx->lt_desc, &alpha, w->d, ctx->lt_A, x, ctx->lt_B,
+ &beta, y, ctx->lt_D, y, ctx->lt_D, &ctx->lt_heur.algo,
+ ctx->lt_workspace, ctx->lt_ws_bytes, ctx->stream);
+ if (mst != CUBLAS_STATUS_SUCCESS) {
+ fprintf(stderr, "cuda: linear_lt matmul st=%d\n", (int)mst);
+ return -1;
+ }
+ return 0;
}
static int gemm_strided_batched(cublasHandle_t blas,
@@ -3017,6 +3053,7 @@ void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) {
cudaFree(ctx->span_lens);
cudaFree(ctx->lt_workspace);
cudaFree(ctx->lt_bias16);
+ linear_lt_plan_free(ctx);
if (ctx->lt)
cublasLtDestroy(ctx->lt);
if (ctx->blas)