aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu20
1 files changed, 14 insertions, 6 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index b030e74..e433b7b 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -1131,10 +1131,14 @@ static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) {
// y = w @ x (+ beta*y). With BF16 weights and an already-BF16 activation
// (x_is_bf16 - the Qwen fused path) both inputs are BF16 with F32 accumulation.
-// For the BERT family the activation stays F32 and the BF16 weight is widened
-// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a
-// storage choice that costs no quality, which the post-norm LayerNorm needs.
-// With F32 weights this is the exact path.
+// For the BERT family in an exact-or-TF32 compute mode the activation stays F32
+// and the BF16 weight is widened back to F32 here, so the GEMM is exact (or
+// TF32, which needs F32 operands): BF16 is then a storage choice that costs no
+// quality, which the mean-subtracting post-attention LayerNorm needs. When the
+// caller opts into 16-bit float compute (gemm mode bf16/16f) exactness is
+// already given up, so BERT takes the direct BF16xBF16 path too: widening to F32
+// would only read full F32 weight bandwidth for a result the tensor core rounds
+// to BF16 anyway. With F32 weights this is the exact path.
static int linear_ex(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const void *x,
@@ -1147,7 +1151,11 @@ static int linear_ex(ffwd_cuda_ctx_t *ctx,
float beta) {
const float alpha = 1.0f;
if (w->bf16) {
- if (!x_is_bf16 && ctx->config.family == FFWD_FAMILY_BERT) {
+ // 16-bit float compute (bf16/16f) reads bf16 weights directly; exact F32
+ // and TF32 need F32 weight operands, so BERT widens its bf16 weight then.
+ cublasComputeType_t gc = gemm_compute();
+ int compute_16bit = (gc == CUBLAS_COMPUTE_32F_FAST_16BF || gc == CUBLAS_COMPUTE_32F_FAST_16F);
+ if (!x_is_bf16 && ctx->config.family == FFWD_FAMILY_BERT && !compute_16bit) {
size_t welems = (size_t)out_dim * (size_t)in_dim;
if (ensure_weight_f32(ctx, welems) != 0)
return -1;
@@ -1158,7 +1166,7 @@ static int linear_ex(ffwd_cuda_ctx_t *ctx,
return -1;
CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
&alpha, ctx->weight_f32, CUDA_R_32F, in_dim, x, CUDA_R_32F,
- x_stride, &beta, y, CUDA_R_32F, out_dim, gemm_compute(),
+ x_stride, &beta, y, CUDA_R_32F, out_dim, gc,
CUBLAS_GEMM_DEFAULT));
return 0;
}