diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:59:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:59:13 +0200 |
| commit | 7087c2928f5b61a73c6b1a15a6c1f32108e3c4b3 (patch) | |
| tree | 7d57508dfa9258c0ee356cf7da91c845032592f2 | |
| parent | e7a67084a9fd71e07946a957817c307e82302131 (diff) | |
| download | ffwd-7087c2928f5b61a73c6b1a15a6c1f32108e3c4b3.tar.gz ffwd-7087c2928f5b61a73c6b1a15a6c1f32108e3c4b3.zip | |
cuda: memory-only bf16 for BERT (f32 compute on bf16 weights)
| -rw-r--r-- | embed_cuda.cu | 52 |
1 files changed, 46 insertions, 6 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu index 94887fb..9464caa 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -64,7 +64,9 @@ struct embed_cuda_ctx { float *qkv; float *attn_out; float *ffn_gate_up; - void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) + void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) + float *weight_f32; // F32 widen scratch for one BF16 weight (memory-only bf16) + size_t weight_f32_elems; float *pooled_out; float *rope_cos; float *rope_sin; @@ -364,6 +366,14 @@ __global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float * out[i] = __float2bfloat16(in[i]); } +// Widen n BF16 values to F32 (used to run an exact F32 GEMM on BF16-stored +// weights - memory-only BF16, no precision loss from the operands). +__global__ static void cast_bf16_to_f32_kernel(float *out, const __nv_bfloat16 *in, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + out[i] = __bfloat162float(in[i]); +} + __device__ static float block_sum(float v) { __shared__ float partial[8]; __shared__ float total; @@ -748,11 +758,25 @@ static cublasComputeType_t gemm_compute(void) { return g_gemm_compute; } -// y = w @ x (+ beta*y). With BF16 weights both GEMM inputs are BF16 and -// accumulation/output stay F32: when x_is_bf16 the activation operand was -// already produced in BF16 by the upstream kernel, otherwise the F32 operand -// is cast into ctx->act_bf16 first. With F32 weights this is the exact path, -// honoring gemm_compute(). +// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only +// BF16 path materializes one F32 weight at a time into this buffer. +static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) { + if (ctx->weight_f32_elems >= elems) + return 0; + cudaFree(ctx->weight_f32); + ctx->weight_f32 = NULL; + ctx->weight_f32_elems = 0; + CUDA_CHECK(cudaMalloc((void **)&ctx->weight_f32, elems * sizeof(float))); + ctx->weight_f32_elems = elems; + return 0; +} + +// y = w @ x (+ beta*y). With BF16 weights and an already-BF16 activation +// (x_is_bf16 - the Qwen fused path) both inputs are BF16 with F32 accumulation. +// For the BERT family the activation stays F32 and the BF16 weight is widened +// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a +// storage choice that costs no quality, which the post-norm LayerNorm needs. +// With F32 weights this is the exact path. static int linear_ex(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w, const void *x, @@ -765,6 +789,21 @@ static int linear_ex(embed_cuda_ctx_t *ctx, float beta) { const float alpha = 1.0f; if (w->bf16) { + if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) { + size_t welems = (size_t)out_dim * (size_t)in_dim; + if (ensure_weight_f32(ctx, welems) != 0) + return -1; + int wt = 256; + cast_bf16_to_f32_kernel<<<(welems + wt - 1) / wt, wt, 0, ctx->stream>>>( + ctx->weight_f32, (const __nv_bfloat16 *)w->d, welems); + if (launch_check() != 0) + return -1; + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, + &alpha, ctx->weight_f32, CUDA_R_32F, in_dim, x, CUDA_R_32F, + x_stride, &beta, y, CUDA_R_32F, out_dim, gemm_compute(), + CUBLAS_GEMM_DEFAULT)); + return 0; + } const void *xb = x; if (!x_is_bf16) { size_t n = (size_t)rows * (size_t)x_stride; @@ -1356,6 +1395,7 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) { cudaFree(ctx->kexp); cudaFree(ctx->vexp); cudaFree(ctx->act_bf16); + cudaFree(ctx->weight_f32); cudaFree(ctx->attn_scores); cudaFree(ctx->attn_probs); cudaFree((void *)ctx->attn_ptrs); |
