aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-11 05:34:23 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit488b0f71443bc93a7ead4206ca4e9d2e09f088bd (patch)
tree7adf081732a8832ccb9f4dbacc6a579d097e8cb2
parente4adb4c8d0a67b9296d7ec0f54901759ed890ebc (diff)
downloadffwd-488b0f71443bc93a7ead4206ca4e9d2e09f088bd.tar.gz
ffwd-488b0f71443bc93a7ead4206ca4e9d2e09f088bd.zip
cuda: fuse QK norm+rope launches, BF16 attention output path
-rw-r--r--embed_cuda.cu144
1 files changed, 99 insertions, 45 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu
index e0ff898..dfc3e32 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -58,6 +58,7 @@ struct pplx_cuda_ctx {
float *kexp;
float *vexp;
float *attn_scores;
+ void *attn_probs; // BF16 softmax output (bf16 weights)
int *span_starts;
int *span_lens;
int pooled_rows_cap;
@@ -371,15 +372,19 @@ __global__ static void rms_norm_kernel(OUT_T *out, const float *x,
x[(size_t)row * dim + d] * inv * weight[d]);
}
-__global__ static void rms_norm_rope_head_kernel(float *x, const float *weight,
- const int *positions,
- const float *cosv,
- const float *sinv,
- int rows, int heads,
- int head_dim,
- int row_stride,
- int base_offset,
- float eps)
+// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with
+// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset.
+__global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm,
+ const float *k_norm,
+ const int *positions,
+ const float *cosv,
+ const float *sinv,
+ int rows, int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ int row_stride,
+ int q_offset, int k_offset,
+ float eps)
{
__shared__ float partial[4];
__shared__ float total;
@@ -391,6 +396,9 @@ __global__ static void rms_norm_rope_head_kernel(float *x, const float *weight,
int half = head_dim / 2;
int pair_d = tid < half ? tid + half : tid - half;
float sign = tid < half ? -1.0f : 1.0f;
+ const float *weight = head < n_heads ? q_norm : k_norm;
+ int base_offset = head < n_heads ? q_offset : k_offset;
+ if (head >= n_heads) head -= n_heads;
size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
float v = tid < head_dim ? x[base + tid] : 0.0f;
float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
@@ -443,8 +451,10 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
// so the attention GEMMs can stride uniformly over query heads.
+// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly.
+template <typename V_T>
__global__ static void attn_expand_kv_kernel(
- float *kexp, float *vexp, const float *qkv, int total, int n_heads,
+ float *kexp, V_T *vexp, const float *qkv, int total, int n_heads,
int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset)
{
int q_dim = n_heads * head_dim;
@@ -458,15 +468,19 @@ __global__ static void attn_expand_kv_kernel(
int kv = h / (n_heads / n_kv_heads);
const float *base = qkv + (size_t)t * qkv_dim;
kexp[idx] = base[k_offset + kv * head_dim + e];
- vexp[idx] = base[v_offset + kv * head_dim + e];
+ store_act(vexp, idx, base[v_offset + kv * head_dim + e]);
}
// Row softmax over the key dimension for one sequence's score tensor laid out
// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i).
// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
-__global__ static void attn_softmax_kernel(float *scores, int L)
+// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
+// probabilities feed a BF16 @V GEMM.
+template <typename P_T>
+__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
{
float *s = scores + (size_t)blockIdx.x * L;
+ P_T *p = probs + (size_t)blockIdx.x * L;
int tid = threadIdx.x;
__shared__ float red[256];
@@ -494,7 +508,8 @@ __global__ static void attn_softmax_kernel(float *scores, int L)
__syncthreads();
}
float inv = 1.0f / red[0];
- for (int j = tid; j < L; j += blockDim.x) s[j] *= inv;
+ for (int j = tid; j < L; j += blockDim.x)
+ store_act(p, (size_t)j, s[j] * inv);
}
__global__ static void mean_pool_kernel(float *out, const float *x,
@@ -644,9 +659,13 @@ static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa,
// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and
// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets
// is the host copy of the packed sequence boundaries.
+// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
+// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
+// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
int batch, int q_offset, int k_offset,
- int v_offset, int qkv_dim, float scale)
+ int v_offset, int qkv_dim, float scale,
+ __nv_bfloat16 *out_bf16)
{
const pplx_config_t *c = &ctx->config;
int q_dim = c->q_dim;
@@ -657,10 +676,17 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
int threads = 256;
size_t exp_count = (size_t)total * q_dim;
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
- 0, ctx->stream>>>(
- ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim,
- k_offset, v_offset);
+ if (out_bf16) {
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
+ 0, ctx->stream>>>(
+ ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H,
+ c->n_kv_heads, hd, qkv_dim, k_offset, v_offset);
+ } else {
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
+ 0, ctx->stream>>>(
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd,
+ qkv_dim, k_offset, v_offset);
+ }
if (launch_check() != 0) return -1;
for (int b = 0; b < batch; b++) {
@@ -668,23 +694,40 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
int L = h_offsets[b + 1] - start;
const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
const float *K = ctx->kexp + (size_t)start * q_dim;
- const float *V = ctx->vexp + (size_t)start * q_dim;
- float *O = ctx->attn_out + (size_t)start * q_dim;
/* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
if (gemm_strided_batched(
ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
&alpha, K, q_dim, hd, Q, qkv_dim, hd,
&beta, ctx->attn_scores, L, (long long)L * L, H) != 0)
return -1;
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, L);
- if (launch_check() != 0) return -1;
- /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- if (gemm_strided_batched(
+ if (out_bf16) {
+ __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs;
+ const __nv_bfloat16 *V =
+ (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
+ __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, P, L);
+ if (launch_check() != 0) return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(
ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L,
- &beta, O, q_dim, hd, H) != 0)
- return -1;
+ &one, V, CUDA_R_16BF, q_dim, hd,
+ P, CUDA_R_16BF, L, (long long)L * L,
+ &beta, O, CUDA_R_16BF, q_dim, hd,
+ H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ } else {
+ const float *V = ctx->vexp + (size_t)start * q_dim;
+ float *O = ctx->attn_out + (size_t)start * q_dim;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, ctx->attn_scores, L);
+ if (launch_check() != 0) return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ if (gemm_strided_batched(
+ ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
+ &one, V, q_dim, hd, ctx->attn_scores, L,
+ (long long)L * L, &beta, O, q_dim, hd, H) != 0)
+ return -1;
+ }
}
return 0;
}
@@ -736,8 +779,14 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
cudaFree(ctx->rope_sin);
cudaFree(ctx->attn_scores);
ctx->attn_scores = NULL;
+ cudaFree(ctx->attn_probs);
+ ctx->attn_probs = NULL;
CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores,
(size_t)c->n_heads * max_seq * max_seq * sizeof(float)));
+ if (g_weights_bf16)
+ CUDA_CHECK(cudaMalloc(&ctx->attn_probs,
+ (size_t)c->n_heads * max_seq * max_seq *
+ sizeof(__nv_bfloat16)));
float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
if (!hcos || !hsin) {
@@ -909,6 +958,7 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
cudaFree(ctx->vexp);
cudaFree(ctx->act_bf16);
cudaFree(ctx->attn_scores);
+ cudaFree(ctx->attn_probs);
cudaFree(ctx->span_starts);
cudaFree(ctx->span_lens);
if (ctx->blas) cublasDestroy(ctx->blas);
@@ -988,11 +1038,10 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
int k_offset = c->q_dim;
int v_offset = c->q_dim + c->kv_dim;
int qkv_dim = c->q_dim + 2 * c->kv_dim;
- // With BF16 weights the norm and SiLU kernels store their results as BF16
- // directly into ctx->act_bf16, which the next projection GEMM consumes
- // before any later producer reuses the buffer (single stream). This
- // removes three cast launches per layer; only the attention output still
- // needs a cast because cuBLAS cannot emit a BF16 C from F32 A/B operands.
+ // With BF16 weights every producer (norm, SiLU, attention @V GEMM)
+ // stores its result as BF16 directly into ctx->act_bf16, which the next
+ // projection GEMM consumes before any later producer reuses the buffer
+ // (single stream). No cast launches remain in the layer loop.
__nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
for (int layer = 0; layer < c->n_layers; layer++) {
cuda_layer_t *l = &ctx->layers[layer];
@@ -1016,23 +1065,28 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
return -1;
}
- rms_norm_rope_head_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>(
- ctx->qkv, l->q_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin,
- total, c->n_heads, c->head_dim, qkv_dim, q_offset,
- c->rms_norm_eps);
- rms_norm_rope_head_kernel<<<dim3(total, c->n_kv_heads), 128, 0, ctx->stream>>>(
- ctx->qkv, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin,
- total, c->n_kv_heads, c->head_dim, qkv_dim, k_offset,
- c->rms_norm_eps);
+ rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads),
+ 128, 0, ctx->stream>>>(
+ ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos,
+ ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim,
+ qkv_dim, q_offset, k_offset, c->rms_norm_eps);
if (launch_check() != 0) return -1;
if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset,
- k_offset, v_offset, qkv_dim, scale) != 0)
+ k_offset, v_offset, qkv_dim, scale,
+ wbf16 ? act : NULL) != 0)
return -1;
- if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
- total, c->q_dim, c->hidden_size) != 0)
- return -1;
+ if (wbf16) {
+ if (linear_bf16x(ctx, &l->wo, act, ctx->x,
+ total, c->q_dim, c->hidden_size,
+ c->q_dim, 1.0f) != 0)
+ return -1;
+ } else {
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
+ total, c->q_dim, c->hidden_size) != 0)
+ return -1;
+ }
int inter_count = total * c->intermediate_size;
if (wbf16) {