aboutsummaryrefslogtreecommitdiff
path: root/embed_cuda.cu
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-11 08:10:21 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit54e6e6bbef396eb1d38c09517d8157301fb8f151 (patch)
tree8d87e945918913363c66bf46eee070ad80d6b4bb /embed_cuda.cu
parent002e648fee56b593b3573f600c73bd67cbd89469 (diff)
downloadffwd-54e6e6bbef396eb1d38c09517d8157301fb8f151.tar.gz
ffwd-54e6e6bbef396eb1d38c09517d8157301fb8f151.zip
cuda: drop BF16 attention internals, keep fused rope + batched F32 attention
Diffstat (limited to 'embed_cuda.cu')
-rw-r--r--embed_cuda.cu167
1 files changed, 42 insertions, 125 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 366572b..8fecf3a 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -58,14 +58,12 @@ struct pplx_cuda_ctx {
float *kexp;
float *vexp;
float *attn_scores;
- void *attn_probs; // BF16 softmax output (bf16 weights)
long long attn_scores_elems;
- long long attn_probs_elems;
// Equal-length micro-batch attention: pointer arrays for batched GEMMs,
// built once per forward pass (layer-invariant buffer addresses).
// attn_G is the number of sequences whose score tensors fit the scratch
// budget at once; 0 disables the batched path for this forward.
- const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads
+ const void **attn_ptrs; // device: [K|Q|S|V|O] x batch x n_heads
const void **attn_ptrs_host;
int attn_ptrs_cap; // capacity in pointer entries
int attn_G;
@@ -465,10 +463,8 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
// so the attention GEMMs can stride uniformly over query heads.
-// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly.
-template <typename V_T>
__global__ static void attn_expand_kv_kernel(
- float *kexp, V_T *vexp, const float *qkv, int total, int n_heads,
+ float *kexp, float *vexp, const float *qkv, int total, int n_heads,
int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset)
{
int q_dim = n_heads * head_dim;
@@ -482,19 +478,15 @@ __global__ static void attn_expand_kv_kernel(
int kv = h / (n_heads / n_kv_heads);
const float *base = qkv + (size_t)t * qkv_dim;
kexp[idx] = base[k_offset + kv * head_dim + e];
- store_act(vexp, idx, base[v_offset + kv * head_dim + e]);
+ vexp[idx] = base[v_offset + kv * head_dim + e];
}
// Row softmax over the key dimension for one sequence's score tensor laid out
// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i).
// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
-// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
-// probabilities feed a BF16 @V GEMM.
-template <typename P_T>
-__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
+__global__ static void attn_softmax_kernel(float *scores, int L)
{
float *s = scores + (size_t)blockIdx.x * L;
- P_T *p = probs + (size_t)blockIdx.x * L;
int tid = threadIdx.x;
__shared__ float red[256];
@@ -522,8 +514,7 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
__syncthreads();
}
float inv = 1.0f / red[0];
- for (int j = tid; j < L; j += blockDim.x)
- store_act(p, (size_t)j, s[j] * inv);
+ for (int j = tid; j < L; j += blockDim.x) s[j] *= inv;
}
__global__ static void mean_pool_kernel(float *out, const float *x,
@@ -695,7 +686,6 @@ static int attn_batched_setup(pplx_cuda_ctx_t *ctx, int batch, int q_offset,
if (G > batch) G = batch;
if (G < 2) return 0;
- int bf16 = g_weights_bf16;
long long want = (long long)G * per;
if (want > ctx->attn_scores_elems) {
cudaFree(ctx->attn_scores);
@@ -705,16 +695,8 @@ static int attn_batched_setup(pplx_cuda_ctx_t *ctx, int batch, int q_offset,
(size_t)want * sizeof(float)));
ctx->attn_scores_elems = want;
}
- if (bf16 && want > ctx->attn_probs_elems) {
- cudaFree(ctx->attn_probs);
- ctx->attn_probs = NULL;
- ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs,
- (size_t)want * sizeof(__nv_bfloat16)));
- ctx->attn_probs_elems = want;
- }
- int entries = 6 * batch * H;
+ int entries = 5 * batch * H;
if (entries > ctx->attn_ptrs_cap) {
cudaFree((void *)ctx->attn_ptrs);
free((void *)ctx->attn_ptrs_host);
@@ -732,13 +714,9 @@ static int attn_batched_setup(pplx_cuda_ctx_t *ctx, int batch, int q_offset,
const void **Q = K + (size_t)batch * H;
const void **S = Q + (size_t)batch * H;
const void **V = S + (size_t)batch * H;
- const void **P = V + (size_t)batch * H;
- const void **O = P + (size_t)batch * H;
- const __nv_bfloat16 *vb = (const __nv_bfloat16 *)ctx->vexp;
- const __nv_bfloat16 *pb = (const __nv_bfloat16 *)ctx->attn_probs;
- const __nv_bfloat16 *ab = (const __nv_bfloat16 *)ctx->act_bf16;
+ const void **O = V + (size_t)batch * H;
for (int b = 0; b < batch; b++) {
- // Score/probability rows cycle every G sequences (chunked reuse).
+ // Score rows cycle every G sequences (chunked reuse).
long long srow = (long long)(b % G) * per;
size_t tok = (size_t)b * L;
for (int h = 0; h < H; h++) {
@@ -746,15 +724,8 @@ static int attn_batched_setup(pplx_cuda_ctx_t *ctx, int batch, int q_offset,
K[i] = ctx->kexp + tok * q_dim + (size_t)h * hd;
Q[i] = ctx->qkv + tok * qkv_dim + q_offset + (size_t)h * hd;
S[i] = ctx->attn_scores + srow + (long long)h * L * L;
- if (bf16) {
- V[i] = vb + tok * q_dim + (size_t)h * hd;
- P[i] = pb + srow + (long long)h * L * L;
- O[i] = ab + tok * q_dim + (size_t)h * hd;
- } else {
- V[i] = ctx->vexp + tok * q_dim + (size_t)h * hd;
- P[i] = S[i];
- O[i] = ctx->attn_out + tok * q_dim + (size_t)h * hd;
- }
+ V[i] = ctx->vexp + tok * q_dim + (size_t)h * hd;
+ O[i] = ctx->attn_out + tok * q_dim + (size_t)h * hd;
}
}
CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host,
@@ -765,13 +736,9 @@ static int attn_batched_setup(pplx_cuda_ctx_t *ctx, int batch, int q_offset,
return 0;
}
-// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
-// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
-// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
int batch, int q_offset, int k_offset,
- int v_offset, int qkv_dim, float scale,
- __nv_bfloat16 *out_bf16)
+ int v_offset, int qkv_dim, float scale)
{
const pplx_config_t *c = &ctx->config;
int q_dim = c->q_dim;
@@ -782,17 +749,10 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
int threads = 256;
size_t exp_count = (size_t)total * q_dim;
- if (out_bf16) {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
- 0, ctx->stream>>>(
- ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H,
- c->n_kv_heads, hd, qkv_dim, k_offset, v_offset);
- } else {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
- 0, ctx->stream>>>(
- ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd,
- qkv_dim, k_offset, v_offset);
- }
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
+ 0, ctx->stream>>>(
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd,
+ qkv_dim, k_offset, v_offset);
if (launch_check() != 0) return -1;
if (ctx->attn_G >= 2) {
@@ -801,11 +761,7 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
const void *const *Q = K + (size_t)batch * H;
void *const *S = (void *const *)(Q + (size_t)batch * H);
const void *const *V = (const void *const *)S + (size_t)batch * H;
- const void *const *P = V + (size_t)batch * H;
- void *const *O = (void *const *)(P + (size_t)batch * H);
- cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F;
- cublasComputeType_t av_ct =
- out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute();
+ void *const *O = (void *const *)(V + (size_t)batch * H);
for (int s = 0; s < batch; s += G) {
int g = batch - s < G ? batch - s : G;
int n = g * H;
@@ -817,21 +773,16 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
Q + off, CUDA_R_32F, qkv_dim,
&beta, S + off, CUDA_R_32F, L,
n, gemm_compute(), CUBLAS_GEMM_DEFAULT));
- if (out_bf16) {
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
- } else {
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
- }
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, L);
if (launch_check() != 0) return -1;
- /* O = P @ V, batched over (seq, head) */
+ /* O = P @ V, batched over (seq, head); P aliases S in place */
CUBLAS_CHECK(cublasGemmBatchedEx(
ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V + off, pv, q_dim,
- P + off, pv, L,
- &beta, O + off, pv, q_dim,
- n, av_ct, CUBLAS_GEMM_DEFAULT));
+ &one, V + off, CUDA_R_32F, q_dim,
+ (const void *const *)S + off, CUDA_R_32F, L,
+ &beta, O + off, CUDA_R_32F, q_dim,
+ n, gemm_compute(), CUBLAS_GEMM_DEFAULT));
}
return 0;
}
@@ -847,34 +798,17 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
&alpha, K, q_dim, hd, Q, qkv_dim, hd,
&beta, ctx->attn_scores, L, (long long)L * L, H) != 0)
return -1;
- if (out_bf16) {
- __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs;
- const __nv_bfloat16 *V =
- (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
- __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, P, L);
- if (launch_check() != 0) return -1;
- /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- CUBLAS_CHECK(cublasGemmStridedBatchedEx(
+ const float *V = ctx->vexp + (size_t)start * q_dim;
+ float *O = ctx->attn_out + (size_t)start * q_dim;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, L);
+ if (launch_check() != 0) return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ if (gemm_strided_batched(
ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, CUDA_R_16BF, q_dim, hd,
- P, CUDA_R_16BF, L, (long long)L * L,
- &beta, O, CUDA_R_16BF, q_dim, hd,
- H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
- } else {
- const float *V = ctx->vexp + (size_t)start * q_dim;
- float *O = ctx->attn_out + (size_t)start * q_dim;
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
- if (launch_check() != 0) return -1;
- /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- if (gemm_strided_batched(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, q_dim, hd, ctx->attn_scores, L,
- (long long)L * L, &beta, O, q_dim, hd, H) != 0)
- return -1;
- }
+ &one, V, q_dim, hd, ctx->attn_scores, L,
+ (long long)L * L, &beta, O, q_dim, hd, H) != 0)
+ return -1;
}
return 0;
}
@@ -933,15 +867,6 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
(size_t)score_elems * sizeof(float)));
ctx->attn_scores_elems = score_elems;
}
- if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) {
- cudaFree(ctx->attn_probs);
- ctx->attn_probs = NULL;
- ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs,
- (size_t)score_elems *
- sizeof(__nv_bfloat16)));
- ctx->attn_probs_elems = score_elems;
- }
float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
if (!hcos || !hsin) {
@@ -1113,7 +1038,6 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
cudaFree(ctx->vexp);
cudaFree(ctx->act_bf16);
cudaFree(ctx->attn_scores);
- cudaFree(ctx->attn_probs);
cudaFree((void *)ctx->attn_ptrs);
free((void *)ctx->attn_ptrs_host);
cudaFree(ctx->span_starts);
@@ -1195,10 +1119,11 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
int k_offset = c->q_dim;
int v_offset = c->q_dim + c->kv_dim;
int qkv_dim = c->q_dim + 2 * c->kv_dim;
- // With BF16 weights every producer (norm, SiLU, attention @V GEMM)
- // stores its result as BF16 directly into ctx->act_bf16, which the next
- // projection GEMM consumes before any later producer reuses the buffer
- // (single stream). No cast launches remain in the layer loop.
+ // With BF16 weights the norm and SiLU kernels store their results as BF16
+ // directly into ctx->act_bf16, which the next projection GEMM consumes
+ // before any later producer reuses the buffer (single stream). The
+ // attention output keeps one cast per layer: cuBLAS cannot emit a BF16 C
+ // from F32 A/B operands, and attention internals stay F32.
__nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
return -1;
@@ -1232,20 +1157,12 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
if (launch_check() != 0) return -1;
if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset,
- k_offset, v_offset, qkv_dim, scale,
- wbf16 ? act : NULL) != 0)
+ k_offset, v_offset, qkv_dim, scale) != 0)
return -1;
- if (wbf16) {
- if (linear_bf16x(ctx, &l->wo, act, ctx->x,
- total, c->q_dim, c->hidden_size,
- c->q_dim, 1.0f) != 0)
- return -1;
- } else {
- if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
- total, c->q_dim, c->hidden_size) != 0)
- return -1;
- }
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
+ total, c->q_dim, c->hidden_size) != 0)
+ return -1;
int inter_count = total * c->intermediate_size;
if (wbf16) {