diff options
Diffstat (limited to 'embed_cuda.cu')
| -rw-r--r-- | embed_cuda.cu | 144 |
1 files changed, 99 insertions, 45 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu index e0ff898..dfc3e32 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -58,6 +58,7 @@ struct pplx_cuda_ctx { float *kexp; float *vexp; float *attn_scores; + void *attn_probs; // BF16 softmax output (bf16 weights) int *span_starts; int *span_lens; int pooled_rows_cap; @@ -371,15 +372,19 @@ __global__ static void rms_norm_kernel(OUT_T *out, const float *x, x[(size_t)row * dim + d] * inv * weight[d]); } -__global__ static void rms_norm_rope_head_kernel(float *x, const float *weight, - const int *positions, - const float *cosv, - const float *sinv, - int rows, int heads, - int head_dim, - int row_stride, - int base_offset, - float eps) +// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with +// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset. +__global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm, + const float *k_norm, + const int *positions, + const float *cosv, + const float *sinv, + int rows, int n_heads, + int n_kv_heads, + int head_dim, + int row_stride, + int q_offset, int k_offset, + float eps) { __shared__ float partial[4]; __shared__ float total; @@ -391,6 +396,9 @@ __global__ static void rms_norm_rope_head_kernel(float *x, const float *weight, int half = head_dim / 2; int pair_d = tid < half ? tid + half : tid - half; float sign = tid < half ? -1.0f : 1.0f; + const float *weight = head < n_heads ? q_norm : k_norm; + int base_offset = head < n_heads ? q_offset : k_offset; + if (head >= n_heads) head -= n_heads; size_t base = (size_t)row * row_stride + base_offset + head * head_dim; float v = tid < head_dim ? x[base + tid] : 0.0f; float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; @@ -443,8 +451,10 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, // Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim] // per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times) // so the attention GEMMs can stride uniformly over query heads. +// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly. +template <typename V_T> __global__ static void attn_expand_kv_kernel( - float *kexp, float *vexp, const float *qkv, int total, int n_heads, + float *kexp, V_T *vexp, const float *qkv, int total, int n_heads, int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset) { int q_dim = n_heads * head_dim; @@ -458,15 +468,19 @@ __global__ static void attn_expand_kv_kernel( int kv = h / (n_heads / n_kv_heads); const float *base = qkv + (size_t)t * qkv_dim; kexp[idx] = base[k_offset + kv * head_dim + e]; - vexp[idx] = base[v_offset + kv * head_dim + e]; + store_act(vexp, idx, base[v_offset + kv * head_dim + e]); } // Row softmax over the key dimension for one sequence's score tensor laid out // as scores[head * L * L + i * L + j]; normalizes across j for each (head, i). // One block per (head, query) row; scores are pre-scaled by the QK^T GEMM. -__global__ static void attn_softmax_kernel(float *scores, int L) +// P_T is float with probs == scores (in place), or __nv_bfloat16 when the +// probabilities feed a BF16 @V GEMM. +template <typename P_T> +__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) { float *s = scores + (size_t)blockIdx.x * L; + P_T *p = probs + (size_t)blockIdx.x * L; int tid = threadIdx.x; __shared__ float red[256]; @@ -494,7 +508,8 @@ __global__ static void attn_softmax_kernel(float *scores, int L) __syncthreads(); } float inv = 1.0f / red[0]; - for (int j = tid; j < L; j += blockDim.x) s[j] *= inv; + for (int j = tid; j < L; j += blockDim.x) + store_act(p, (size_t)j, s[j] * inv); } __global__ static void mean_pool_kernel(float *out, const float *x, @@ -644,9 +659,13 @@ static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, // -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and // ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets // is the host copy of the packed sequence boundaries. +// out_bf16 != NULL selects the BF16-weight layout: V and the softmax +// probabilities are stored BF16 and the @V GEMM emits BF16 attention output +// at out_bf16 (stride q_dim), ready for the wo projection without a cast. static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, int batch, int q_offset, int k_offset, - int v_offset, int qkv_dim, float scale) + int v_offset, int qkv_dim, float scale, + __nv_bfloat16 *out_bf16) { const pplx_config_t *c = &ctx->config; int q_dim = c->q_dim; @@ -657,10 +676,17 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, int threads = 256; size_t exp_count = (size_t)total * q_dim; - attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, - 0, ctx->stream>>>( - ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, - k_offset, v_offset); + if (out_bf16) { + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, + 0, ctx->stream>>>( + ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, + c->n_kv_heads, hd, qkv_dim, k_offset, v_offset); + } else { + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, + 0, ctx->stream>>>( + ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, + qkv_dim, k_offset, v_offset); + } if (launch_check() != 0) return -1; for (int b = 0; b < batch; b++) { @@ -668,23 +694,40 @@ static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, int L = h_offsets[b + 1] - start; const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; const float *K = ctx->kexp + (size_t)start * q_dim; - const float *V = ctx->vexp + (size_t)start * q_dim; - float *O = ctx->attn_out + (size_t)start * q_dim; /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ if (gemm_strided_batched( ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) return -1; - attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( - ctx->attn_scores, L); - if (launch_check() != 0) return -1; - /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ - if (gemm_strided_batched( + if (out_bf16) { + __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs; + const __nv_bfloat16 *V = + (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim; + __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim; + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, P, L); + if (launch_check() != 0) return -1; + /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ + CUBLAS_CHECK(cublasGemmStridedBatchedEx( ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L, - &beta, O, q_dim, hd, H) != 0) - return -1; + &one, V, CUDA_R_16BF, q_dim, hd, + P, CUDA_R_16BF, L, (long long)L * L, + &beta, O, CUDA_R_16BF, q_dim, hd, + H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } else { + const float *V = ctx->vexp + (size_t)start * q_dim; + float *O = ctx->attn_out + (size_t)start * q_dim; + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, ctx->attn_scores, L); + if (launch_check() != 0) return -1; + /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ + if (gemm_strided_batched( + ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, + &one, V, q_dim, hd, ctx->attn_scores, L, + (long long)L * L, &beta, O, q_dim, hd, H) != 0) + return -1; + } } return 0; } @@ -736,8 +779,14 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se cudaFree(ctx->rope_sin); cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; + cudaFree(ctx->attn_probs); + ctx->attn_probs = NULL; CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)c->n_heads * max_seq * max_seq * sizeof(float))); + if (g_weights_bf16) + CUDA_CHECK(cudaMalloc(&ctx->attn_probs, + (size_t)c->n_heads * max_seq * max_seq * + sizeof(__nv_bfloat16))); float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); if (!hcos || !hsin) { @@ -909,6 +958,7 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx) cudaFree(ctx->vexp); cudaFree(ctx->act_bf16); cudaFree(ctx->attn_scores); + cudaFree(ctx->attn_probs); cudaFree(ctx->span_starts); cudaFree(ctx->span_lens); if (ctx->blas) cublasDestroy(ctx->blas); @@ -988,11 +1038,10 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, int k_offset = c->q_dim; int v_offset = c->q_dim + c->kv_dim; int qkv_dim = c->q_dim + 2 * c->kv_dim; - // With BF16 weights the norm and SiLU kernels store their results as BF16 - // directly into ctx->act_bf16, which the next projection GEMM consumes - // before any later producer reuses the buffer (single stream). This - // removes three cast launches per layer; only the attention output still - // needs a cast because cuBLAS cannot emit a BF16 C from F32 A/B operands. + // With BF16 weights every producer (norm, SiLU, attention @V GEMM) + // stores its result as BF16 directly into ctx->act_bf16, which the next + // projection GEMM consumes before any later producer reuses the buffer + // (single stream). No cast launches remain in the layer loop. __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16; for (int layer = 0; layer < c->n_layers; layer++) { cuda_layer_t *l = &ctx->layers[layer]; @@ -1016,23 +1065,28 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, return -1; } - rms_norm_rope_head_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>( - ctx->qkv, l->q_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, - total, c->n_heads, c->head_dim, qkv_dim, q_offset, - c->rms_norm_eps); - rms_norm_rope_head_kernel<<<dim3(total, c->n_kv_heads), 128, 0, ctx->stream>>>( - ctx->qkv, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, - total, c->n_kv_heads, c->head_dim, qkv_dim, k_offset, - c->rms_norm_eps); + rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), + 128, 0, ctx->stream>>>( + ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, + ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim, + qkv_dim, q_offset, k_offset, c->rms_norm_eps); if (launch_check() != 0) return -1; if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, - k_offset, v_offset, qkv_dim, scale) != 0) + k_offset, v_offset, qkv_dim, scale, + wbf16 ? act : NULL) != 0) return -1; - if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, - total, c->q_dim, c->hidden_size) != 0) - return -1; + if (wbf16) { + if (linear_bf16x(ctx, &l->wo, act, ctx->x, + total, c->q_dim, c->hidden_size, + c->q_dim, 1.0f) != 0) + return -1; + } else { + if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, + total, c->q_dim, c->hidden_size) != 0) + return -1; + } int inter_count = total * c->intermediate_size; if (wbf16) { |
