diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 03:34:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 03:34:13 +0200 |
| commit | 7a4106012b7b3d8c26878deb6d1720ac978b67db (patch) | |
| tree | e9a9d4d33c19eb15d2f72e675f26f13f66e08e50 /libffwd | |
| parent | 76758db35873a8f236e6ea0df544e71391edc4b1 (diff) | |
| download | ffwd-7a4106012b7b3d8c26878deb6d1720ac978b67db.tar.gz ffwd-7a4106012b7b3d8c26878deb6d1720ac978b67db.zip | |
cuda: trim bert reduced scratch
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/cuda.cu | 286 |
1 files changed, 213 insertions, 73 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index 5c5927f..cbc7a41 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -80,6 +80,7 @@ struct ffwd_cuda_ctx { float *ffwd_ln_b; int seq_cap; + int scratch_key; int batch_cap; int max_seq_cap; float *x; @@ -2826,9 +2827,150 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, return 0; } +enum { BERT_BF16_ACT_MIN_TOKENS = 1024 }; +enum { BERT_RESIDUAL16_MIN_TOKENS = 16384 }; +enum { BERT_FFN16_MIN_TOKENS = 1024 }; +enum { BERT_QKV16_MIN_TOKENS = 1024 }; + +typedef struct { + int compute_16bit; + int bf16_flash; + int f16_flash; + int flash; + int bf16_act; + int f16_act; + cuda_dtype_t act_dtype; + int bf16_resid; + int f16_resid; + int use_resid16; + int ffn16; + int delta16; + int cublaslt_avail; + int cublaslt; + int qkv16; + int direct_kv; +} bert_cuda_plan_t; + +static int bert_flash_mode(void); +static int bert_direct_kv_mode(void); +static int bert_bf16_act_mode(void); +static int bert_residual16_mode(void); +static int bert_ffn16_mode(void); +static int bert_delta16_mode(void); +static int bert_cublaslt_mode(void); +static int bert_qkv16_mode(void); + +static void bert_cuda_plan(ffwd_cuda_ctx_t *ctx, int total, bert_cuda_plan_t *p) { + memset(p, 0, sizeof(*p)); + p->act_dtype = CUDA_DTYPE_F32; + if (!ctx || ctx->config.family != FFWD_FAMILY_BERT || ctx->config.n_layers <= 0) + return; + + const ffwd_config_t *c = &ctx->config; + cublasComputeType_t gc = gemm_compute(); + p->compute_16bit = (gc == CUBLAS_COMPUTE_32F_FAST_16BF || gc == CUBLAS_COMPUTE_32F_FAST_16F); + + int bert_flash = bert_flash_mode(); + p->bf16_flash = p->compute_16bit && ctx->layers[0].qkv.bf16 && + ((c->head_dim == 64 && bert_flash >= 0) || (c->head_dim == 32 && bert_flash > 0)); + p->f16_flash = p->compute_16bit && ctx->layers[0].qkv.f16 && + ((c->head_dim == 64 && bert_flash >= 0) || (c->head_dim == 32 && bert_flash > 0)); + p->flash = p->bf16_flash || p->f16_flash; + + int act_mode = bert_bf16_act_mode(); + p->bf16_act = p->bf16_flash || (p->compute_16bit && ctx->layers[0].qkv.bf16 && act_mode >= 0 && + (act_mode == 1 || total >= BERT_BF16_ACT_MIN_TOKENS)); + p->f16_act = p->f16_flash; + p->act_dtype = p->f16_act ? CUDA_DTYPE_F16 : (p->bf16_act ? CUDA_DTYPE_BF16 : CUDA_DTYPE_F32); + + int resid_mode = bert_residual16_mode(); + p->bf16_resid = p->bf16_act && p->flash && resid_mode >= 0 && + (resid_mode > 0 || total >= BERT_RESIDUAL16_MIN_TOKENS); + p->f16_resid = p->f16_act && p->flash && resid_mode >= 0; + p->use_resid16 = p->bf16_resid || p->f16_resid; + + int ffn16_mode = bert_ffn16_mode(); + p->ffn16 = p->act_dtype != CUDA_DTYPE_F32 && ffn16_mode >= 0 && + (ffn16_mode > 0 || total >= BERT_FFN16_MIN_TOKENS); + p->delta16 = p->use_resid16 && bert_delta16_mode() >= 0; + + p->cublaslt_avail = + bert_cublaslt_mode() >= 0 && p->act_dtype != CUDA_DTYPE_F32 && ctx->lt != NULL; + p->cublaslt = p->cublaslt_avail && c->ffn_act == FFWD_ACT_GELU_ERF; + + int qkv16_mode = bert_qkv16_mode(); + p->qkv16 = p->flash && p->cublaslt_avail && qkv16_mode >= 0 && + (qkv16_mode > 0 || total >= BERT_QKV16_MIN_TOKENS); + p->direct_kv = p->flash && p->qkv16 && bert_direct_kv_mode() >= 0; +} + static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) { const ffwd_config_t *c = &ctx->config; - if (total > ctx->seq_cap) { + bert_cuda_plan_t bp; + bert_cuda_plan(ctx, total, &bp); + + int is_bert = c->family == FFWD_FAMILY_BERT; + int need_x = 1; + int need_x_norm = !is_bert; + int need_qkv = !is_bert || !bp.qkv16; + int need_qkv_16 = is_bert && bp.qkv16; + int need_attn_out = !is_bert || !bp.flash; + int need_ffn_gate_up = 1; + int need_ffn_gate_up_16 = 0; + int need_resid_delta_16 = is_bert && bp.delta16; + int need_x_bf16 = is_bert && bp.bf16_resid; + int need_x_f16 = is_bert && bp.f16_resid; + int need_kexp = 1; + int need_kexp_bf16 = ctx->weights_bf16; + int need_kexp_f16 = ctx->weights_f16; + int need_vexp = 1; + int vexp_elem_bytes = (int)sizeof(float); + int need_act_bf16 = ctx->weights_bf16; + int need_act_f16 = ctx->weights_f16; + int act_cols = 2 * c->intermediate_size; + int ffn_gate_up_cols = 2 * c->intermediate_size; + + if (is_bert) { + need_x_norm = 0; + need_ffn_gate_up = !bp.cublaslt && (bp.act_dtype == CUDA_DTYPE_F32 || !bp.ffn16); + need_ffn_gate_up_16 = bp.act_dtype != CUDA_DTYPE_F32 && bp.ffn16 && !bp.cublaslt; + ffn_gate_up_cols = c->intermediate_size; + need_kexp = !bp.flash || !bp.direct_kv; + need_kexp_bf16 = bp.flash && !bp.direct_kv && bp.act_dtype == CUDA_DTYPE_BF16; + need_kexp_f16 = bp.flash && !bp.direct_kv && bp.act_dtype == CUDA_DTYPE_F16; + need_vexp = !bp.flash || !bp.direct_kv; + vexp_elem_bytes = + bp.flash && bp.act_dtype != CUDA_DTYPE_F32 ? (int)sizeof(uint16_t) : (int)sizeof(float); + need_act_bf16 = bp.act_dtype == CUDA_DTYPE_BF16; + need_act_f16 = bp.act_dtype == CUDA_DTYPE_F16; + act_cols = c->intermediate_size; + if (act_cols < c->hidden_size) + act_cols = c->hidden_size; + if (act_cols < c->q_dim) + act_cols = c->q_dim; + } + + int scratch_key = 1; + scratch_key |= need_x_norm << 1; + scratch_key |= need_qkv << 2; + scratch_key |= need_qkv_16 << 3; + scratch_key |= need_attn_out << 4; + scratch_key |= need_ffn_gate_up << 5; + scratch_key |= need_ffn_gate_up_16 << 6; + scratch_key |= need_resid_delta_16 << 7; + scratch_key |= need_x_bf16 << 8; + scratch_key |= need_x_f16 << 9; + scratch_key |= need_kexp << 10; + scratch_key |= need_kexp_bf16 << 11; + scratch_key |= need_kexp_f16 << 12; + scratch_key |= need_vexp << 13; + scratch_key |= (vexp_elem_bytes == (int)sizeof(uint16_t)) << 14; + scratch_key |= need_act_bf16 << 15; + scratch_key |= need_act_f16 << 16; + scratch_key |= is_bert << 17; + scratch_key |= (bp.flash ? 1 : 0) << 18; + + if (total > ctx->seq_cap || scratch_key != ctx->scratch_key) { cudaFree(ctx->x); cudaFree(ctx->x_norm); cudaFree(ctx->qkv); @@ -2847,6 +2989,10 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se cudaFree(ctx->vexp); cudaFree(ctx->act_bf16); cudaFree(ctx->act_f16); + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + cudaFree(ctx->attn_scores); + cudaFree(ctx->attn_probs); ctx->x = ctx->x_norm = ctx->qkv = NULL; ctx->qkv_16 = NULL; ctx->attn_out = ctx->ffn_gate_up = NULL; @@ -2860,42 +3006,57 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se ctx->kexp_f16 = NULL; ctx->act_bf16 = NULL; ctx->act_f16 = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, - (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); - if ((ctx->weights_bf16 || ctx->weights_f16) && c->family == FFWD_FAMILY_BERT) + ctx->rope_cos = NULL; + ctx->rope_sin = NULL; + ctx->attn_scores = NULL; + ctx->attn_probs = NULL; + ctx->attn_scores_elems = 0; + ctx->attn_probs_elems = 0; + ctx->max_seq_cap = 0; + ctx->scratch_key = 0; + if (need_x) + CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); + if (need_x_norm) + CUDA_CHECK( + cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + if (need_qkv) + CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, + (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); + if (need_qkv_16) CUDA_CHECK(cudaMalloc(&ctx->qkv_16, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(uint16_t))); - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, - (size_t)total * 2 * c->intermediate_size * sizeof(float))); - if (ctx->weights_bf16 && c->family == FFWD_FAMILY_BERT) + if (need_attn_out) + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); + if (need_ffn_gate_up) + CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, + (size_t)total * ffn_gate_up_cols * sizeof(float))); + if (need_x_bf16) CUDA_CHECK( cudaMalloc(&ctx->x_bf16, (size_t)total * c->hidden_size * sizeof(__nv_bfloat16))); - if (ctx->weights_f16 && c->family == FFWD_FAMILY_BERT) + if (need_x_f16) CUDA_CHECK(cudaMalloc(&ctx->x_f16, (size_t)total * c->hidden_size * sizeof(__half))); CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); - CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); - if (ctx->weights_bf16) + if (need_kexp) + CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); + if (need_kexp_bf16) CUDA_CHECK(cudaMalloc(&ctx->kexp_bf16, (size_t)total * c->q_dim * sizeof(__nv_bfloat16))); - if (ctx->weights_f16) + if (need_kexp_f16) CUDA_CHECK(cudaMalloc(&ctx->kexp_f16, (size_t)total * c->q_dim * sizeof(__half))); - CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); - if (ctx->weights_bf16) - CUDA_CHECK(cudaMalloc(&ctx->act_bf16, - (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16))); - if (ctx->weights_f16) - CUDA_CHECK( - cudaMalloc(&ctx->act_f16, (size_t)total * 2 * c->intermediate_size * sizeof(__half))); - if ((ctx->weights_bf16 || ctx->weights_f16) && c->family == FFWD_FAMILY_BERT) + if (need_vexp) + CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * vexp_elem_bytes)); + if (need_act_bf16) + CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * act_cols * sizeof(__nv_bfloat16))); + if (need_act_f16) + CUDA_CHECK(cudaMalloc(&ctx->act_f16, (size_t)total * act_cols * sizeof(__half))); + if (need_ffn_gate_up_16) CUDA_CHECK(cudaMalloc(&ctx->ffn_gate_up_16, (size_t)total * c->intermediate_size * sizeof(uint16_t))); - if ((ctx->weights_bf16 || ctx->weights_f16) && c->family == FFWD_FAMILY_BERT) + if (need_resid_delta_16) CUDA_CHECK( cudaMalloc(&ctx->resid_delta_16, (size_t)total * c->hidden_size * sizeof(uint16_t))); ctx->seq_cap = total; + ctx->scratch_key = scratch_key; } if (batch + 1 > ctx->batch_cap) { cudaFree(ctx->offsets); @@ -2909,21 +3070,29 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se if (max_seq > ctx->max_seq_cap) { cudaFree(ctx->rope_cos); cudaFree(ctx->rope_sin); + ctx->rope_cos = NULL; + ctx->rope_sin = NULL; long long score_elems = (long long)c->n_heads * max_seq * max_seq; - if (score_elems > ctx->attn_scores_elems) { + int need_materialized_scores = !is_bert || !bp.flash; + if (need_materialized_scores && score_elems > ctx->attn_scores_elems) { cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; ctx->attn_scores_elems = 0; CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float))); ctx->attn_scores_elems = score_elems; } - if ((ctx->weights_bf16 || ctx->weights_f16) && score_elems > ctx->attn_probs_elems) { + if (need_materialized_scores && (ctx->weights_bf16 || ctx->weights_f16) && + score_elems > ctx->attn_probs_elems) { cudaFree(ctx->attn_probs); ctx->attn_probs = NULL; ctx->attn_probs_elems = 0; CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(uint16_t))); ctx->attn_probs_elems = score_elems; } + if (is_bert) { + ctx->max_seq_cap = max_seq; + return 0; + } float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); if (!hcos || !hsin) { @@ -3282,10 +3451,6 @@ static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx, // LayerNorm reductions in F32. It is auto-enabled only for large packed BERT // batches where L4 validation showed a stable long-sequence win; the env override // remains for A/B. -enum { BERT_BF16_ACT_MIN_TOKENS = 1024 }; -enum { BERT_RESIDUAL16_MIN_TOKENS = 16384 }; -enum { BERT_FFN16_MIN_TOKENS = 1024 }; -enum { BERT_QKV16_MIN_TOKENS = 1024 }; // Returns 1 (force on), -1 (force off), or 0 (auto: gate on token count). static int bert_bf16_act_mode(void) { static int init = 0, mode = 0; @@ -3399,25 +3564,14 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp int threads = 256; size_t ln_smem = (size_t)hidden * sizeof(float); - // Reduced-precision activation path: BF16 keeps the validated conservative - // gates; explicit FP16 storage uses the same F32-reduction LayerNorm kernels - // but stores the persistent BERT residual stream as FP16 unless the residual - // override forces it off. - cublasComputeType_t gc = gemm_compute(); - int compute_16bit = (gc == CUBLAS_COMPUTE_32F_FAST_16BF || gc == CUBLAS_COMPUTE_32F_FAST_16F); - int act_mode = bert_bf16_act_mode(); - int bert_flash = bert_flash_mode(); - int bf16_flash = - compute_16bit && c->n_layers > 0 && ctx->layers[0].qkv.bf16 && - ((c->head_dim == 64 && bert_flash >= 0) || (c->head_dim == 32 && bert_flash > 0)); - int f16_flash = compute_16bit && c->n_layers > 0 && ctx->layers[0].qkv.f16 && - ((c->head_dim == 64 && bert_flash >= 0) || (c->head_dim == 32 && bert_flash > 0)); - int flash = bf16_flash || f16_flash; - int bf16_act = - bf16_flash || (compute_16bit && c->n_layers > 0 && ctx->layers[0].qkv.bf16 && act_mode >= 0 && - (act_mode == 1 || total >= BERT_BF16_ACT_MIN_TOKENS)); - int f16_act = f16_flash; - cuda_dtype_t act_dtype = f16_act ? CUDA_DTYPE_F16 : (bf16_act ? CUDA_DTYPE_BF16 : CUDA_DTYPE_F32); + // Reduced-precision activation path. The same plan is used by + // ensure_buffers, so the forward loop only references scratch that was + // allocated for this exact path. + bert_cuda_plan_t bp; + bert_cuda_plan(ctx, total, &bp); + int flash = bp.flash; + int bf16_act = bp.bf16_act; + cuda_dtype_t act_dtype = bp.act_dtype; void *act16 = act_dtype == CUDA_DTYPE_F16 ? ctx->act_f16 : (act_dtype == CUDA_DTYPE_BF16 ? ctx->act_bf16 : NULL); void *resid16 = act_dtype == CUDA_DTYPE_F16 ? ctx->x_f16 @@ -3426,28 +3580,14 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp __nv_bfloat16 *resid_bf16 = (__nv_bfloat16 *)ctx->x_bf16; __half *act_f16 = (__half *)ctx->act_f16; __half *resid_f16 = (__half *)ctx->x_f16; - int resid_mode = bert_residual16_mode(); - int bf16_resid = bf16_act && flash && resid_bf16 && resid_mode >= 0 && - (resid_mode > 0 || total >= BERT_RESIDUAL16_MIN_TOKENS); - int f16_resid = f16_act && flash && resid_f16 && resid_mode >= 0; - int use_resid16 = bf16_resid || f16_resid; - int ffn16_mode = bert_ffn16_mode(); - int ffn16 = act_dtype != CUDA_DTYPE_F32 && ctx->ffn_gate_up_16 && ffn16_mode >= 0 && - (ffn16_mode > 0 || total >= BERT_FFN16_MIN_TOKENS); - // The delta path only exists when the residual stream is already 16-bit (it - // replaces the F32 projection delta the *_resid_* LayerNorm kernels read), so - // it rides residual16's gate: on whenever residual16 is, unless forced off. - int delta16 = use_resid16 && ctx->resid_delta_16 && bert_delta16_mode() >= 0; - // cuBLASLt fused epilogues, default on for the 16-bit BERT path (wins at every - // measured batch/length incl. batch 1, so not token-gated; FFWD_CUDA_CUBLASLT=0 - // forces the GEMM + separate kernels back for A/B). `cublaslt_avail` gates the - // qkv BIAS epilogue (any activation); the gate-up GELU_BIAS epilogue also needs - // exact erf-GeLU (the cuBLASLt GELU epilogue is erf-only). - int cublaslt_avail = bert_cublaslt_mode() >= 0 && act_dtype != CUDA_DTYPE_F32 && ctx->lt != NULL; - int cublaslt = cublaslt_avail && c->ffn_act == FFWD_ACT_GELU_ERF; - int qkv16_mode = bert_qkv16_mode(); - int qkv16 = flash && cublaslt_avail && ctx->qkv_16 && qkv16_mode >= 0 && - (qkv16_mode > 0 || total >= BERT_QKV16_MIN_TOKENS); + int bf16_resid = bp.bf16_resid; + int f16_resid = bp.f16_resid; + int use_resid16 = bp.use_resid16; + int ffn16 = bp.ffn16; + int delta16 = bp.delta16; + int cublaslt_avail = bp.cublaslt_avail; + int cublaslt = bp.cublaslt; + int qkv16 = bp.qkv16; if (bf16_resid) { bert_embed_layer_norm_bf16_kernel<<<total, 256, ln_smem, ctx->stream>>>( @@ -3474,9 +3614,9 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp if (launch_check() != 0) return -1; - // The materialized BERT fallback uses F32 attention here; BERT flash ignores - // this pointer layout and writes BF16 output directly to `act`. - if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, CUDA_DTYPE_F32) != 0) + // The materialized BERT fallback uses F32 attention here. BERT flash keeps + // scores in registers and does not use the pointer-array batched GEMM setup. + if (!flash && attn_batched_setup(ctx, batch, q_offset, qkv_dim, CUDA_DTYPE_F32) != 0) return -1; for (int layer = 0; layer < c->n_layers; layer++) { |
