aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-23 02:12:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-23 02:12:05 +0200
commitefacda4c9cfe3dee45c632fd913716f92350b9d1 (patch)
tree2d624bc47972d10a979aab29e2c7f90d015ffc6c /libffwd
parent475366f81460604bc778c0d0a3fff773fa83ae74 (diff)
downloadffwd-efacda4c9cfe3dee45c632fd913716f92350b9d1.tar.gz
ffwd-efacda4c9cfe3dee45c632fd913716f92350b9d1.zip
cuda: write bert residual delta 16-bit on the residual16 path
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu165
1 files changed, 155 insertions, 10 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 1434cc3..2fdb12a 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -77,6 +77,7 @@ struct ffwd_cuda_ctx {
float *attn_out;
float *ffn_gate_up;
void *ffn_gate_up_16; // BERT reduced-precision FFN intermediate
+ void *resid_delta_16; // BERT reduced-precision residual delta (wo/down output)
void *x_bf16; // Persistent BERT BF16 residual stream (gated)
void *x_f16; // Persistent BERT FP16 residual stream (explicit f16 path)
void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
@@ -1743,6 +1744,85 @@ __global__ static void layer_norm_bias_resid_f16_kernel(__half *out,
}
}
+// Reduced-precision residual delta: identical reductions to the *_resid_*
+// kernels above, but the freshly-projected delta (wo / down output) is read as
+// the active 16-bit dtype instead of F32. The residual add and the LayerNorm
+// mean/variance stay in F32; only the projection output is rounded, which
+// removes one F32 [rows x dim] write+reread per attention/FFN block. Active with
+// the 16-bit residual stream (FFWD_CUDA_BERT_DELTA16=0 forces the F32 delta back
+// for A/B); validated at parity, cosine vs F32 ~0.99998 on bge-base.
+__global__ static void layer_norm_bias_resid_bf16_delta16_kernel(__nv_bfloat16 *out,
+ const __nv_bfloat16 *resid,
+ const __nv_bfloat16 *delta,
+ const float *bias,
+ const float *gamma,
+ const float *beta,
+ int rows,
+ int dim,
+ float eps) {
+ extern __shared__ float rowbuf[];
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ if (row >= rows)
+ return;
+ const __nv_bfloat16 *dr = delta + (size_t)row * dim;
+ const __nv_bfloat16 *rr = resid + (size_t)row * dim;
+ float s = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float v = __bfloat162float(dr[d]) + __bfloat162float(rr[d]) + bias[d];
+ rowbuf[d] = v;
+ s += v;
+ }
+ float mean = block_sum(s) / (float)dim;
+ float vs = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float c = rowbuf[d] - mean;
+ vs += c * c;
+ }
+ float inv = rsqrtf(block_sum(vs) / (float)dim + eps);
+ __nv_bfloat16 *o = out + (size_t)row * dim;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float nv = (rowbuf[d] - mean) * inv * gamma[d] + beta[d];
+ o[d] = __float2bfloat16(nv);
+ }
+}
+
+__global__ static void layer_norm_bias_resid_f16_delta16_kernel(__half *out,
+ const __half *resid,
+ const __half *delta,
+ const float *bias,
+ const float *gamma,
+ const float *beta,
+ int rows,
+ int dim,
+ float eps) {
+ extern __shared__ float rowbuf[];
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ if (row >= rows)
+ return;
+ const __half *dr = delta + (size_t)row * dim;
+ const __half *rr = resid + (size_t)row * dim;
+ float s = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float v = __half2float(dr[d]) + __half2float(rr[d]) + bias[d];
+ rowbuf[d] = v;
+ s += v;
+ }
+ float mean = block_sum(s) / (float)dim;
+ float vs = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float c = rowbuf[d] - mean;
+ vs += c * c;
+ }
+ float inv = rsqrtf(block_sum(vs) / (float)dim + eps);
+ __half *o = out + (size_t)row * dim;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float nv = (rowbuf[d] - mean) * inv * gamma[d] + beta[d];
+ o[d] = __float2half(nv);
+ }
+}
+
__global__ static void bias_gelu_kernel(float *x, const float *bias, int rows, int dim) {
size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
size_t n = (size_t)rows * dim;
@@ -2488,6 +2568,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
cudaFree(ctx->attn_out);
cudaFree(ctx->ffn_gate_up);
cudaFree(ctx->ffn_gate_up_16);
+ cudaFree(ctx->resid_delta_16);
cudaFree(ctx->x_bf16);
cudaFree(ctx->x_f16);
cudaFree(ctx->token_ids);
@@ -2501,6 +2582,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
ctx->x = ctx->x_norm = ctx->qkv = NULL;
ctx->attn_out = ctx->ffn_gate_up = NULL;
ctx->ffn_gate_up_16 = NULL;
+ ctx->resid_delta_16 = NULL;
ctx->x_bf16 = NULL;
ctx->x_f16 = NULL;
ctx->token_ids = ctx->positions = NULL;
@@ -2538,6 +2620,9 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
if ((ctx->weights_bf16 || ctx->weights_f16) && c->family == FFWD_FAMILY_BERT)
CUDA_CHECK(cudaMalloc(&ctx->ffn_gate_up_16,
(size_t)total * c->intermediate_size * sizeof(uint16_t)));
+ if ((ctx->weights_bf16 || ctx->weights_f16) && c->family == FFWD_FAMILY_BERT)
+ CUDA_CHECK(
+ cudaMalloc(&ctx->resid_delta_16, (size_t)total * c->hidden_size * sizeof(uint16_t)));
ctx->seq_cap = total;
}
if (batch + 1 > ctx->batch_cap) {
@@ -2787,6 +2872,7 @@ void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) {
cudaFree(ctx->attn_out);
cudaFree(ctx->ffn_gate_up);
cudaFree(ctx->ffn_gate_up_16);
+ cudaFree(ctx->resid_delta_16);
cudaFree(ctx->x_bf16);
cudaFree(ctx->x_f16);
cudaFree(ctx->pooled_out);
@@ -2954,6 +3040,24 @@ static int bert_ffn16_mode(void) {
return mode;
}
+// BERT reduced-precision residual delta. Rides the residual16 gate (on whenever
+// the 16-bit residual stream is active); 0/off forces the F32 projection delta
+// back for A/B, 1/on is explicit-on. It writes the wo/down projection output as
+// the active 16-bit dtype (read back by the *_delta16 LayerNorm kernels) instead
+// of an F32 delta.
+static int bert_delta16_mode(void) {
+ static int init = 0, mode = 0;
+ if (!init) {
+ const char *e = getenv("FFWD_CUDA_BERT_DELTA16");
+ if (e && (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "force")))
+ mode = 1;
+ else if (e && (!strcmp(e, "0") || !strcmp(e, "off")))
+ mode = -1;
+ init = 1;
+ }
+ return mode;
+}
+
static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch) {
const ffwd_config_t *c = &ctx->config;
int total = 0;
@@ -3004,6 +3108,10 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp
int ffn16_mode = bert_ffn16_mode();
int ffn16 = act_dtype != CUDA_DTYPE_F32 && ctx->ffn_gate_up_16 && ffn16_mode >= 0 &&
(ffn16_mode > 0 || total >= BERT_FFN16_MIN_TOKENS);
+ // The delta path only exists when the residual stream is already 16-bit (it
+ // replaces the F32 projection delta the *_resid_* LayerNorm kernels read), so
+ // it rides residual16's gate: on whenever residual16 is, unless forced off.
+ int delta16 = use_resid16 && ctx->resid_delta_16 && bert_delta16_mode() >= 0;
if (bf16_resid) {
bert_embed_layer_norm_bf16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
@@ -3067,20 +3175,38 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp
// (also emitting bf16 act for the FFN gate-up GEMM in the bf16 path). With
// flash the attn output is already bf16 in `act`, so wo reads it directly.
if (flash && act_dtype == CUDA_DTYPE_BF16) {
- if (linear_bf16x(ctx, &l->wo, act_bf16, ctx->x, total, c->q_dim, hidden, c->q_dim,
- use_resid16 ? 0.0f : 1.0f) != 0)
+ if (delta16) {
+ if (linear_16x_to_16(ctx, &l->wo, act_bf16, CUDA_DTYPE_BF16, ctx->resid_delta_16,
+ CUDA_DTYPE_BF16, total, c->q_dim, hidden, c->q_dim) != 0)
+ return -1;
+ } else if (linear_bf16x(ctx, &l->wo, act_bf16, ctx->x, total, c->q_dim, hidden, c->q_dim,
+ use_resid16 ? 0.0f : 1.0f) != 0) {
return -1;
+ }
} else if (flash && act_dtype == CUDA_DTYPE_F16) {
- if (linear_f16x(ctx, &l->wo, act_f16, ctx->x, total, c->q_dim, hidden, c->q_dim,
- use_resid16 ? 0.0f : 1.0f) != 0)
+ if (delta16) {
+ if (linear_16x_to_16(ctx, &l->wo, act_f16, CUDA_DTYPE_F16, ctx->resid_delta_16,
+ CUDA_DTYPE_F16, total, c->q_dim, hidden, c->q_dim) != 0)
+ return -1;
+ } else if (linear_f16x(ctx, &l->wo, act_f16, ctx->x, total, c->q_dim, hidden, c->q_dim,
+ use_resid16 ? 0.0f : 1.0f) != 0) {
return -1;
+ }
} else if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0) {
return -1;
}
- if (bf16_resid) {
+ if (bf16_resid && delta16) {
+ layer_norm_bias_resid_bf16_delta16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
+ resid_bf16, resid_bf16, (const __nv_bfloat16 *)ctx->resid_delta_16, l->o_bias,
+ l->input_norm, l->attn_ln_bias, total, hidden, eps);
+ } else if (bf16_resid) {
layer_norm_bias_resid_bf16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
resid_bf16, resid_bf16, ctx->x, l->o_bias, l->input_norm, l->attn_ln_bias, total,
hidden, eps);
+ } else if (f16_resid && delta16) {
+ layer_norm_bias_resid_f16_delta16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
+ resid_f16, resid_f16, (const __half *)ctx->resid_delta_16, l->o_bias, l->input_norm,
+ l->attn_ln_bias, total, hidden, eps);
} else if (f16_resid) {
layer_norm_bias_resid_f16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
resid_f16, resid_f16, ctx->x, l->o_bias, l->input_norm, l->attn_ln_bias, total,
@@ -3174,21 +3300,40 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp
if (launch_check() != 0)
return -1;
if (act_dtype == CUDA_DTYPE_BF16) {
- if (linear_bf16x(ctx, &l->down_proj, act_bf16, ctx->x, total, inter, hidden, inter,
- use_resid16 ? 0.0f : 1.0f) != 0)
+ if (delta16) {
+ if (linear_16x_to_16(ctx, &l->down_proj, act_bf16, CUDA_DTYPE_BF16,
+ ctx->resid_delta_16, CUDA_DTYPE_BF16, total, inter, hidden,
+ inter) != 0)
+ return -1;
+ } else if (linear_bf16x(ctx, &l->down_proj, act_bf16, ctx->x, total, inter, hidden, inter,
+ use_resid16 ? 0.0f : 1.0f) != 0) {
return -1;
+ }
} else if (act_dtype == CUDA_DTYPE_F16) {
- if (linear_f16x(ctx, &l->down_proj, act_f16, ctx->x, total, inter, hidden, inter,
- use_resid16 ? 0.0f : 1.0f) != 0)
+ if (delta16) {
+ if (linear_16x_to_16(ctx, &l->down_proj, act_f16, CUDA_DTYPE_F16, ctx->resid_delta_16,
+ CUDA_DTYPE_F16, total, inter, hidden, inter) != 0)
+ return -1;
+ } else if (linear_f16x(ctx, &l->down_proj, act_f16, ctx->x, total, inter, hidden, inter,
+ use_resid16 ? 0.0f : 1.0f) != 0) {
return -1;
+ }
} else {
if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0)
return -1;
}
- if (bf16_resid) {
+ if (bf16_resid && delta16) {
+ layer_norm_bias_resid_bf16_delta16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
+ resid_bf16, resid_bf16, (const __nv_bfloat16 *)ctx->resid_delta_16, l->ffn_out_bias,
+ l->post_attn_norm, l->ffn_ln_bias, total, hidden, eps);
+ } else if (bf16_resid) {
layer_norm_bias_resid_bf16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
resid_bf16, resid_bf16, ctx->x, l->ffn_out_bias, l->post_attn_norm, l->ffn_ln_bias,
total, hidden, eps);
+ } else if (f16_resid && delta16) {
+ layer_norm_bias_resid_f16_delta16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
+ resid_f16, resid_f16, (const __half *)ctx->resid_delta_16, l->ffn_out_bias,
+ l->post_attn_norm, l->ffn_ln_bias, total, hidden, eps);
} else if (f16_resid) {
layer_norm_bias_resid_f16_kernel<<<total, 256, ln_smem, ctx->stream>>>(
resid_f16, resid_f16, ctx->x, l->ffn_out_bias, l->post_attn_norm, l->ffn_ln_bias,