From 9bfcc72ee6b2568d0f9fc05a28f9ccc3da176713 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sun, 24 May 2026 01:02:17 +0200 Subject: kernels,cpu: fuse tiny BF16 gate up projections --- pplx_embed.c | 29 +++++++++++++++-- qwen_asr_kernels.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ qwen_asr_kernels.h | 7 ++++ 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/pplx_embed.c b/pplx_embed.c index 5ee7051..ecc7fa0 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -65,6 +65,9 @@ struct pplx_workspace { int pplx_verbose = 0; int qwen_verbose = 0; +#define PPLX_BF16_QKV_FUSE_MAX_SEQ 16 +#define PPLX_BF16_PAIR_FUSE_MAX_SEQ 4 + /* ======================================================================== * Size helpers * ======================================================================== */ @@ -396,7 +399,8 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x, int seq_len, int in_dim, int q_dim, int kv_dim) { if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && - wv->dtype == DTYPE_BF16 && seq_len <= 16) { + wv->dtype == DTYPE_BF16 && + seq_len <= PPLX_BF16_QKV_FUSE_MAX_SEQ) { qwen_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, (const uint16_t *)wk->data, @@ -410,6 +414,24 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x, linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim); } +static void linear_pair_weight(float *a, float *b, const float *x, + const pplx_weight_ref_t *wa, + const pplx_weight_ref_t *wb, + int seq_len, int in_dim, int a_dim, int b_dim) +{ + if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && + seq_len <= PPLX_BF16_PAIR_FUSE_MAX_SEQ) { + qwen_linear_nobias_bf16_pair(a, b, x, + (const uint16_t *)wa->data, + (const uint16_t *)wb->data, + seq_len, in_dim, a_dim, b_dim); + return; + } + + linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim); + linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim); +} + /* ======================================================================== * Working buffer management * ======================================================================== */ @@ -750,8 +772,9 @@ static int forward_packed_inplace(const pplx_model_t *model, qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - linear_nobias_weight(ffn_gate, x_norm, &l->gate_proj, total_seq, hidden, inter); - linear_nobias_weight(ffn_up, x_norm, &l->up_proj, total_seq, hidden, inter); + linear_pair_weight(ffn_gate, ffn_up, x_norm, + &l->gate_proj, &l->up_proj, + total_seq, hidden, inter, inter); qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index a569302..56ec900 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -411,6 +411,99 @@ static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16, bf16_linear_rows_worker(0, 1, &task); } +typedef struct { + float *a; + float *b; + const float *x; + const uint16_t *Wa_bf16; + const uint16_t *Wb_bf16; + int seq_len; + int in_dim; + int a_dim; + int b_dim; + int total_dim; +} pair_matvec_task_t; + +static void pair_matvec_range(const pair_matvec_task_t *t, int row, + int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; + int a_end = t->a_dim; + int b_end = a_end + t->b_dim; + + if (start < a_end) { + int s = start; + int e = end < a_end ? end : a_end; + if (s < e) { + bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, + t->Wa_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > a_end && start < b_end) { + int s = start > a_end ? start - a_end : 0; + int e_abs = end < b_end ? end : b_end; + int e = e_abs - a_end; + if (s < e) { + bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, + t->Wb_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } +} + +static void pair_matvec_worker(int tid, int n_threads, void *arg) { + pair_matvec_task_t *t = (pair_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) end = total_work; + if (start >= end) return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + pair_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + +void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, int in_dim, int a_dim, + int b_dim) { + if (seq_len <= 0) return; + + if (tp.n_threads <= 1) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(a + (size_t)s * a_dim, x_row, + Wa_bf16, NULL, in_dim, a_dim); + bf16_matvec_fused(b + (size_t)s * b_dim, x_row, + Wb_bf16, NULL, in_dim, b_dim); + } + return; + } + + pair_matvec_task_t task = { + .a = a, + .b = b, + .x = x, + .Wa_bf16 = Wa_bf16, + .Wb_bf16 = Wb_bf16, + .seq_len = seq_len, + .in_dim = in_dim, + .a_dim = a_dim, + .b_dim = b_dim, + .total_dim = a_dim + b_dim, + }; + parallel_for(pair_matvec_worker, &task); +} + typedef struct { float *q; float *k; diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index 91124e3..92082f1 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -41,6 +41,13 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); +/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ +void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, int in_dim, int a_dim, + int b_dim); + /* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, const uint16_t *Wq_bf16, -- cgit v1.3.1