aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-24 01:02:17 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:50 +0200
commit9bfcc72ee6b2568d0f9fc05a28f9ccc3da176713 (patch)
tree724a5c293a7c6bf7fe028936c38e85e9919534b9
parent4f0dd572471121135213e939b675fdc1cac32786 (diff)
downloadffwd-9bfcc72ee6b2568d0f9fc05a28f9ccc3da176713.tar.gz
ffwd-9bfcc72ee6b2568d0f9fc05a28f9ccc3da176713.zip
kernels,cpu: fuse tiny BF16 gate up projections
-rw-r--r--pplx_embed.c29
-rw-r--r--qwen_asr_kernels.c93
-rw-r--r--qwen_asr_kernels.h7
3 files changed, 126 insertions, 3 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index 5ee7051..ecc7fa0 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -65,6 +65,9 @@ struct pplx_workspace {
int pplx_verbose = 0;
int qwen_verbose = 0;
+#define PPLX_BF16_QKV_FUSE_MAX_SEQ 16
+#define PPLX_BF16_PAIR_FUSE_MAX_SEQ 4
+
/* ========================================================================
* Size helpers
* ======================================================================== */
@@ -396,7 +399,8 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x,
int seq_len, int in_dim, int q_dim, int kv_dim)
{
if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 &&
- wv->dtype == DTYPE_BF16 && seq_len <= 16) {
+ wv->dtype == DTYPE_BF16 &&
+ seq_len <= PPLX_BF16_QKV_FUSE_MAX_SEQ) {
qwen_linear_nobias_bf16_qkv(q, k, v, x,
(const uint16_t *)wq->data,
(const uint16_t *)wk->data,
@@ -410,6 +414,24 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x,
linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim);
}
+static void linear_pair_weight(float *a, float *b, const float *x,
+ const pplx_weight_ref_t *wa,
+ const pplx_weight_ref_t *wb,
+ int seq_len, int in_dim, int a_dim, int b_dim)
+{
+ if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
+ seq_len <= PPLX_BF16_PAIR_FUSE_MAX_SEQ) {
+ qwen_linear_nobias_bf16_pair(a, b, x,
+ (const uint16_t *)wa->data,
+ (const uint16_t *)wb->data,
+ seq_len, in_dim, a_dim, b_dim);
+ return;
+ }
+
+ linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim);
+ linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim);
+}
+
/* ========================================================================
* Working buffer management
* ======================================================================== */
@@ -750,8 +772,9 @@ static int forward_packed_inplace(const pplx_model_t *model,
qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- linear_nobias_weight(ffn_gate, x_norm, &l->gate_proj, total_seq, hidden, inter);
- linear_nobias_weight(ffn_up, x_norm, &l->up_proj, total_seq, hidden, inter);
+ linear_pair_weight(ffn_gate, ffn_up, x_norm,
+ &l->gate_proj, &l->up_proj,
+ total_seq, hidden, inter, inter);
qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index a569302..56ec900 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -412,6 +412,99 @@ static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16,
}
typedef struct {
+ float *a;
+ float *b;
+ const float *x;
+ const uint16_t *Wa_bf16;
+ const uint16_t *Wb_bf16;
+ int seq_len;
+ int in_dim;
+ int a_dim;
+ int b_dim;
+ int total_dim;
+} pair_matvec_task_t;
+
+static void pair_matvec_range(const pair_matvec_task_t *t, int row,
+ int start, int end) {
+ const float *x_row = t->x + (size_t)row * t->in_dim;
+ int a_end = t->a_dim;
+ int b_end = a_end + t->b_dim;
+
+ if (start < a_end) {
+ int s = start;
+ int e = end < a_end ? end : a_end;
+ if (s < e) {
+ bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row,
+ t->Wa_bf16 + (size_t)s * t->in_dim,
+ NULL, t->in_dim, e - s);
+ }
+ }
+
+ if (end > a_end && start < b_end) {
+ int s = start > a_end ? start - a_end : 0;
+ int e_abs = end < b_end ? end : b_end;
+ int e = e_abs - a_end;
+ if (s < e) {
+ bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row,
+ t->Wb_bf16 + (size_t)s * t->in_dim,
+ NULL, t->in_dim, e - s);
+ }
+ }
+}
+
+static void pair_matvec_worker(int tid, int n_threads, void *arg) {
+ pair_matvec_task_t *t = (pair_matvec_task_t *)arg;
+ int total_work = t->seq_len * t->total_dim;
+ int chunk = (total_work + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > total_work) end = total_work;
+ if (start >= end) return;
+
+ while (start < end) {
+ int row = start / t->total_dim;
+ int pos = start - row * t->total_dim;
+ int row_end = (row + 1) * t->total_dim;
+ int stop = end < row_end ? end : row_end;
+ pair_matvec_range(t, row, pos, stop - row * t->total_dim);
+ start = stop;
+ }
+}
+
+void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x,
+ const uint16_t *Wa_bf16,
+ const uint16_t *Wb_bf16,
+ int seq_len, int in_dim, int a_dim,
+ int b_dim) {
+ if (seq_len <= 0) return;
+
+ if (tp.n_threads <= 1) {
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + (size_t)s * in_dim;
+ bf16_matvec_fused(a + (size_t)s * a_dim, x_row,
+ Wa_bf16, NULL, in_dim, a_dim);
+ bf16_matvec_fused(b + (size_t)s * b_dim, x_row,
+ Wb_bf16, NULL, in_dim, b_dim);
+ }
+ return;
+ }
+
+ pair_matvec_task_t task = {
+ .a = a,
+ .b = b,
+ .x = x,
+ .Wa_bf16 = Wa_bf16,
+ .Wb_bf16 = Wb_bf16,
+ .seq_len = seq_len,
+ .in_dim = in_dim,
+ .a_dim = a_dim,
+ .b_dim = b_dim,
+ .total_dim = a_dim + b_dim,
+ };
+ parallel_for(pair_matvec_worker, &task);
+}
+
+typedef struct {
float *q;
float *k;
float *v;
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index 91124e3..92082f1 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -41,6 +41,13 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
int seq_len, int in_dim, int out_dim);
+/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
+void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x,
+ const uint16_t *Wa_bf16,
+ const uint16_t *Wb_bf16,
+ int seq_len, int in_dim, int a_dim,
+ int b_dim);
+
/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
const uint16_t *Wq_bf16,