aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-24 00:57:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:21 +0200
commit4f0dd572471121135213e939b675fdc1cac32786 (patch)
treeed988c22638c91790a0a9575974f3ab4c32bfd96
parentc33fe775e97c19a94a892bdfaf52314f26ada6c2 (diff)
downloadffwd-4f0dd572471121135213e939b675fdc1cac32786.tar.gz
ffwd-4f0dd572471121135213e939b675fdc1cac32786.zip
kernels,cpu: fuse small BF16 QKV projections
-rw-r--r--pplx_embed.c27
-rw-r--r--qwen_asr_kernels.c55
-rw-r--r--qwen_asr_kernels.h5
3 files changed, 67 insertions, 20 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index 7c168b7..5ee7051 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -389,6 +389,27 @@ static void linear_nobias_weight(float *y, const float *x,
}
}
+static void linear_qkv_weight(float *q, float *k, float *v, const float *x,
+ const pplx_weight_ref_t *wq,
+ const pplx_weight_ref_t *wk,
+ const pplx_weight_ref_t *wv,
+ int seq_len, int in_dim, int q_dim, int kv_dim)
+{
+ if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 &&
+ wv->dtype == DTYPE_BF16 && seq_len <= 16) {
+ qwen_linear_nobias_bf16_qkv(q, k, v, x,
+ (const uint16_t *)wq->data,
+ (const uint16_t *)wk->data,
+ (const uint16_t *)wv->data,
+ seq_len, in_dim, q_dim, kv_dim);
+ return;
+ }
+
+ linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim);
+ linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim);
+ linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim);
+}
+
/* ========================================================================
* Working buffer management
* ======================================================================== */
@@ -704,9 +725,9 @@ static int forward_packed_inplace(const pplx_model_t *model,
qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- linear_nobias_weight(q_buf, x_norm, &l->wq, total_seq, hidden, q_dim);
- linear_nobias_weight(k_buf, x_norm, &l->wk, total_seq, hidden, kv_dim);
- linear_nobias_weight(v_buf, x_norm, &l->wv, total_seq, hidden, kv_dim);
+ linear_qkv_weight(q_buf, k_buf, v_buf, x_norm,
+ &l->wq, &l->wk, &l->wv,
+ total_seq, hidden, q_dim, kv_dim);
qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index c299bc6..a569302 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -422,17 +422,13 @@ typedef struct {
int in_dim;
int q_dim;
int kv_dim;
+ int seq_len;
int total_dim;
} qkv_matvec_task_t;
-static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
- qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg;
- int chunk = (t->total_dim + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > t->total_dim) end = t->total_dim;
- if (start >= end) return;
-
+static void qkv_matvec_range(const qkv_matvec_task_t *t, int row,
+ int start, int end) {
+ const float *x_row = t->x + (size_t)row * t->in_dim;
int q_end = t->q_dim;
int k_end = q_end + t->kv_dim;
int v_end = k_end + t->kv_dim;
@@ -441,7 +437,7 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
int s = start;
int e = end < q_end ? end : q_end;
if (s < e) {
- bf16_matvec_fused(t->q + s, t->x,
+ bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row,
t->Wq_bf16 + (size_t)s * t->in_dim,
NULL, t->in_dim, e - s);
}
@@ -452,7 +448,7 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
int e_abs = end < k_end ? end : k_end;
int e = e_abs - q_end;
if (s < e) {
- bf16_matvec_fused(t->k + s, t->x,
+ bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row,
t->Wk_bf16 + (size_t)s * t->in_dim,
NULL, t->in_dim, e - s);
}
@@ -463,22 +459,50 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
int e_abs = end < v_end ? end : v_end;
int e = e_abs - k_end;
if (s < e) {
- bf16_matvec_fused(t->v + s, t->x,
+ bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row,
t->Wv_bf16 + (size_t)s * t->in_dim,
NULL, t->in_dim, e - s);
}
}
}
+static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
+ qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg;
+ int total_work = t->seq_len * t->total_dim;
+ int chunk = (total_work + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > total_work) end = total_work;
+ if (start >= end) return;
+
+ while (start < end) {
+ int row = start / t->total_dim;
+ int pos = start - row * t->total_dim;
+ int row_end = (row + 1) * t->total_dim;
+ int stop = end < row_end ? end : row_end;
+ qkv_matvec_range(t, row, pos, stop - row * t->total_dim);
+ start = stop;
+ }
+}
+
void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
const uint16_t *Wq_bf16,
const uint16_t *Wk_bf16,
const uint16_t *Wv_bf16,
- int in_dim, int q_dim, int kv_dim) {
+ int seq_len, int in_dim, int q_dim,
+ int kv_dim) {
+ if (seq_len <= 0) return;
+
if (tp.n_threads <= 1) {
- bf16_matvec_fused(q, x, Wq_bf16, NULL, in_dim, q_dim);
- bf16_matvec_fused(k, x, Wk_bf16, NULL, in_dim, kv_dim);
- bf16_matvec_fused(v, x, Wv_bf16, NULL, in_dim, kv_dim);
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + (size_t)s * in_dim;
+ bf16_matvec_fused(q + (size_t)s * q_dim, x_row,
+ Wq_bf16, NULL, in_dim, q_dim);
+ bf16_matvec_fused(k + (size_t)s * kv_dim, x_row,
+ Wk_bf16, NULL, in_dim, kv_dim);
+ bf16_matvec_fused(v + (size_t)s * kv_dim, x_row,
+ Wv_bf16, NULL, in_dim, kv_dim);
+ }
return;
}
@@ -493,6 +517,7 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
.in_dim = in_dim,
.q_dim = q_dim,
.kv_dim = kv_dim,
+ .seq_len = seq_len,
.total_dim = q_dim + 2 * kv_dim,
};
parallel_for(qkv_matvec_worker, &task);
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index 7e0a0f0..91124e3 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -41,12 +41,13 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
int seq_len, int in_dim, int out_dim);
-/* seq=1 decoder fast path: compute Q/K/V matvecs with one threaded dispatch */
+/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
const uint16_t *Wq_bf16,
const uint16_t *Wk_bf16,
const uint16_t *Wv_bf16,
- int in_dim, int q_dim, int kv_dim);
+ int seq_len, int in_dim, int q_dim,
+ int kv_dim);
void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
int M, int K, int N);