diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-24 00:57:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:21 +0200 |
| commit | 4f0dd572471121135213e939b675fdc1cac32786 (patch) | |
| tree | ed988c22638c91790a0a9575974f3ab4c32bfd96 | |
| parent | c33fe775e97c19a94a892bdfaf52314f26ada6c2 (diff) | |
| download | ffwd-4f0dd572471121135213e939b675fdc1cac32786.tar.gz ffwd-4f0dd572471121135213e939b675fdc1cac32786.zip | |
kernels,cpu: fuse small BF16 QKV projections
| -rw-r--r-- | pplx_embed.c | 27 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 55 | ||||
| -rw-r--r-- | qwen_asr_kernels.h | 5 |
3 files changed, 67 insertions, 20 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index 7c168b7..5ee7051 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -389,6 +389,27 @@ static void linear_nobias_weight(float *y, const float *x, } } +static void linear_qkv_weight(float *q, float *k, float *v, const float *x, + const pplx_weight_ref_t *wq, + const pplx_weight_ref_t *wk, + const pplx_weight_ref_t *wv, + int seq_len, int in_dim, int q_dim, int kv_dim) +{ + if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && + wv->dtype == DTYPE_BF16 && seq_len <= 16) { + qwen_linear_nobias_bf16_qkv(q, k, v, x, + (const uint16_t *)wq->data, + (const uint16_t *)wk->data, + (const uint16_t *)wv->data, + seq_len, in_dim, q_dim, kv_dim); + return; + } + + linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim); + linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim); + linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim); +} + /* ======================================================================== * Working buffer management * ======================================================================== */ @@ -704,9 +725,9 @@ static int forward_packed_inplace(const pplx_model_t *model, qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - linear_nobias_weight(q_buf, x_norm, &l->wq, total_seq, hidden, q_dim); - linear_nobias_weight(k_buf, x_norm, &l->wk, total_seq, hidden, kv_dim); - linear_nobias_weight(v_buf, x_norm, &l->wv, total_seq, hidden, kv_dim); + linear_qkv_weight(q_buf, k_buf, v_buf, x_norm, + &l->wq, &l->wk, &l->wv, + total_seq, hidden, q_dim, kv_dim); qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index c299bc6..a569302 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -422,17 +422,13 @@ typedef struct { int in_dim; int q_dim; int kv_dim; + int seq_len; int total_dim; } qkv_matvec_task_t; -static void qkv_matvec_worker(int tid, int n_threads, void *arg) { - qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; - int chunk = (t->total_dim + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->total_dim) end = t->total_dim; - if (start >= end) return; - +static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, + int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; int q_end = t->q_dim; int k_end = q_end + t->kv_dim; int v_end = k_end + t->kv_dim; @@ -441,7 +437,7 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { int s = start; int e = end < q_end ? end : q_end; if (s < e) { - bf16_matvec_fused(t->q + s, t->x, + bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } @@ -452,7 +448,7 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { int e_abs = end < k_end ? end : k_end; int e = e_abs - q_end; if (s < e) { - bf16_matvec_fused(t->k + s, t->x, + bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } @@ -463,22 +459,50 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { int e_abs = end < v_end ? end : v_end; int e = e_abs - k_end; if (s < e) { - bf16_matvec_fused(t->v + s, t->x, + bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } } +static void qkv_matvec_worker(int tid, int n_threads, void *arg) { + qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) end = total_work; + if (start >= end) return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + qkv_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, const uint16_t *Wq_bf16, const uint16_t *Wk_bf16, const uint16_t *Wv_bf16, - int in_dim, int q_dim, int kv_dim) { + int seq_len, int in_dim, int q_dim, + int kv_dim) { + if (seq_len <= 0) return; + if (tp.n_threads <= 1) { - bf16_matvec_fused(q, x, Wq_bf16, NULL, in_dim, q_dim); - bf16_matvec_fused(k, x, Wk_bf16, NULL, in_dim, kv_dim); - bf16_matvec_fused(v, x, Wv_bf16, NULL, in_dim, kv_dim); + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(q + (size_t)s * q_dim, x_row, + Wq_bf16, NULL, in_dim, q_dim); + bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, + Wk_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, + Wv_bf16, NULL, in_dim, kv_dim); + } return; } @@ -493,6 +517,7 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, .in_dim = in_dim, .q_dim = q_dim, .kv_dim = kv_dim, + .seq_len = seq_len, .total_dim = q_dim + 2 * kv_dim, }; parallel_for(qkv_matvec_worker, &task); diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index 7e0a0f0..91124e3 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -41,12 +41,13 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); -/* seq=1 decoder fast path: compute Q/K/V matvecs with one threaded dispatch */ +/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, const uint16_t *Wq_bf16, const uint16_t *Wk_bf16, const uint16_t *Wv_bf16, - int in_dim, int q_dim, int kv_dim); + int seq_len, int in_dim, int q_dim, + int kv_dim); void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, int M, int K, int N); |
