aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 09:40:35 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitc709fc62ebe0eb2ca596316e5dd22f212766ff19 (patch)
tree125f809084628c4dc7dc3df249ff7fd82861764d
parent308bb1b6a058855bb1a2cc45db13f83810cf91b6 (diff)
downloadffwd-c709fc62ebe0eb2ca596316e5dd22f212766ff19.tar.gz
ffwd-c709fc62ebe0eb2ca596316e5dd22f212766ff19.zip
cpu: move BF16 widening scratch into the workspace
-rw-r--r--embed.c67
-rw-r--r--embed_internal.h6
-rw-r--r--qwen_kernels.c135
-rw-r--r--qwen_kernels.h5
4 files changed, 61 insertions, 152 deletions
diff --git a/embed.c b/embed.c
index 59ccc10..9d5e0cc 100644
--- a/embed.c
+++ b/embed.c
@@ -405,20 +405,42 @@ static void copy_weight_row(float *dst, const pplx_weight_ref_t *w,
}
}
-static void linear_nobias_weight(float *y, const float *x,
- const pplx_weight_ref_t *w,
+static int ensure_bf16_widen(pplx_workspace_t *ws, size_t count)
+{
+ if (ws->bf16_widen_count >= count) return 0;
+ float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float));
+ if (!p) return -1;
+ ws->bf16_widen = p;
+ ws->bf16_widen_count = count;
+ return 0;
+}
+
+static void linear_nobias_weight(pplx_workspace_t *ws, float *y,
+ const float *x, const pplx_weight_ref_t *w,
int seq_len, int in_dim, int out_dim)
{
if (w->dtype == DTYPE_F32) {
qwen_linear_nobias(y, x, (const float *)w->data,
seq_len, in_dim, out_dim);
- } else {
- qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data,
- seq_len, in_dim, out_dim);
+ return;
}
+ if (seq_len > 16) {
+ /* Widen the weight matrix once into workspace scratch and use the
+ * BLAS-backed F32 path; below that the fused BF16 row kernels win. */
+ size_t count = (size_t)out_dim * (size_t)in_dim;
+ if (ensure_bf16_widen(ws, count) == 0) {
+ qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data,
+ count);
+ qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
+ return;
+ }
+ }
+ qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data,
+ seq_len, in_dim, out_dim);
}
-static void linear_qkv_weight(float *q, float *k, float *v, const float *x,
+static void linear_qkv_weight(pplx_workspace_t *ws, float *q, float *k,
+ float *v, const float *x,
const pplx_weight_ref_t *wq,
const pplx_weight_ref_t *wk,
const pplx_weight_ref_t *wv,
@@ -435,12 +457,13 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x,
return;
}
- linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim);
- linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim);
- linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim);
+ linear_nobias_weight(ws, q, x, wq, seq_len, in_dim, q_dim);
+ linear_nobias_weight(ws, k, x, wk, seq_len, in_dim, kv_dim);
+ linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim);
}
-static void linear_pair_weight(float *a, float *b, const float *x,
+static void linear_pair_weight(pplx_workspace_t *ws, float *a, float *b,
+ const float *x,
const pplx_weight_ref_t *wa,
const pplx_weight_ref_t *wb,
int seq_len, int in_dim, int a_dim, int b_dim)
@@ -454,8 +477,8 @@ static void linear_pair_weight(float *a, float *b, const float *x,
return;
}
- linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim);
- linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim);
+ linear_nobias_weight(ws, a, x, wa, seq_len, in_dim, a_dim);
+ linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim);
}
/* ========================================================================
@@ -738,6 +761,7 @@ void pplx_workspace_free(pplx_workspace_t *ws)
free(ws->attn_scores);
free(ws->offsets);
free(ws->rope_cos); free(ws->rope_sin);
+ free(ws->bf16_widen);
free(ws);
}
@@ -867,8 +891,8 @@ int pplx_late_model_encode_tokens(const pplx_late_model_t *model,
n_tokens, ws->states) != 0)
return -1;
- linear_nobias_weight(out_vectors, ws->states, &model->projection,
- n_tokens, hidden, dim);
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states,
+ &model->projection, n_tokens, hidden, dim);
if (normalize) {
for (int i = 0; i < n_tokens; i++) {
if (pplx_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
@@ -978,6 +1002,13 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws)
if (add_size(total, ws->attn_scores_bytes, &total) != 0)
return SIZE_MAX;
+ size_t widen_bytes = 0;
+ if (ws->bf16_widen_count > 0 &&
+ mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0)
+ return SIZE_MAX;
+ if (add_size(total, widen_bytes, &total) != 0)
+ return SIZE_MAX;
+
return total;
}
@@ -1088,7 +1119,7 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- linear_qkv_weight(q_buf, k_buf, v_buf, x_norm,
+ linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm,
&l->wq, &l->wk, &l->wv,
total_seq, hidden, q_dim, kv_dim);
@@ -1109,16 +1140,16 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
n_kv_heads, head_dim, scale, ws->attn_scores,
ws->attn_scores_bytes);
- linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
+ linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- linear_pair_weight(ffn_gate, ffn_up, x_norm,
+ linear_pair_weight(ws, ffn_gate, ffn_up, x_norm,
&l->gate_proj, &l->up_proj,
total_seq, hidden, inter, inter);
qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
- linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
+ linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
}
diff --git a/embed_internal.h b/embed_internal.h
index 7d6f398..53debee 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -59,6 +59,12 @@ struct pplx_workspace {
float *rope_cos;
float *rope_sin;
int rope_cache_cap;
+
+ /* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
+ * largest projection. Workspace-owned so concurrent workspaces on one
+ * model never share conversion scratch. */
+ float *bf16_widen;
+ size_t bf16_widen_count;
};
#endif /* PPLX_EMBED_INTERNAL_H */
diff --git a/qwen_kernels.c b/qwen_kernels.c
index 5cf38cd..bfa0691 100644
--- a/qwen_kernels.c
+++ b/qwen_kernels.c
@@ -261,7 +261,7 @@ void qwen_linear_nobias(float *y, const float *x, const float *W,
}
/* Convert bf16 buffer to f32 buffer */
-static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
+void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
size_t i = 0;
uint32_t *d = (uint32_t *)(void *)dst;
@@ -281,100 +281,6 @@ static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
#endif
}
-/* Reusable scratch buffer for bf16->f32 conversion */
-static float *bf16_scratch = NULL;
-static size_t bf16_scratch_cap = 0;
-
-static float *bf16_get_scratch(size_t n) {
- if (n > bf16_scratch_cap) {
- free(bf16_scratch);
- bf16_scratch = (float *)malloc(n * sizeof(float));
- bf16_scratch_cap = bf16_scratch ? n : 0;
- }
- return bf16_scratch;
-}
-
-typedef struct {
- const uint16_t *src;
- size_t n;
- float *dst_f32;
-} bf16_cache_entry_t;
-
-static bf16_cache_entry_t *bf16_cache = NULL;
-static int bf16_cache_len = 0;
-static int bf16_cache_cap = 0;
-static size_t bf16_cache_bytes = 0;
-static size_t bf16_cache_limit_bytes = 0;
-static int bf16_cache_limit_init = 0;
-
-static void bf16_cache_init_limit(void) {
- if (bf16_cache_limit_init) return;
- bf16_cache_limit_init = 1;
-
- /* Default OFF. Override with QWEN_BF16_CACHE_MB=<n> to enable. */
- unsigned long long mb = 0;
- const char *env = getenv("QWEN_BF16_CACHE_MB");
- if (env && env[0] != '\0') {
- char *end = NULL;
- unsigned long long v = strtoull(env, &end, 10);
- if (end != env) mb = v;
- }
- bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL);
-
- if (qwen_verbose >= 2) {
- fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb);
- }
-}
-
-static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) {
- bf16_cache_init_limit();
-
- for (int i = 0; i < bf16_cache_len; i++) {
- if (bf16_cache[i].src == src && bf16_cache[i].n == n) {
- return bf16_cache[i].dst_f32;
- }
- }
-
- if (bf16_cache_limit_bytes == 0) return NULL;
-
- size_t bytes = n * sizeof(float);
- if (bytes > bf16_cache_limit_bytes) return NULL;
- if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL;
-
- float *dst = (float *)malloc(bytes);
- if (!dst) return NULL;
- bf16_to_f32_buf(dst, src, n);
-
- if (bf16_cache_len == bf16_cache_cap) {
- int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256;
- bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc(
- bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t));
- if (!tmp) {
- free(dst);
- return NULL;
- }
- bf16_cache = tmp;
- bf16_cache_cap = new_cap;
- }
-
- bf16_cache[bf16_cache_len].src = src;
- bf16_cache[bf16_cache_len].n = n;
- bf16_cache[bf16_cache_len].dst_f32 = dst;
- bf16_cache_len++;
- bf16_cache_bytes += bytes;
- return dst;
-}
-
-static const float *bf16_get_f32_view(const uint16_t *src, size_t n) {
- const float *cached = bf16_get_cached_f32(src, n);
- if (cached) return cached;
-
- float *scratch = bf16_get_scratch(n);
- if (!scratch) return NULL;
- bf16_to_f32_buf(scratch, src, n);
- return scratch;
-}
-
/*
* Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias
* Processes 2 output rows at a time to amortize x vector loads.
@@ -673,30 +579,10 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
return;
}
- if (seq_len <= 16) {
- bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
- return;
- }
- size_t n = (size_t)out_dim * in_dim;
- const float *W_f32 = bf16_get_f32_view(W_bf16, n);
- if (!W_f32) return;
- qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim);
-}
-
-void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
- const float *b, int seq_len, int in_dim, int out_dim) {
- if (seq_len == 1) {
- bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim);
- return;
- }
- if (seq_len <= 16) {
- bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim);
- return;
- }
- size_t n = (size_t)out_dim * in_dim;
- const float *W_f32 = bf16_get_f32_view(W_bf16, n);
- if (!W_f32) return;
- qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim);
+ /* Callers route longer sequences through an F32-widened weight matrix in
+ * caller-owned scratch (see embed.c); this per-row path stays correct for
+ * any length. */
+ bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
}
/* Find argmax over a range of output rows [start, end).
@@ -758,17 +644,6 @@ int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16,
return best;
}
-void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
- int M, int K, int N) {
- if (M == 1) {
- bf16_matvec_threaded(C, A, B_bf16, NULL, K, N);
- } else {
- size_t n = (size_t)N * K;
- const float *B_f32 = bf16_get_f32_view(B_bf16, n);
- if (!B_f32) return;
- qwen_matmul_t(C, A, B_f32, M, K, N);
- }
-}
/* ========================================================================
* 2D Convolution (im2col + BLAS sgemm)
diff --git a/qwen_kernels.h b/qwen_kernels.h
index e92f944..5281361 100644
--- a/qwen_kernels.h
+++ b/qwen_kernels.h
@@ -35,9 +35,8 @@ void qwen_linear_nobias(float *y, const float *x, const float *W,
int seq_len, int in_dim, int out_dim);
/* bf16 weight variants */
-void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
- const float *b, int seq_len, int in_dim, int out_dim);
+void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n);
void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
int seq_len, int in_dim, int out_dim);
@@ -56,8 +55,6 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
int seq_len, int in_dim, int q_dim,
int kv_dim);
-void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
- int M, int K, int N);
/* Dot product using the best available local SIMD implementation. */
float qwen_dot_f32(const float *a, const float *b, int n);