diff options
| -rw-r--r-- | embed.c | 67 | ||||
| -rw-r--r-- | embed_internal.h | 6 | ||||
| -rw-r--r-- | qwen_kernels.c | 135 | ||||
| -rw-r--r-- | qwen_kernels.h | 5 |
4 files changed, 61 insertions, 152 deletions
@@ -405,20 +405,42 @@ static void copy_weight_row(float *dst, const pplx_weight_ref_t *w, } } -static void linear_nobias_weight(float *y, const float *x, - const pplx_weight_ref_t *w, +static int ensure_bf16_widen(pplx_workspace_t *ws, size_t count) +{ + if (ws->bf16_widen_count >= count) return 0; + float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float)); + if (!p) return -1; + ws->bf16_widen = p; + ws->bf16_widen_count = count; + return 0; +} + +static void linear_nobias_weight(pplx_workspace_t *ws, float *y, + const float *x, const pplx_weight_ref_t *w, int seq_len, int in_dim, int out_dim) { if (w->dtype == DTYPE_F32) { qwen_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); - } else { - qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, - seq_len, in_dim, out_dim); + return; } + if (seq_len > 16) { + /* Widen the weight matrix once into workspace scratch and use the + * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ + size_t count = (size_t)out_dim * (size_t)in_dim; + if (ensure_bf16_widen(ws, count) == 0) { + qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, + count); + qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); + return; + } + } + qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, + seq_len, in_dim, out_dim); } -static void linear_qkv_weight(float *q, float *k, float *v, const float *x, +static void linear_qkv_weight(pplx_workspace_t *ws, float *q, float *k, + float *v, const float *x, const pplx_weight_ref_t *wq, const pplx_weight_ref_t *wk, const pplx_weight_ref_t *wv, @@ -435,12 +457,13 @@ static void linear_qkv_weight(float *q, float *k, float *v, const float *x, return; } - linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim); - linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim); - linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim); + linear_nobias_weight(ws, q, x, wq, seq_len, in_dim, q_dim); + linear_nobias_weight(ws, k, x, wk, seq_len, in_dim, kv_dim); + linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim); } -static void linear_pair_weight(float *a, float *b, const float *x, +static void linear_pair_weight(pplx_workspace_t *ws, float *a, float *b, + const float *x, const pplx_weight_ref_t *wa, const pplx_weight_ref_t *wb, int seq_len, int in_dim, int a_dim, int b_dim) @@ -454,8 +477,8 @@ static void linear_pair_weight(float *a, float *b, const float *x, return; } - linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim); - linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim); + linear_nobias_weight(ws, a, x, wa, seq_len, in_dim, a_dim); + linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim); } /* ======================================================================== @@ -738,6 +761,7 @@ void pplx_workspace_free(pplx_workspace_t *ws) free(ws->attn_scores); free(ws->offsets); free(ws->rope_cos); free(ws->rope_sin); + free(ws->bf16_widen); free(ws); } @@ -867,8 +891,8 @@ int pplx_late_model_encode_tokens(const pplx_late_model_t *model, n_tokens, ws->states) != 0) return -1; - linear_nobias_weight(out_vectors, ws->states, &model->projection, - n_tokens, hidden, dim); + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, + &model->projection, n_tokens, hidden, dim); if (normalize) { for (int i = 0; i < n_tokens; i++) { if (pplx_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) @@ -978,6 +1002,13 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws) if (add_size(total, ws->attn_scores_bytes, &total) != 0) return SIZE_MAX; + size_t widen_bytes = 0; + if (ws->bf16_widen_count > 0 && + mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) + return SIZE_MAX; + if (add_size(total, widen_bytes, &total) != 0) + return SIZE_MAX; + return total; } @@ -1088,7 +1119,7 @@ static int forward_packed_slice_inplace(const pplx_model_t *model, qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - linear_qkv_weight(q_buf, k_buf, v_buf, x_norm, + linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden, q_dim, kv_dim); @@ -1109,16 +1140,16 @@ static int forward_packed_slice_inplace(const pplx_model_t *model, n_kv_heads, head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); - linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); + linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - linear_pair_weight(ffn_gate, ffn_up, x_norm, + linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, hidden, inter, inter); qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); - linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); + linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); } diff --git a/embed_internal.h b/embed_internal.h index 7d6f398..53debee 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -59,6 +59,12 @@ struct pplx_workspace { float *rope_cos; float *rope_sin; int rope_cache_cap; + + /* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the + * largest projection. Workspace-owned so concurrent workspaces on one + * model never share conversion scratch. */ + float *bf16_widen; + size_t bf16_widen_count; }; #endif /* PPLX_EMBED_INTERNAL_H */ diff --git a/qwen_kernels.c b/qwen_kernels.c index 5cf38cd..bfa0691 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -261,7 +261,7 @@ void qwen_linear_nobias(float *y, const float *x, const float *W, } /* Convert bf16 buffer to f32 buffer */ -static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { +void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { #if defined(__ARM_NEON) || defined(__ARM_NEON__) size_t i = 0; uint32_t *d = (uint32_t *)(void *)dst; @@ -281,100 +281,6 @@ static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { #endif } -/* Reusable scratch buffer for bf16->f32 conversion */ -static float *bf16_scratch = NULL; -static size_t bf16_scratch_cap = 0; - -static float *bf16_get_scratch(size_t n) { - if (n > bf16_scratch_cap) { - free(bf16_scratch); - bf16_scratch = (float *)malloc(n * sizeof(float)); - bf16_scratch_cap = bf16_scratch ? n : 0; - } - return bf16_scratch; -} - -typedef struct { - const uint16_t *src; - size_t n; - float *dst_f32; -} bf16_cache_entry_t; - -static bf16_cache_entry_t *bf16_cache = NULL; -static int bf16_cache_len = 0; -static int bf16_cache_cap = 0; -static size_t bf16_cache_bytes = 0; -static size_t bf16_cache_limit_bytes = 0; -static int bf16_cache_limit_init = 0; - -static void bf16_cache_init_limit(void) { - if (bf16_cache_limit_init) return; - bf16_cache_limit_init = 1; - - /* Default OFF. Override with QWEN_BF16_CACHE_MB=<n> to enable. */ - unsigned long long mb = 0; - const char *env = getenv("QWEN_BF16_CACHE_MB"); - if (env && env[0] != '\0') { - char *end = NULL; - unsigned long long v = strtoull(env, &end, 10); - if (end != env) mb = v; - } - bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL); - - if (qwen_verbose >= 2) { - fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb); - } -} - -static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) { - bf16_cache_init_limit(); - - for (int i = 0; i < bf16_cache_len; i++) { - if (bf16_cache[i].src == src && bf16_cache[i].n == n) { - return bf16_cache[i].dst_f32; - } - } - - if (bf16_cache_limit_bytes == 0) return NULL; - - size_t bytes = n * sizeof(float); - if (bytes > bf16_cache_limit_bytes) return NULL; - if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL; - - float *dst = (float *)malloc(bytes); - if (!dst) return NULL; - bf16_to_f32_buf(dst, src, n); - - if (bf16_cache_len == bf16_cache_cap) { - int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256; - bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc( - bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t)); - if (!tmp) { - free(dst); - return NULL; - } - bf16_cache = tmp; - bf16_cache_cap = new_cap; - } - - bf16_cache[bf16_cache_len].src = src; - bf16_cache[bf16_cache_len].n = n; - bf16_cache[bf16_cache_len].dst_f32 = dst; - bf16_cache_len++; - bf16_cache_bytes += bytes; - return dst; -} - -static const float *bf16_get_f32_view(const uint16_t *src, size_t n) { - const float *cached = bf16_get_cached_f32(src, n); - if (cached) return cached; - - float *scratch = bf16_get_scratch(n); - if (!scratch) return NULL; - bf16_to_f32_buf(scratch, src, n); - return scratch; -} - /* * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias * Processes 2 output rows at a time to amortize x vector loads. @@ -673,30 +579,10 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; } - if (seq_len <= 16) { - bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); - return; - } - size_t n = (size_t)out_dim * in_dim; - const float *W_f32 = bf16_get_f32_view(W_bf16, n); - if (!W_f32) return; - qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim); -} - -void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, - const float *b, int seq_len, int in_dim, int out_dim) { - if (seq_len == 1) { - bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim); - return; - } - if (seq_len <= 16) { - bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim); - return; - } - size_t n = (size_t)out_dim * in_dim; - const float *W_f32 = bf16_get_f32_view(W_bf16, n); - if (!W_f32) return; - qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim); + /* Callers route longer sequences through an F32-widened weight matrix in + * caller-owned scratch (see embed.c); this per-row path stays correct for + * any length. */ + bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } /* Find argmax over a range of output rows [start, end). @@ -758,17 +644,6 @@ int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, return best; } -void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, - int M, int K, int N) { - if (M == 1) { - bf16_matvec_threaded(C, A, B_bf16, NULL, K, N); - } else { - size_t n = (size_t)N * K; - const float *B_f32 = bf16_get_f32_view(B_bf16, n); - if (!B_f32) return; - qwen_matmul_t(C, A, B_f32, M, K, N); - } -} /* ======================================================================== * 2D Convolution (im2col + BLAS sgemm) diff --git a/qwen_kernels.h b/qwen_kernels.h index e92f944..5281361 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -35,9 +35,8 @@ void qwen_linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); /* bf16 weight variants */ -void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, - const float *b, int seq_len, int in_dim, int out_dim); +void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); @@ -56,8 +55,6 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, int seq_len, int in_dim, int q_dim, int kv_dim); -void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, - int M, int K, int N); /* Dot product using the best available local SIMD implementation. */ float qwen_dot_f32(const float *a, const float *b, int n); |
