aboutsummaryrefslogtreecommitdiff
path: root/pplx_embed.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-02 09:08:54 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit8d412f8a225f3e337dac271ea2491ac3c1c099af (patch)
tree6e43dd291c46f5b440d109f1967d5423c5dc84c9 /pplx_embed.c
parent3f248b27a5482514f80deb687450dbd3a11d41b2 (diff)
downloadffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.tar.gz
ffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.zip
cpu: reuse tiled attention scratch buffers
Diffstat (limited to 'pplx_embed.c')
-rw-r--r--pplx_embed.c26
1 files changed, 23 insertions, 3 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index 4aaeda1..1acd06f 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -53,6 +53,8 @@ struct pplx_workspace {
float *proj_out; /* [seq, hidden] */
float *ffn_gate; /* [seq, intermediate] */
float *ffn_up; /* [seq, intermediate] */
+ float *attn_scores; /* reusable BLAS attention score tiles */
+ size_t attn_scores_bytes;
int *offsets; /* [batch + 1] packed sequence offsets */
int offsets_cap;
@@ -497,6 +499,19 @@ static int ensure_buffers(pplx_workspace_t *ws, const pplx_config_t *c, int seq)
return 0;
}
+static void ensure_attention_scores(pplx_workspace_t *ws, const int *offsets,
+ int batch)
+{
+ size_t bytes =
+ qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes) return;
+
+ void *p = realloc(ws->attn_scores, bytes);
+ if (!p) return;
+ ws->attn_scores = (float *)p;
+ ws->attn_scores_bytes = bytes;
+}
+
/* ========================================================================
* RoPE cache
* ======================================================================== */
@@ -722,6 +737,7 @@ void pplx_workspace_free(pplx_workspace_t *ws)
free(ws->q); free(ws->k); free(ws->v);
free(ws->attn_out); free(ws->proj_out);
free(ws->ffn_gate); free(ws->ffn_up);
+ free(ws->attn_scores);
free(ws->offsets);
free(ws->rope_cos); free(ws->rope_sin);
free(ws);
@@ -786,6 +802,8 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws)
return SIZE_MAX;
if (add_size(total, offset_bytes, &total) != 0)
return SIZE_MAX;
+ if (add_size(total, ws->attn_scores_bytes, &total) != 0)
+ return SIZE_MAX;
return total;
}
@@ -853,6 +871,7 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
if (ensure_buffers(ws, cfg, total_seq) != 0) return -1;
if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1;
+ ensure_attention_scores(ws, offsets, batch);
const float *rope_cos = ws->rope_cos;
const float *rope_sin = ws->rope_sin;
@@ -912,9 +931,10 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
rope_cos, rope_sin, len, n_kv_heads, head_dim);
}
- qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf,
- offsets, batch, n_heads,
- n_kv_heads, head_dim, scale);
+ qwen_bidirectional_gqa_attention_packed_with_scratch(
+ attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads,
+ n_kv_heads, head_dim, scale, ws->attn_scores,
+ ws->attn_scores_bytes);
linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);