aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pplx_embed.c26
-rw-r--r--qwen_asr_kernels.c58
-rw-r--r--qwen_asr_kernels.h8
3 files changed, 77 insertions, 15 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index 4aaeda1..1acd06f 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -53,6 +53,8 @@ struct pplx_workspace {
float *proj_out; /* [seq, hidden] */
float *ffn_gate; /* [seq, intermediate] */
float *ffn_up; /* [seq, intermediate] */
+ float *attn_scores; /* reusable BLAS attention score tiles */
+ size_t attn_scores_bytes;
int *offsets; /* [batch + 1] packed sequence offsets */
int offsets_cap;
@@ -497,6 +499,19 @@ static int ensure_buffers(pplx_workspace_t *ws, const pplx_config_t *c, int seq)
return 0;
}
+static void ensure_attention_scores(pplx_workspace_t *ws, const int *offsets,
+ int batch)
+{
+ size_t bytes =
+ qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes) return;
+
+ void *p = realloc(ws->attn_scores, bytes);
+ if (!p) return;
+ ws->attn_scores = (float *)p;
+ ws->attn_scores_bytes = bytes;
+}
+
/* ========================================================================
* RoPE cache
* ======================================================================== */
@@ -722,6 +737,7 @@ void pplx_workspace_free(pplx_workspace_t *ws)
free(ws->q); free(ws->k); free(ws->v);
free(ws->attn_out); free(ws->proj_out);
free(ws->ffn_gate); free(ws->ffn_up);
+ free(ws->attn_scores);
free(ws->offsets);
free(ws->rope_cos); free(ws->rope_sin);
free(ws);
@@ -786,6 +802,8 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws)
return SIZE_MAX;
if (add_size(total, offset_bytes, &total) != 0)
return SIZE_MAX;
+ if (add_size(total, ws->attn_scores_bytes, &total) != 0)
+ return SIZE_MAX;
return total;
}
@@ -853,6 +871,7 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
if (ensure_buffers(ws, cfg, total_seq) != 0) return -1;
if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1;
+ ensure_attention_scores(ws, offsets, batch);
const float *rope_cos = ws->rope_cos;
const float *rope_sin = ws->rope_sin;
@@ -912,9 +931,10 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
rope_cos, rope_sin, len, n_kv_heads, head_dim);
}
- qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf,
- offsets, batch, n_heads,
- n_kv_heads, head_dim, scale);
+ qwen_bidirectional_gqa_attention_packed_with_scratch(
+ attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads,
+ n_kv_heads, head_dim, scale, ws->attn_scores,
+ ws->attn_scores_bytes);
linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index 369e050..efa606c 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1451,11 +1451,33 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
}
}
-void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale) {
+size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ const int *offsets, int batch) {
+#ifdef USE_BLAS
+ if (!offsets || batch <= 0) return 0;
+
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ if (len > max_seq) max_seq = len;
+ }
+ if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0;
+ if ((size_t)max_seq > SIZE_MAX / (sizeof(float) *
+ QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads))
+ return 0;
+ return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE *
+ (size_t)max_seq * sizeof(float);
+#else
+ (void)offsets;
+ (void)batch;
+ return 0;
+#endif
+}
+
+void qwen_bidirectional_gqa_attention_packed_with_scratch(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes) {
long long qk_work = 0;
int max_seq = 0;
for (int b = 0; b < batch; b++) {
@@ -1466,13 +1488,12 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
float *scores = NULL;
#ifdef USE_BLAS
- /* Keep score scratch linear in sequence length; OOM falls back online. */
- if (max_seq >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ &&
- (size_t)max_seq <= SIZE_MAX / (sizeof(float) *
- QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) {
- size_t n = (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * max_seq;
- scores = malloc(n * sizeof(*scores));
- }
+ size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required) scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
#endif
packed_gqa_attn_task_t task = {
.out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
@@ -1485,6 +1506,19 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
} else {
packed_gqa_attn_worker(0, 1, &task);
}
+}
+
+void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale) {
+ size_t scratch_bytes =
+ qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
+ qwen_bidirectional_gqa_attention_packed_with_scratch(
+ out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
+ scores, scratch_bytes);
free(scores);
}
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index 92082f1..c32ec3a 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -150,6 +150,14 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
int n_heads, int n_kv_heads,
int head_dim, float scale);
+/* Optional reusable scratch for the BLAS-tiled packed attention path. */
+size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ const int *offsets, int batch);
+void qwen_bidirectional_gqa_attention_packed_with_scratch(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes);
+
/* ========================================================================
* Position Embeddings
* ======================================================================== */