aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-20 05:26:35 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:17:30 +0200
commit7a88087ba0b5b1b388234f3d2a907673c63bd8cb (patch)
tree3a656367ab23d65d625c9cbbb3014ed109a2ebd2
parent47859fbfdc02329090c59357cbb826b42c946bf7 (diff)
downloadffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.tar.gz
ffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.zip
all: support batched embedding attention
-rw-r--r--main.c283
-rw-r--r--pplx_embed.c261
-rw-r--r--pplx_embed.h31
-rw-r--r--pplx_embed_mlx.c196
-rw-r--r--pplx_embed_mlx.h17
-rw-r--r--qwen_asr_kernels.c95
-rw-r--r--qwen_asr_kernels.h16
7 files changed, 689 insertions, 210 deletions
diff --git a/main.c b/main.c
index 9236a47..95c382e 100644
--- a/main.c
+++ b/main.c
@@ -30,6 +30,7 @@ static void print_usage(const char *prog)
" -d <dir> Model directory (required)\n"
" --mlx Use Apple MLX GPU backend\n"
" --daemon Read lines from stdin, write JSON embeddings to stdout\n"
+ " -b <n> Max texts per engine batch (default: all; daemon: 1)\n"
" -t <n> CPU threads (default: all cores)\n"
" -e Print raw embeddings (with multiple texts)\n"
" -v Verbose (-vv for debug)\n"
@@ -61,46 +62,48 @@ typedef struct {
} engine_t;
typedef struct {
- float *emb; /* malloc'd float[dim], caller frees */
- int n_tokens;
- double ms;
-} embed_result_t;
+ int *ids;
+ int n_tokens;
+} token_buf_t;
-static embed_result_t embed_text(engine_t *e, const char *text)
+static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
{
- embed_result_t r = {0};
+#ifdef USE_MLX
+ if (e->mlx_ctx)
+ return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings);
+ else
+#endif
+ return pplx_embed_batch(e->ctx, inputs, batch, out_embeddings);
+}
- int n = 0;
- int *ids = qwen_tokenizer_encode(e->tok, text, &n);
- if (!ids || n == 0) {
+static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
+{
+ memset(out, 0, sizeof(*out));
+ out->ids = qwen_tokenizer_encode(e->tok, text, &out->n_tokens);
+ if (!out->ids || out->n_tokens == 0) {
fprintf(stderr, "tokenization failed: %s\n", text);
- free(ids);
- return r;
+ free(out->ids);
+ out->ids = NULL;
+ out->n_tokens = 0;
+ return -1;
}
if (pplx_verbose >= 1) {
- fprintf(stderr, "tokens (%d): ", n);
- for (int i = 0; i < n && i < 20; i++) fprintf(stderr, "%d ", ids[i]);
- if (n > 20) fprintf(stderr, "...");
+ fprintf(stderr, "tokens (%d): ", out->n_tokens);
+ for (int i = 0; i < out->n_tokens && i < 20; i++)
+ fprintf(stderr, "%d ", out->ids[i]);
+ if (out->n_tokens > 20) fprintf(stderr, "...");
fprintf(stderr, "\n");
}
+ return 0;
+}
- double t0 = now_ms();
-#ifdef USE_MLX
- if (e->mlx_ctx)
- r.emb = pplx_mlx_embed(e->mlx_ctx, ids, n);
- else
-#endif
- r.emb = pplx_embed(e->ctx, ids, n);
- r.ms = now_ms() - t0;
- r.n_tokens = n;
- free(ids);
-
- if (!r.emb) fprintf(stderr, "forward pass failed\n");
- if (pplx_verbose >= 1)
- fprintf(stderr, "embed: %d tokens in %.1f ms\n", r.n_tokens, r.ms);
-
- return r;
+static void free_tokens(token_buf_t *tokens, int n)
+{
+ if (!tokens) return;
+ for (int i = 0; i < n; i++)
+ free(tokens[i].ids);
}
/* ========================================================================
@@ -131,12 +134,86 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens, double
* Daemon mode: read lines from stdin, write JSON to stdout
* ======================================================================== */
-static int run_daemon(engine_t *e)
+static int process_daemon_batch(engine_t *e, char **lines, int n_lines)
+{
+ if (n_lines <= 0) return 0;
+
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
+ pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t));
+ int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
+ float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
+ if (!tokens || !inputs || !input_to_line || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens); free(inputs); free(input_to_line); free(embs);
+ for (int i = 0; i < n_lines; i++)
+ printf("{\"error\":\"embedding failed\"}\n");
+ fflush(stdout);
+ return 1;
+ }
+
+ int n_inputs = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
+ inputs[n_inputs].ids = tokens[i].ids;
+ inputs[n_inputs].n_tokens = tokens[i].n_tokens;
+ input_to_line[n_inputs] = i;
+ n_inputs++;
+ }
+ }
+
+ double ms = 0.0;
+ int rc = 0;
+ if (n_inputs > 0) {
+ double t0 = now_ms();
+ rc = engine_embed_batch(e, inputs, n_inputs, embs);
+ ms = now_ms() - t0;
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
+ }
+
+ int next_valid = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (!tokens[i].ids) {
+ printf("{\"error\":\"tokenization failed\"}\n");
+ continue;
+ }
+ if (rc != 0) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ while (next_valid < n_inputs && input_to_line[next_valid] != i)
+ next_valid++;
+ if (next_valid >= n_inputs) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ print_embedding_json(embs + (size_t)next_valid * e->dim,
+ e->dim, tokens[i].n_tokens, ms);
+ next_valid++;
+ }
+ fflush(stdout);
+
+ free_tokens(tokens, n_lines);
+ free(tokens); free(inputs); free(input_to_line); free(embs);
+ return rc == 0 ? 0 : 1;
+}
+
+static int run_daemon(engine_t *e, int batch_size)
{
char line[65536];
+ if (batch_size <= 0) batch_size = 1;
if (pplx_verbose >= 1)
- fprintf(stderr, "daemon: ready, reading from stdin\n");
+ fprintf(stderr, "daemon: ready, reading from stdin (batch_size=%d)\n",
+ batch_size);
+
+ char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
+ if (!batch_lines) {
+ fprintf(stderr, "OOM\n");
+ return 1;
+ }
+ int n_batch = 0;
+ int rc = 0;
while (fgets(line, sizeof(line), stdin)) {
/* Strip newline */
@@ -149,47 +226,71 @@ static int run_daemon(engine_t *e)
fprintf(stderr, "daemon: \"%.*s%s\"\n",
(int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
- embed_result_t r = embed_text(e, line);
- if (r.emb) {
- print_embedding_json(r.emb, e->dim, r.n_tokens, r.ms);
- free(r.emb);
- } else {
- /* Output an error line so the consumer stays in sync */
- printf("{\"error\":\"embedding failed\"}\n");
- fflush(stdout);
+ batch_lines[n_batch] = strdup(line);
+ if (!batch_lines[n_batch]) {
+ fprintf(stderr, "OOM\n");
+ rc = 1;
+ break;
+ }
+ n_batch++;
+
+ if (n_batch == batch_size) {
+ if (process_daemon_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++) {
+ free(batch_lines[i]);
+ batch_lines[i] = NULL;
+ }
+ n_batch = 0;
}
}
+ if (n_batch > 0) {
+ if (process_daemon_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++)
+ free(batch_lines[i]);
+ }
+ free(batch_lines);
+
if (pplx_verbose >= 1)
fprintf(stderr, "daemon: stdin EOF\n");
- return 0;
+ return rc;
}
/* ========================================================================
* Batch mode: embed args or stdin lines, then print similarity or vectors
* ======================================================================== */
-static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int print_embs)
+static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
+{
+ if (*n_texts == *cap) {
+ int new_cap = *cap ? *cap * 2 : 8;
+ char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
+ if (!new_texts) return -1;
+ *texts = new_texts;
+ *cap = new_cap;
+ }
+
+ (*texts)[*n_texts] = strdup(s);
+ if (!(*texts)[*n_texts]) return -1;
+ (*n_texts)++;
+ return 0;
+}
+
+static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
+ int print_embs, int batch_size)
{
int n_texts = 0, cap = 0;
char **texts = NULL;
- float **embs = NULL;
-
-#define APPEND(s) do { \
- if (n_texts == cap) { \
- cap = cap ? cap * 2 : 8; \
- texts = realloc(texts, cap * sizeof(char *)); \
- embs = realloc(embs, cap * sizeof(float *)); \
- if (!texts || !embs) { fprintf(stderr, "OOM\n"); return 1; } \
- } \
- texts[n_texts] = strdup(s); \
- embs[n_texts] = NULL; \
- n_texts++; \
-} while (0)
+ int rc = 0;
if (arg_start < argc) {
- for (int i = arg_start; i < argc; i++) APPEND(argv[i]);
+ for (int i = arg_start; i < argc; i++) {
+ if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
+ goto oom_texts;
+ }
} else {
char line[65536];
while (fgets(line, sizeof(line), stdin)) {
@@ -197,29 +298,59 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int prin
while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
line[--l] = '\0';
if (l == 0) continue;
- APPEND(line);
+ if (append_text(&texts, &n_texts, &cap, line) != 0)
+ goto oom_texts;
}
}
-#undef APPEND
if (n_texts == 0) {
fprintf(stderr, "no input texts\n");
- free(texts); free(embs);
+ free(texts);
return 1;
}
int dim = e->dim;
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
+ pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t));
+ float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
+ if (!tokens || !inputs || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens); free(inputs); free(embs);
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts);
+ return 1;
+ }
for (int i = 0; i < n_texts; i++) {
if (pplx_verbose >= 1)
fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
- embed_result_t r = embed_text(e, texts[i]);
- embs[i] = r.emb;
- if (!embs[i]) goto done;
+ if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
+ rc = 1;
+ goto done;
+ }
+ inputs[i].ids = tokens[i].ids;
+ inputs[i].n_tokens = tokens[i].n_tokens;
+ }
+
+ int max_batch = batch_size > 0 ? batch_size : n_texts;
+ for (int start = 0; start < n_texts; start += max_batch) {
+ int cur = n_texts - start;
+ if (cur > max_batch) cur = max_batch;
+
+ double t0 = now_ms();
+ if (engine_embed_batch(e, inputs + start, cur,
+ embs + (size_t)start * dim) != 0) {
+ fprintf(stderr, "forward pass failed\n");
+ rc = 1;
+ goto done;
+ }
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n",
+ start, start + cur - 1, cur, now_ms() - t0);
}
if (n_texts == 1) {
- print_embedding_raw(embs[0], dim);
+ print_embedding_raw(embs, dim);
} else {
printf("Cosine similarity matrix (%d texts):\n", n_texts);
printf("%-4s", "");
@@ -228,22 +359,30 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int prin
for (int i = 0; i < n_texts; i++) {
printf("[%d] ", i);
for (int j = 0; j < n_texts; j++)
- printf(" %.4f", (double)pplx_cosine_similarity(embs[i], embs[j], dim));
+ printf(" %.4f", (double)pplx_cosine_similarity(
+ embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
printf(" \"%s\"\n", texts[i]);
}
if (print_embs) {
printf("\nEmbeddings:\n");
for (int i = 0; i < n_texts; i++) {
printf("[%d] ", i);
- print_embedding_raw(embs[i], dim);
+ print_embedding_raw(embs + (size_t)i * dim, dim);
}
}
}
done:
- for (int i = 0; i < n_texts; i++) { free(texts[i]); free(embs[i]); }
- free(texts); free(embs);
- return 0;
+ free_tokens(tokens, n_texts);
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts); free(tokens); free(inputs); free(embs);
+ return rc;
+
+oom_texts:
+ fprintf(stderr, "OOM\n");
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts);
+ return 1;
}
/* ========================================================================
@@ -258,12 +397,16 @@ int main(int argc, char *argv[])
int verbose = 0;
int use_mlx = 0;
int daemon = 0;
+ int batch_size = 0;
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
const char *f = argv[arg_start];
if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); }
+ else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg_start]);
+ }
else if (!strcmp(f, "-e")) { print_embs = 1; }
else if (!strcmp(f, "-v")) { verbose++; }
else if (!strcmp(f, "-vv")) { verbose = 2; }
@@ -330,9 +473,9 @@ int main(int argc, char *argv[])
/* Run */
int rc;
if (daemon)
- rc = run_daemon(&e);
+ rc = run_daemon(&e, batch_size > 0 ? batch_size : 1);
else
- rc = run_batch(&e, argc, argv, arg_start, print_embs);
+ rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size);
/* Cleanup */
if (e.ctx) pplx_free(e.ctx);
diff --git a/pplx_embed.c b/pplx_embed.c
index dcd1acd..942cfa9 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -316,28 +316,54 @@ void pplx_free(pplx_ctx_t *ctx)
}
/* ========================================================================
- * Forward pass (returns full token-level embeddings)
+ * Packed forward pass
* ======================================================================== */
-float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+static int build_offsets(const pplx_input_t *inputs, int batch,
+ int *offsets, int *total_out, int *max_out)
{
- if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+ if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
+ return -1;
+
+ int total = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return -1;
+ if (total > INT32_MAX - inputs[b].n_tokens)
+ return -1;
+ offsets[b] = total;
+ total += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
+ }
+ offsets[batch] = total;
+ *total_out = total;
+ *max_out = max_seq;
+ return 0;
+}
+
+static int forward_packed_inplace(pplx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, const int *offsets,
+ int total_seq, int max_seq)
+{
+ if (!ctx || !inputs || batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0)
+ return -1;
const pplx_config_t *cfg = &ctx->config;
const pplx_weights_t *w = &ctx->weights;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
int n_kv_heads = cfg->n_kv_heads;
- int head_dim = cfg->head_dim;
- int q_dim = cfg->q_dim;
- int kv_dim = cfg->kv_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->rms_norm_eps;
- int seq = n_tokens;
+ int head_dim = cfg->head_dim;
+ int q_dim = cfg->q_dim;
+ int kv_dim = cfg->kv_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->rms_norm_eps;
- if (ensure_buffers(ctx, seq) != 0) return NULL;
- if (ensure_rope_cache(ctx, seq) != 0) return NULL;
+ if (ensure_buffers(ctx, total_seq) != 0) return -1;
+ if (ensure_rope_cache(ctx, max_seq) != 0) return -1;
const float *rope_cos = ctx->rope_cos;
const float *rope_sin = ctx->rope_sin;
@@ -353,106 +379,179 @@ float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
float *ffn_up = ctx->ffn_up;
float *ffn_out = ctx->ffn_out;
- /* 1. Token embedding lookup */
- for (int i = 0; i < seq; i++) {
- int id = token_ids[i];
- if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "pplx: invalid token id %d at position %d\n", id, i);
- return NULL;
+ /* 1. Token embedding lookup into packed [total_seq, hidden]. */
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ for (int i = 0; i < inputs[b].n_tokens; i++) {
+ int id = inputs[b].ids[i];
+ if (id < 0 || id >= cfg->vocab_size) {
+ fprintf(stderr, "pplx: invalid token id %d at input %d position %d\n",
+ id, b, i);
+ return -1;
+ }
+ memcpy(x + (size_t)(start + i) * hidden,
+ w->embed_tokens + (size_t)id * hidden,
+ (size_t)hidden * sizeof(float));
}
- memcpy(x + i * hidden,
- w->embed_tokens + (size_t)id * hidden,
- hidden * sizeof(float));
}
float scale = 1.0f / sqrtf((float)head_dim);
- /* 2. Transformer layers */
+ /* 2. Transformer layers. Linear/MLP ops run over all packed tokens;
+ * attention is explicitly block-diagonal by sequence offsets. */
for (int layer = 0; layer < cfg->n_layers; layer++) {
const pplx_layer_t *l = &w->layers[layer];
- /* Input RMSNorm */
- qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps);
+ qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- /* QKV projections */
- qwen_linear_nobias(q_buf, x_norm, l->wq, seq, hidden, q_dim);
- qwen_linear_nobias(k_buf, x_norm, l->wk, seq, hidden, kv_dim);
- qwen_linear_nobias(v_buf, x_norm, l->wv, seq, hidden, kv_dim);
+ qwen_linear_nobias(q_buf, x_norm, l->wq, total_seq, hidden, q_dim);
+ qwen_linear_nobias(k_buf, x_norm, l->wk, total_seq, hidden, kv_dim);
+ qwen_linear_nobias(v_buf, x_norm, l->wv, total_seq, hidden, kv_dim);
- /* Per-head Q/K RMSNorm */
- qwen_rms_norm_per_head(q_buf, l->q_norm, seq, n_heads, head_dim, eps);
- qwen_rms_norm_per_head(k_buf, l->k_norm, seq, n_kv_heads, head_dim, eps);
+ qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
- /* RoPE */
- qwen_apply_rope_neox(q_buf, rope_cos, rope_sin, seq, n_heads, head_dim);
- qwen_apply_rope_neox(k_buf, rope_cos, rope_sin, seq, n_kv_heads, head_dim);
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ int len = offsets[b + 1] - start;
+ qwen_apply_rope_neox(q_buf + (size_t)start * q_dim,
+ rope_cos, rope_sin, len, n_heads, head_dim);
+ qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim,
+ rope_cos, rope_sin, len, n_kv_heads, head_dim);
+ }
- /* Bidirectional GQA attention (q_offset = seq-1 removes causal mask) */
- qwen_causal_attention(attn_out, q_buf, k_buf, v_buf,
- seq, seq, n_heads, n_kv_heads,
- head_dim, scale, seq - 1);
+ qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf,
+ offsets, batch, n_heads,
+ n_kv_heads, head_dim, scale);
- /* Output projection + residual */
- qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden);
- qwen_add_inplace(x, proj_out, seq * hidden);
+ qwen_linear_nobias(proj_out, attn_out, l->wo, total_seq, q_dim, hidden);
+ qwen_add_inplace(x, proj_out, total_seq * hidden);
- /* Post-attention RMSNorm */
- qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps);
+ qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- /* SwiGLU MLP */
- qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter);
- qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter);
- qwen_silu(ffn_gate, seq * inter);
- qwen_mul_inplace(ffn_gate, ffn_up, seq * inter);
- qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden);
- qwen_add_inplace(x, ffn_out, seq * hidden);
+ qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, total_seq, hidden, inter);
+ qwen_linear_nobias(ffn_up, x_norm, l->up_proj, total_seq, hidden, inter);
+ qwen_silu(ffn_gate, total_seq * inter);
+ qwen_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
+ qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, total_seq, inter, hidden);
+ qwen_add_inplace(x, ffn_out, total_seq * hidden);
}
- /* 3. Final RMSNorm (in-place) */
- qwen_rms_norm(x, x, w->norm, seq, hidden, eps);
+ qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps);
+ return 0;
+}
- /* Copy result out (caller frees) */
- size_t out_size = (size_t)seq * hidden * sizeof(float);
- float *out = (float *)malloc(out_size);
- if (out) memcpy(out, x, out_size);
- return out;
+static void pool_embeddings(const pplx_ctx_t *ctx, const int *offsets,
+ int batch, float *out_embeddings)
+{
+ int hidden = ctx->config.hidden_size;
+ const float *x = ctx->x;
+
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ int end = offsets[b + 1];
+ int len = end - start;
+ float *emb = out_embeddings + (size_t)b * hidden;
+
+ memset(emb, 0, (size_t)hidden * sizeof(float));
+ for (int i = start; i < end; i++) {
+ const float *row = x + (size_t)i * hidden;
+ for (int d = 0; d < hidden; d++)
+ emb[d] += row[d];
+ }
+
+ float inv_len = 1.0f / (float)len;
+ for (int d = 0; d < hidden; d++)
+ emb[d] *= inv_len;
+
+ float norm_sq = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ norm_sq += emb[d] * emb[d];
+ float nv = sqrtf(norm_sq);
+ if (nv > 1e-8f) {
+ float inv = 1.0f / nv;
+ for (int d = 0; d < hidden; d++)
+ emb[d] *= inv;
+ }
+ }
}
/* ========================================================================
- * Embed (forward + mean pool + L2 norm)
+ * Public forward/embed APIs
* ======================================================================== */
-float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+int pplx_forward_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_states)
{
- float *all = pplx_forward(ctx, token_ids, n_tokens);
- if (!all) return NULL;
+ if (!ctx || !token_ids || n_tokens <= 0 || !out_states) return -1;
- int hidden = ctx->config.hidden_size;
- int seq = n_tokens;
+ pplx_input_t input = { token_ids, n_tokens };
+ int offsets[2] = { 0, n_tokens };
+ if (forward_packed_inplace(ctx, &input, 1, offsets, n_tokens, n_tokens) != 0)
+ return -1;
- /* Mean pool */
- float *emb = (float *)calloc(hidden, sizeof(float));
- if (!emb) { free(all); return NULL; }
+ size_t out_size = (size_t)n_tokens * ctx->config.hidden_size * sizeof(float);
+ memcpy(out_states, ctx->x, out_size);
+ return 0;
+}
- for (int i = 0; i < seq; i++) {
- const float *row = all + i * hidden;
- for (int d = 0; d < hidden; d++)
- emb[d] += row[d];
+float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ size_t n = (size_t)n_tokens * ctx->config.hidden_size;
+ float *out = (float *)malloc(n * sizeof(float));
+ if (!out) return NULL;
+
+ if (pplx_forward_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
}
- free(all);
+ return out;
+}
- float inv_seq = 1.0f / (float)seq;
- for (int d = 0; d < hidden; d++) emb[d] *= inv_seq;
+int pplx_embed_batch(pplx_ctx_t *ctx, const pplx_input_t *inputs, int batch,
+ float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+
+ int *offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!offsets) return -1;
- /* L2 normalize */
- float norm_sq = 0.0f;
- for (int d = 0; d < hidden; d++) norm_sq += emb[d] * emb[d];
- float nv = sqrtf(norm_sq);
- if (nv > 1e-8f) {
- float inv = 1.0f / nv;
- for (int d = 0; d < hidden; d++) emb[d] *= inv;
+ int total_seq = 0;
+ int max_seq = 0;
+ if (build_offsets(inputs, batch, offsets, &total_seq, &max_seq) != 0) {
+ free(offsets);
+ return -1;
}
+ int rc = forward_packed_inplace(ctx, inputs, batch, offsets, total_seq, max_seq);
+ if (rc == 0)
+ pool_embeddings(ctx, offsets, batch, out_embeddings);
+
+ free(offsets);
+ return rc;
+}
+
+int pplx_embed_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ int hidden = ctx->config.hidden_size;
+ float *emb = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!emb) return NULL;
+
+ if (pplx_embed_into(ctx, token_ids, n_tokens, emb) != 0) {
+ free(emb);
+ return NULL;
+ }
return emb;
}
diff --git a/pplx_embed.h b/pplx_embed.h
index 67b193e..1dd50f2 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -92,6 +92,11 @@ typedef struct {
int rope_cache_cap;
} pplx_ctx_t;
+typedef struct {
+ const int *ids;
+ int n_tokens;
+} pplx_input_t;
+
/* ========================================================================
* API
* ======================================================================== */
@@ -124,6 +129,25 @@ void pplx_free(pplx_ctx_t *ctx);
float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
/*
+ * Compute one embedding into caller-provided out[hidden_size].
+ * Returns 0 on success, -1 on error.
+ */
+int pplx_embed_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_embedding);
+
+/*
+ * Compute embeddings for a true packed/ragged batch.
+ *
+ * out_embeddings is caller-provided [batch, hidden_size].
+ * Each sequence attends only to tokens from the same input, and RoPE
+ * positions restart at zero for every input.
+ *
+ * Returns 0 on success, -1 on error.
+ */
+int pplx_embed_batch(pplx_ctx_t *ctx, const pplx_input_t *inputs, int batch,
+ float *out_embeddings);
+
+/*
* Run the transformer forward pass WITHOUT pooling.
* Returns the full [n_tokens * hidden_size] output after final RMSNorm.
* Caller frees the returned buffer. NULL on error.
@@ -134,6 +158,13 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
/*
+ * Run the transformer forward pass into caller-provided
+ * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error.
+ */
+int pplx_forward_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_states);
+
+/*
* Cosine similarity between two L2-normalized vectors.
*/
float pplx_cosine_similarity(const float *a, const float *b, int dim);
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 3c82db1..d6fff8f 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -239,104 +239,152 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
* Forward pass
* ======================================================================== */
-float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
+int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
{
- if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
const pplx_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int n_heads = c->n_heads;
+ int hidden = c->hidden_size;
+ int n_heads = c->n_heads;
int n_kv_heads = c->n_kv_heads;
- int head_dim = c->head_dim;
- int q_dim = c->q_dim;
- int seq = n_tokens;
- float scale = 1.0f / sqrtf((float)head_dim);
+ int head_dim = c->head_dim;
+ int q_dim = c->q_dim;
+ int max_seq = 0;
+ int has_padding = 0;
+
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ if (max_seq <= 0) return -1;
+
+ for (int b = 0; b < batch; b++) {
+ if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ }
+
+ size_t elems = (size_t)batch * (size_t)max_seq;
+ int *padded_ids = (int *)malloc(elems * sizeof(int));
+ float *pool_mask_data = (float *)malloc(elems * sizeof(float));
+ float *attn_mask_data = has_padding ? (float *)malloc(elems * sizeof(float)) : NULL;
+ float *len_data = (float *)malloc((size_t)batch * sizeof(float));
+ if (!padded_ids || !pool_mask_data || (has_padding && !attn_mask_data) || !len_data) {
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ return -1;
+ }
+
+ for (int b = 0; b < batch; b++) {
+ len_data[b] = (float)inputs[b].n_tokens;
+ for (int t = 0; t < max_seq; t++) {
+ size_t idx = (size_t)b * max_seq + t;
+ if (t < inputs[b].n_tokens) {
+ int id = inputs[b].ids[t];
+ if (id < 0 || id >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ id, b, t);
+ free(padded_ids); free(pool_mask_data);
+ free(attn_mask_data); free(len_data);
+ return -1;
+ }
+ padded_ids[idx] = id;
+ pool_mask_data[idx] = 1.0f;
+ if (has_padding) attn_mask_data[idx] = 0.0f;
+ } else {
+ padded_ids[idx] = 0;
+ pool_mask_data[idx] = 0.0f;
+ if (has_padding) attn_mask_data[idx] = -1.0e9f;
+ }
+ }
+ }
+
+ int ids_shape[] = {batch, max_seq};
+ int pool_mask_shape[] = {batch, max_seq, 1};
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ int len_shape[] = {batch, 1};
+
+ mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ mlx_array pool_mask = mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32);
+ mlx_array attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array lengths = mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32);
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+
+ if (!arr_ok(ids) || !arr_ok(pool_mask) || (has_padding && !arr_ok(attn_mask)) ||
+ !arr_ok(lengths)) {
+ mlx_array_free(ids); mlx_array_free(pool_mask);
+ if (has_padding) mlx_array_free(attn_mask);
+ mlx_array_free(lengths);
+ return -1;
+ }
+
+ float scale = 1.0f / sqrtf((float)head_dim);
+ mlx_array null_arr = (mlx_array){0};
- /* 1. Embedding lookup */
- int ids_shape[] = {seq};
- mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32);
- mlx_array x = mlx_array_new();
+ /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
+ mlx_array x = mlx_array_new();
mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
mlx_array_free(ids);
- /* 2. Transformer layers */
+ /* 2. Transformer layers. */
for (int layer = 0; layer < c->n_layers; layer++) {
mlx_layer_t *l = &ctx->layers[layer];
- /* Input RMSNorm */
mlx_array xn = mlx_array_new();
mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
- /* QKV projections */
mlx_array q_flat = linear(xn, l->wq, S);
mlx_array k_flat = linear(xn, l->wk, S);
mlx_array v_flat = linear(xn, l->wv, S);
mlx_array_free(xn);
- /* Reshape to [seq, n_heads, head_dim] */
- int q_shape[] = {seq, n_heads, head_dim};
- int k_shape[] = {seq, n_kv_heads, head_dim};
+ int q_shape[] = {batch, max_seq, n_heads, head_dim};
+ int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
mlx_array q = mlx_array_new();
mlx_array k = mlx_array_new();
mlx_array v = mlx_array_new();
- mlx_reshape(&q, q_flat, q_shape, 3, S);
- mlx_reshape(&k, k_flat, k_shape, 3, S);
- mlx_reshape(&v, v_flat, k_shape, 3, S);
+ mlx_reshape(&q, q_flat, q_shape, 4, S);
+ mlx_reshape(&k, k_flat, k_shape, 4, S);
+ mlx_reshape(&v, v_flat, k_shape, 4, S);
mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
- /* Per-head Q/K RMSNorm */
mlx_array qn = mlx_array_new();
mlx_array kn = mlx_array_new();
mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
mlx_array_free(q); mlx_array_free(k);
- /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */
+ /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
int perm[] = {0, 2, 1, 3};
- mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new();
- mlx_expand_dims(&qe, qn, 0, S);
- mlx_expand_dims(&ke, kn, 0, S);
- mlx_expand_dims(&ve, v, 0, S);
- mlx_array_free(qn); mlx_array_free(kn);
-
mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, qe, perm, 4, S);
- mlx_transpose_axes(&kt, ke, perm, 4, S);
- mlx_transpose_axes(&vt, ve, perm, 4, S);
- mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve);
- mlx_array_free(v);
+ mlx_transpose_axes(&qt, qn, perm, 4, S);
+ mlx_transpose_axes(&kt, kn, perm, 4, S);
+ mlx_transpose_axes(&vt, v, perm, 4, S);
+ mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v);
- /* RoPE (NeoX: traditional=false) */
mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
- mlx_array null_arr = (mlx_array){0};
mlx_array qr = mlx_array_new(), kr = mlx_array_new();
mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
mlx_array_free(qt); mlx_array_free(kt);
- /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */
mlx_array attn = mlx_array_new();
mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale, "", null_arr, null_arr, S);
+ &attn, qr, kr, vt, scale, "",
+ has_padding ? attn_mask : null_arr, null_arr, S);
mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
- /* Transpose back and reshape to [seq, q_dim] */
mlx_array attn_t = mlx_array_new();
mlx_transpose_axes(&attn_t, attn, perm, 4, S);
mlx_array_free(attn);
- mlx_array attn_sq = mlx_array_new();
- mlx_squeeze_axis(&attn_sq, attn_t, 0, S);
- mlx_array_free(attn_t);
-
- int flat_shape[] = {seq, q_dim};
+ int flat_shape[] = {batch, max_seq, q_dim};
mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S);
- mlx_array_free(attn_sq);
+ mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S);
+ mlx_array_free(attn_t);
- /* Output projection + residual */
mlx_array proj = linear(attn_flat, l->wo, S);
mlx_array_free(attn_flat);
mlx_array x2 = mlx_array_new();
@@ -344,11 +392,9 @@ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
mlx_array_free(x); mlx_array_free(proj);
x = x2;
- /* Post-attention RMSNorm */
mlx_array xn2 = mlx_array_new();
mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
- /* SwiGLU MLP */
mlx_array gate = linear(xn2, l->gate_proj, S);
mlx_array up = linear(xn2, l->up_proj, S);
mlx_array_free(xn2);
@@ -372,32 +418,64 @@ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
x = x3;
}
- /* 3. Final RMSNorm */
+ /* 3. Final RMSNorm. */
mlx_array x_normed = mlx_array_new();
mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
mlx_array_free(x);
- /* 4. Mean pool over seq dim */
- mlx_array emb = mlx_array_new();
- mlx_mean_axis(&emb, x_normed, 0, false, S);
+ /* 4. Masked mean pool over T. */
+ mlx_array masked = mlx_array_new();
+ mlx_multiply(&masked, x_normed, pool_mask, S);
mlx_array_free(x_normed);
+ mlx_array_free(pool_mask);
- /* 5. L2 normalize */
- int norm_axes[] = {0};
+ mlx_array emb_sum = mlx_array_new();
+ mlx_sum_axis(&emb_sum, masked, 1, false, S);
+ mlx_array_free(masked);
+
+ mlx_array emb = mlx_array_new();
+ mlx_divide(&emb, emb_sum, lengths, S);
+ mlx_array_free(emb_sum); mlx_array_free(lengths);
+
+ /* 5. L2 normalize each row. */
+ int norm_axes[] = {1};
mlx_array nv = mlx_array_new();
mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S);
mlx_array emb_n = mlx_array_new();
mlx_divide(&emb_n, emb, nv, S);
mlx_array_free(emb); mlx_array_free(nv);
+ if (has_padding) mlx_array_free(attn_mask);
- /* 6. Eval and copy to CPU */
+ /* 6. Eval and copy [B, hidden] to CPU. */
mlx_array_eval(emb_n);
const float *data = mlx_array_data_float32(emb_n);
- float *out = NULL;
+ int rc = -1;
if (data) {
- out = (float *)malloc(hidden * sizeof(float));
- if (out) memcpy(out, data, hidden * sizeof(float));
+ memcpy(out_embeddings, data, (size_t)batch * hidden * sizeof(float));
+ rc = 0;
}
mlx_array_free(emb_n);
+ return rc;
+}
+
+int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ int hidden = ctx->config.hidden_size;
+ float *out = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!out) return NULL;
+
+ if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
+ }
return out;
}
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
index a1544ef..e528367 100644
--- a/pplx_embed_mlx.h
+++ b/pplx_embed_mlx.h
@@ -28,6 +28,23 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx);
*/
float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
+/*
+ * Compute one embedding into caller-provided out[hidden_size].
+ * Returns 0 on success, -1 on error.
+ */
+int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding);
+
+/*
+ * Compute a true padded dense batch on MLX.
+ *
+ * out_embeddings is caller-provided [batch, hidden_size].
+ * Padded tokens are masked out of attention keys and mean pooling.
+ * Returns 0 on success, -1 on error.
+ */
+int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings);
+
/* Get the config (so main.c can read hidden_size etc.) */
const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx);
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index 0ea3cc6..bc93f8a 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1135,6 +1135,101 @@ void qwen_causal_attention(float *out, const float *Q, const float *K, const flo
head_dim, scale, q_offset, 0, n_heads);
}
+static void qwen_bidirectional_gqa_attention_packed_heads(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, int head_start, int head_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+
+ for (int h = head_start; h < head_end; h++) {
+ int kv_h = h / heads_per_kv;
+
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ int end = offsets[b + 1];
+
+ for (int i = start; i < end; i++) {
+ const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
+ float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+
+ for (int j = start; j < end; j++) {
+ const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+
+ float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ qwen_vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f) {
+ float inv_sum = 1.0f / sum_exp;
+ qwen_vec_scale_inplace(o_row, inv_sum, head_dim);
+ }
+ }
+ }
+ }
+}
+
+typedef struct {
+ float *out;
+ const float *Q;
+ const float *K;
+ const float *V;
+ const int *offsets;
+ int batch;
+ int n_heads, n_kv_heads;
+ int head_dim;
+ float scale;
+} packed_gqa_attn_task_t;
+
+static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
+ packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+ int chunk = (t->n_heads + n_threads - 1) / n_threads;
+ int h0 = tid * chunk;
+ int h1 = h0 + chunk;
+ if (h1 > t->n_heads) h1 = t->n_heads;
+ if (h0 >= h1) return;
+
+ qwen_bidirectional_gqa_attention_packed_heads(
+ t->out, t->Q, t->K, t->V, t->offsets, t->batch,
+ t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h0, h1);
+}
+
+void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale) {
+ if (tp.n_threads > 1 && n_heads >= 2 && batch > 0) {
+ packed_gqa_attn_task_t task = {
+ .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
+ .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim, .scale = scale
+ };
+ parallel_for(packed_gqa_attn_worker, &task);
+ return;
+ }
+
+ qwen_bidirectional_gqa_attention_packed_heads(
+ out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, 0, n_heads);
+}
+
/* ========================================================================
* Position Embeddings
* ======================================================================== */
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index c59aeff..ce5e738 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -124,6 +124,22 @@ void qwen_causal_attention(float *out, const float *Q, const float *K, const flo
int seq_q, int seq_k, int n_heads, int n_kv_heads,
int head_dim, float scale, int q_offset);
+/*
+ * Bidirectional GQA attention over a packed/ragged batch.
+ *
+ * Q: [total_seq, n_heads * head_dim]
+ * K,V: [total_seq, n_kv_heads * head_dim]
+ * offsets: [batch + 1], token spans for each sequence.
+ *
+ * Attention is block-diagonal: tokens only attend within their own
+ * offsets[b]..offsets[b+1] span.
+ */
+void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale);
+
/* ========================================================================
* Position Embeddings
* ======================================================================== */