diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-20 05:26:35 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:17:30 +0200 |
| commit | 7a88087ba0b5b1b388234f3d2a907673c63bd8cb (patch) | |
| tree | 3a656367ab23d65d625c9cbbb3014ed109a2ebd2 | |
| parent | 47859fbfdc02329090c59357cbb826b42c946bf7 (diff) | |
| download | ffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.tar.gz ffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.zip | |
all: support batched embedding attention
| -rw-r--r-- | main.c | 283 | ||||
| -rw-r--r-- | pplx_embed.c | 261 | ||||
| -rw-r--r-- | pplx_embed.h | 31 | ||||
| -rw-r--r-- | pplx_embed_mlx.c | 196 | ||||
| -rw-r--r-- | pplx_embed_mlx.h | 17 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 95 | ||||
| -rw-r--r-- | qwen_asr_kernels.h | 16 |
7 files changed, 689 insertions, 210 deletions
@@ -30,6 +30,7 @@ static void print_usage(const char *prog) " -d <dir> Model directory (required)\n" " --mlx Use Apple MLX GPU backend\n" " --daemon Read lines from stdin, write JSON embeddings to stdout\n" + " -b <n> Max texts per engine batch (default: all; daemon: 1)\n" " -t <n> CPU threads (default: all cores)\n" " -e Print raw embeddings (with multiple texts)\n" " -v Verbose (-vv for debug)\n" @@ -61,46 +62,48 @@ typedef struct { } engine_t; typedef struct { - float *emb; /* malloc'd float[dim], caller frees */ - int n_tokens; - double ms; -} embed_result_t; + int *ids; + int n_tokens; +} token_buf_t; -static embed_result_t embed_text(engine_t *e, const char *text) +static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, + int batch, float *out_embeddings) { - embed_result_t r = {0}; +#ifdef USE_MLX + if (e->mlx_ctx) + return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings); + else +#endif + return pplx_embed_batch(e->ctx, inputs, batch, out_embeddings); +} - int n = 0; - int *ids = qwen_tokenizer_encode(e->tok, text, &n); - if (!ids || n == 0) { +static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) +{ + memset(out, 0, sizeof(*out)); + out->ids = qwen_tokenizer_encode(e->tok, text, &out->n_tokens); + if (!out->ids || out->n_tokens == 0) { fprintf(stderr, "tokenization failed: %s\n", text); - free(ids); - return r; + free(out->ids); + out->ids = NULL; + out->n_tokens = 0; + return -1; } if (pplx_verbose >= 1) { - fprintf(stderr, "tokens (%d): ", n); - for (int i = 0; i < n && i < 20; i++) fprintf(stderr, "%d ", ids[i]); - if (n > 20) fprintf(stderr, "..."); + fprintf(stderr, "tokens (%d): ", out->n_tokens); + for (int i = 0; i < out->n_tokens && i < 20; i++) + fprintf(stderr, "%d ", out->ids[i]); + if (out->n_tokens > 20) fprintf(stderr, "..."); fprintf(stderr, "\n"); } + return 0; +} - double t0 = now_ms(); -#ifdef USE_MLX - if (e->mlx_ctx) - r.emb = pplx_mlx_embed(e->mlx_ctx, ids, n); - else -#endif - r.emb = pplx_embed(e->ctx, ids, n); - r.ms = now_ms() - t0; - r.n_tokens = n; - free(ids); - - if (!r.emb) fprintf(stderr, "forward pass failed\n"); - if (pplx_verbose >= 1) - fprintf(stderr, "embed: %d tokens in %.1f ms\n", r.n_tokens, r.ms); - - return r; +static void free_tokens(token_buf_t *tokens, int n) +{ + if (!tokens) return; + for (int i = 0; i < n; i++) + free(tokens[i].ids); } /* ======================================================================== @@ -131,12 +134,86 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens, double * Daemon mode: read lines from stdin, write JSON to stdout * ======================================================================== */ -static int run_daemon(engine_t *e) +static int process_daemon_batch(engine_t *e, char **lines, int n_lines) +{ + if (n_lines <= 0) return 0; + + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); + pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t)); + int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int)); + float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); + if (!tokens || !inputs || !input_to_line || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); free(inputs); free(input_to_line); free(embs); + for (int i = 0; i < n_lines; i++) + printf("{\"error\":\"embedding failed\"}\n"); + fflush(stdout); + return 1; + } + + int n_inputs = 0; + for (int i = 0; i < n_lines; i++) { + if (tokenize_text(e, lines[i], &tokens[i]) == 0) { + inputs[n_inputs].ids = tokens[i].ids; + inputs[n_inputs].n_tokens = tokens[i].n_tokens; + input_to_line[n_inputs] = i; + n_inputs++; + } + } + + double ms = 0.0; + int rc = 0; + if (n_inputs > 0) { + double t0 = now_ms(); + rc = engine_embed_batch(e, inputs, n_inputs, embs); + ms = now_ms() - t0; + if (pplx_verbose >= 1) + fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms); + } + + int next_valid = 0; + for (int i = 0; i < n_lines; i++) { + if (!tokens[i].ids) { + printf("{\"error\":\"tokenization failed\"}\n"); + continue; + } + if (rc != 0) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + while (next_valid < n_inputs && input_to_line[next_valid] != i) + next_valid++; + if (next_valid >= n_inputs) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + print_embedding_json(embs + (size_t)next_valid * e->dim, + e->dim, tokens[i].n_tokens, ms); + next_valid++; + } + fflush(stdout); + + free_tokens(tokens, n_lines); + free(tokens); free(inputs); free(input_to_line); free(embs); + return rc == 0 ? 0 : 1; +} + +static int run_daemon(engine_t *e, int batch_size) { char line[65536]; + if (batch_size <= 0) batch_size = 1; if (pplx_verbose >= 1) - fprintf(stderr, "daemon: ready, reading from stdin\n"); + fprintf(stderr, "daemon: ready, reading from stdin (batch_size=%d)\n", + batch_size); + + char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); + if (!batch_lines) { + fprintf(stderr, "OOM\n"); + return 1; + } + int n_batch = 0; + int rc = 0; while (fgets(line, sizeof(line), stdin)) { /* Strip newline */ @@ -149,47 +226,71 @@ static int run_daemon(engine_t *e) fprintf(stderr, "daemon: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); - embed_result_t r = embed_text(e, line); - if (r.emb) { - print_embedding_json(r.emb, e->dim, r.n_tokens, r.ms); - free(r.emb); - } else { - /* Output an error line so the consumer stays in sync */ - printf("{\"error\":\"embedding failed\"}\n"); - fflush(stdout); + batch_lines[n_batch] = strdup(line); + if (!batch_lines[n_batch]) { + fprintf(stderr, "OOM\n"); + rc = 1; + break; + } + n_batch++; + + if (n_batch == batch_size) { + if (process_daemon_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) { + free(batch_lines[i]); + batch_lines[i] = NULL; + } + n_batch = 0; } } + if (n_batch > 0) { + if (process_daemon_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) + free(batch_lines[i]); + } + free(batch_lines); + if (pplx_verbose >= 1) fprintf(stderr, "daemon: stdin EOF\n"); - return 0; + return rc; } /* ======================================================================== * Batch mode: embed args or stdin lines, then print similarity or vectors * ======================================================================== */ -static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int print_embs) +static int append_text(char ***texts, int *n_texts, int *cap, const char *s) +{ + if (*n_texts == *cap) { + int new_cap = *cap ? *cap * 2 : 8; + char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); + if (!new_texts) return -1; + *texts = new_texts; + *cap = new_cap; + } + + (*texts)[*n_texts] = strdup(s); + if (!(*texts)[*n_texts]) return -1; + (*n_texts)++; + return 0; +} + +static int run_batch(engine_t *e, int argc, char **argv, int arg_start, + int print_embs, int batch_size) { int n_texts = 0, cap = 0; char **texts = NULL; - float **embs = NULL; - -#define APPEND(s) do { \ - if (n_texts == cap) { \ - cap = cap ? cap * 2 : 8; \ - texts = realloc(texts, cap * sizeof(char *)); \ - embs = realloc(embs, cap * sizeof(float *)); \ - if (!texts || !embs) { fprintf(stderr, "OOM\n"); return 1; } \ - } \ - texts[n_texts] = strdup(s); \ - embs[n_texts] = NULL; \ - n_texts++; \ -} while (0) + int rc = 0; if (arg_start < argc) { - for (int i = arg_start; i < argc; i++) APPEND(argv[i]); + for (int i = arg_start; i < argc; i++) { + if (append_text(&texts, &n_texts, &cap, argv[i]) != 0) + goto oom_texts; + } } else { char line[65536]; while (fgets(line, sizeof(line), stdin)) { @@ -197,29 +298,59 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int prin while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) line[--l] = '\0'; if (l == 0) continue; - APPEND(line); + if (append_text(&texts, &n_texts, &cap, line) != 0) + goto oom_texts; } } -#undef APPEND if (n_texts == 0) { fprintf(stderr, "no input texts\n"); - free(texts); free(embs); + free(texts); return 1; } int dim = e->dim; + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t)); + pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t)); + float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); + if (!tokens || !inputs || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); free(inputs); free(embs); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); + return 1; + } for (int i = 0; i < n_texts; i++) { if (pplx_verbose >= 1) fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); - embed_result_t r = embed_text(e, texts[i]); - embs[i] = r.emb; - if (!embs[i]) goto done; + if (tokenize_text(e, texts[i], &tokens[i]) != 0) { + rc = 1; + goto done; + } + inputs[i].ids = tokens[i].ids; + inputs[i].n_tokens = tokens[i].n_tokens; + } + + int max_batch = batch_size > 0 ? batch_size : n_texts; + for (int start = 0; start < n_texts; start += max_batch) { + int cur = n_texts - start; + if (cur > max_batch) cur = max_batch; + + double t0 = now_ms(); + if (engine_embed_batch(e, inputs + start, cur, + embs + (size_t)start * dim) != 0) { + fprintf(stderr, "forward pass failed\n"); + rc = 1; + goto done; + } + if (pplx_verbose >= 1) + fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", + start, start + cur - 1, cur, now_ms() - t0); } if (n_texts == 1) { - print_embedding_raw(embs[0], dim); + print_embedding_raw(embs, dim); } else { printf("Cosine similarity matrix (%d texts):\n", n_texts); printf("%-4s", ""); @@ -228,22 +359,30 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int prin for (int i = 0; i < n_texts; i++) { printf("[%d] ", i); for (int j = 0; j < n_texts; j++) - printf(" %.4f", (double)pplx_cosine_similarity(embs[i], embs[j], dim)); + printf(" %.4f", (double)pplx_cosine_similarity( + embs + (size_t)i * dim, embs + (size_t)j * dim, dim)); printf(" \"%s\"\n", texts[i]); } if (print_embs) { printf("\nEmbeddings:\n"); for (int i = 0; i < n_texts; i++) { printf("[%d] ", i); - print_embedding_raw(embs[i], dim); + print_embedding_raw(embs + (size_t)i * dim, dim); } } } done: - for (int i = 0; i < n_texts; i++) { free(texts[i]); free(embs[i]); } - free(texts); free(embs); - return 0; + free_tokens(tokens, n_texts); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); free(tokens); free(inputs); free(embs); + return rc; + +oom_texts: + fprintf(stderr, "OOM\n"); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); + return 1; } /* ======================================================================== @@ -258,12 +397,16 @@ int main(int argc, char *argv[]) int verbose = 0; int use_mlx = 0; int daemon = 0; + int batch_size = 0; int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { const char *f = argv[arg_start]; if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); } + else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg_start]); + } else if (!strcmp(f, "-e")) { print_embs = 1; } else if (!strcmp(f, "-v")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } @@ -330,9 +473,9 @@ int main(int argc, char *argv[]) /* Run */ int rc; if (daemon) - rc = run_daemon(&e); + rc = run_daemon(&e, batch_size > 0 ? batch_size : 1); else - rc = run_batch(&e, argc, argv, arg_start, print_embs); + rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size); /* Cleanup */ if (e.ctx) pplx_free(e.ctx); diff --git a/pplx_embed.c b/pplx_embed.c index dcd1acd..942cfa9 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -316,28 +316,54 @@ void pplx_free(pplx_ctx_t *ctx) } /* ======================================================================== - * Forward pass (returns full token-level embeddings) + * Packed forward pass * ======================================================================== */ -float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +static int build_offsets(const pplx_input_t *inputs, int batch, + int *offsets, int *total_out, int *max_out) { - if (!ctx || !token_ids || n_tokens <= 0) return NULL; + if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) + return -1; + + int total = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return -1; + if (total > INT32_MAX - inputs[b].n_tokens) + return -1; + offsets[b] = total; + total += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + } + offsets[batch] = total; + *total_out = total; + *max_out = max_seq; + return 0; +} + +static int forward_packed_inplace(pplx_ctx_t *ctx, const pplx_input_t *inputs, + int batch, const int *offsets, + int total_seq, int max_seq) +{ + if (!ctx || !inputs || batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0) + return -1; const pplx_config_t *cfg = &ctx->config; const pplx_weights_t *w = &ctx->weights; - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; int n_kv_heads = cfg->n_kv_heads; - int head_dim = cfg->head_dim; - int q_dim = cfg->q_dim; - int kv_dim = cfg->kv_dim; - int inter = cfg->intermediate_size; - float eps = cfg->rms_norm_eps; - int seq = n_tokens; + int head_dim = cfg->head_dim; + int q_dim = cfg->q_dim; + int kv_dim = cfg->kv_dim; + int inter = cfg->intermediate_size; + float eps = cfg->rms_norm_eps; - if (ensure_buffers(ctx, seq) != 0) return NULL; - if (ensure_rope_cache(ctx, seq) != 0) return NULL; + if (ensure_buffers(ctx, total_seq) != 0) return -1; + if (ensure_rope_cache(ctx, max_seq) != 0) return -1; const float *rope_cos = ctx->rope_cos; const float *rope_sin = ctx->rope_sin; @@ -353,106 +379,179 @@ float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) float *ffn_up = ctx->ffn_up; float *ffn_out = ctx->ffn_out; - /* 1. Token embedding lookup */ - for (int i = 0; i < seq; i++) { - int id = token_ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "pplx: invalid token id %d at position %d\n", id, i); - return NULL; + /* 1. Token embedding lookup into packed [total_seq, hidden]. */ + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + for (int i = 0; i < inputs[b].n_tokens; i++) { + int id = inputs[b].ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "pplx: invalid token id %d at input %d position %d\n", + id, b, i); + return -1; + } + memcpy(x + (size_t)(start + i) * hidden, + w->embed_tokens + (size_t)id * hidden, + (size_t)hidden * sizeof(float)); } - memcpy(x + i * hidden, - w->embed_tokens + (size_t)id * hidden, - hidden * sizeof(float)); } float scale = 1.0f / sqrtf((float)head_dim); - /* 2. Transformer layers */ + /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; + * attention is explicitly block-diagonal by sequence offsets. */ for (int layer = 0; layer < cfg->n_layers; layer++) { const pplx_layer_t *l = &w->layers[layer]; - /* Input RMSNorm */ - qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps); + qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - /* QKV projections */ - qwen_linear_nobias(q_buf, x_norm, l->wq, seq, hidden, q_dim); - qwen_linear_nobias(k_buf, x_norm, l->wk, seq, hidden, kv_dim); - qwen_linear_nobias(v_buf, x_norm, l->wv, seq, hidden, kv_dim); + qwen_linear_nobias(q_buf, x_norm, l->wq, total_seq, hidden, q_dim); + qwen_linear_nobias(k_buf, x_norm, l->wk, total_seq, hidden, kv_dim); + qwen_linear_nobias(v_buf, x_norm, l->wv, total_seq, hidden, kv_dim); - /* Per-head Q/K RMSNorm */ - qwen_rms_norm_per_head(q_buf, l->q_norm, seq, n_heads, head_dim, eps); - qwen_rms_norm_per_head(k_buf, l->k_norm, seq, n_kv_heads, head_dim, eps); + qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); - /* RoPE */ - qwen_apply_rope_neox(q_buf, rope_cos, rope_sin, seq, n_heads, head_dim); - qwen_apply_rope_neox(k_buf, rope_cos, rope_sin, seq, n_kv_heads, head_dim); + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + int len = offsets[b + 1] - start; + qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, + rope_cos, rope_sin, len, n_heads, head_dim); + qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, + rope_cos, rope_sin, len, n_kv_heads, head_dim); + } - /* Bidirectional GQA attention (q_offset = seq-1 removes causal mask) */ - qwen_causal_attention(attn_out, q_buf, k_buf, v_buf, - seq, seq, n_heads, n_kv_heads, - head_dim, scale, seq - 1); + qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf, + offsets, batch, n_heads, + n_kv_heads, head_dim, scale); - /* Output projection + residual */ - qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden); - qwen_add_inplace(x, proj_out, seq * hidden); + qwen_linear_nobias(proj_out, attn_out, l->wo, total_seq, q_dim, hidden); + qwen_add_inplace(x, proj_out, total_seq * hidden); - /* Post-attention RMSNorm */ - qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps); + qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - /* SwiGLU MLP */ - qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter); - qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter); - qwen_silu(ffn_gate, seq * inter); - qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); - qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden); - qwen_add_inplace(x, ffn_out, seq * hidden); + qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, total_seq, hidden, inter); + qwen_linear_nobias(ffn_up, x_norm, l->up_proj, total_seq, hidden, inter); + qwen_silu(ffn_gate, total_seq * inter); + qwen_mul_inplace(ffn_gate, ffn_up, total_seq * inter); + qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, total_seq, inter, hidden); + qwen_add_inplace(x, ffn_out, total_seq * hidden); } - /* 3. Final RMSNorm (in-place) */ - qwen_rms_norm(x, x, w->norm, seq, hidden, eps); + qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); + return 0; +} - /* Copy result out (caller frees) */ - size_t out_size = (size_t)seq * hidden * sizeof(float); - float *out = (float *)malloc(out_size); - if (out) memcpy(out, x, out_size); - return out; +static void pool_embeddings(const pplx_ctx_t *ctx, const int *offsets, + int batch, float *out_embeddings) +{ + int hidden = ctx->config.hidden_size; + const float *x = ctx->x; + + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + int end = offsets[b + 1]; + int len = end - start; + float *emb = out_embeddings + (size_t)b * hidden; + + memset(emb, 0, (size_t)hidden * sizeof(float)); + for (int i = start; i < end; i++) { + const float *row = x + (size_t)i * hidden; + for (int d = 0; d < hidden; d++) + emb[d] += row[d]; + } + + float inv_len = 1.0f / (float)len; + for (int d = 0; d < hidden; d++) + emb[d] *= inv_len; + + float norm_sq = 0.0f; + for (int d = 0; d < hidden; d++) + norm_sq += emb[d] * emb[d]; + float nv = sqrtf(norm_sq); + if (nv > 1e-8f) { + float inv = 1.0f / nv; + for (int d = 0; d < hidden; d++) + emb[d] *= inv; + } + } } /* ======================================================================== - * Embed (forward + mean pool + L2 norm) + * Public forward/embed APIs * ======================================================================== */ -float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +int pplx_forward_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens, + float *out_states) { - float *all = pplx_forward(ctx, token_ids, n_tokens); - if (!all) return NULL; + if (!ctx || !token_ids || n_tokens <= 0 || !out_states) return -1; - int hidden = ctx->config.hidden_size; - int seq = n_tokens; + pplx_input_t input = { token_ids, n_tokens }; + int offsets[2] = { 0, n_tokens }; + if (forward_packed_inplace(ctx, &input, 1, offsets, n_tokens, n_tokens) != 0) + return -1; - /* Mean pool */ - float *emb = (float *)calloc(hidden, sizeof(float)); - if (!emb) { free(all); return NULL; } + size_t out_size = (size_t)n_tokens * ctx->config.hidden_size * sizeof(float); + memcpy(out_states, ctx->x, out_size); + return 0; +} - for (int i = 0; i < seq; i++) { - const float *row = all + i * hidden; - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; +float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + size_t n = (size_t)n_tokens * ctx->config.hidden_size; + float *out = (float *)malloc(n * sizeof(float)); + if (!out) return NULL; + + if (pplx_forward_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; } - free(all); + return out; +} - float inv_seq = 1.0f / (float)seq; - for (int d = 0; d < hidden; d++) emb[d] *= inv_seq; +int pplx_embed_batch(pplx_ctx_t *ctx, const pplx_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + + int *offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!offsets) return -1; - /* L2 normalize */ - float norm_sq = 0.0f; - for (int d = 0; d < hidden; d++) norm_sq += emb[d] * emb[d]; - float nv = sqrtf(norm_sq); - if (nv > 1e-8f) { - float inv = 1.0f / nv; - for (int d = 0; d < hidden; d++) emb[d] *= inv; + int total_seq = 0; + int max_seq = 0; + if (build_offsets(inputs, batch, offsets, &total_seq, &max_seq) != 0) { + free(offsets); + return -1; } + int rc = forward_packed_inplace(ctx, inputs, batch, offsets, total_seq, max_seq); + if (rc == 0) + pool_embeddings(ctx, offsets, batch, out_embeddings); + + free(offsets); + return rc; +} + +int pplx_embed_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens, + float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_embed_batch(ctx, &input, 1, out_embedding); +} + +float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + int hidden = ctx->config.hidden_size; + float *emb = (float *)malloc((size_t)hidden * sizeof(float)); + if (!emb) return NULL; + + if (pplx_embed_into(ctx, token_ids, n_tokens, emb) != 0) { + free(emb); + return NULL; + } return emb; } diff --git a/pplx_embed.h b/pplx_embed.h index 67b193e..1dd50f2 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -92,6 +92,11 @@ typedef struct { int rope_cache_cap; } pplx_ctx_t; +typedef struct { + const int *ids; + int n_tokens; +} pplx_input_t; + /* ======================================================================== * API * ======================================================================== */ @@ -124,6 +129,25 @@ void pplx_free(pplx_ctx_t *ctx); float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); /* + * Compute one embedding into caller-provided out[hidden_size]. + * Returns 0 on success, -1 on error. + */ +int pplx_embed_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens, + float *out_embedding); + +/* + * Compute embeddings for a true packed/ragged batch. + * + * out_embeddings is caller-provided [batch, hidden_size]. + * Each sequence attends only to tokens from the same input, and RoPE + * positions restart at zero for every input. + * + * Returns 0 on success, -1 on error. + */ +int pplx_embed_batch(pplx_ctx_t *ctx, const pplx_input_t *inputs, int batch, + float *out_embeddings); + +/* * Run the transformer forward pass WITHOUT pooling. * Returns the full [n_tokens * hidden_size] output after final RMSNorm. * Caller frees the returned buffer. NULL on error. @@ -134,6 +158,13 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); /* + * Run the transformer forward pass into caller-provided + * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error. + */ +int pplx_forward_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens, + float *out_states); + +/* * Cosine similarity between two L2-normalized vectors. */ float pplx_cosine_similarity(const float *a, const float *b, int dim); diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 3c82db1..d6fff8f 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -239,104 +239,152 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx) * Forward pass * ======================================================================== */ -float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) +int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings) { - if (!ctx || !token_ids || n_tokens <= 0) return NULL; + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; const pplx_config_t *c = &ctx->config; mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int n_heads = c->n_heads; + int hidden = c->hidden_size; + int n_heads = c->n_heads; int n_kv_heads = c->n_kv_heads; - int head_dim = c->head_dim; - int q_dim = c->q_dim; - int seq = n_tokens; - float scale = 1.0f / sqrtf((float)head_dim); + int head_dim = c->head_dim; + int q_dim = c->q_dim; + int max_seq = 0; + int has_padding = 0; + + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + } + if (max_seq <= 0) return -1; + + for (int b = 0; b < batch; b++) { + if (inputs[b].n_tokens != max_seq) has_padding = 1; + } + + size_t elems = (size_t)batch * (size_t)max_seq; + int *padded_ids = (int *)malloc(elems * sizeof(int)); + float *pool_mask_data = (float *)malloc(elems * sizeof(float)); + float *attn_mask_data = has_padding ? (float *)malloc(elems * sizeof(float)) : NULL; + float *len_data = (float *)malloc((size_t)batch * sizeof(float)); + if (!padded_ids || !pool_mask_data || (has_padding && !attn_mask_data) || !len_data) { + free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + return -1; + } + + for (int b = 0; b < batch; b++) { + len_data[b] = (float)inputs[b].n_tokens; + for (int t = 0; t < max_seq; t++) { + size_t idx = (size_t)b * max_seq + t; + if (t < inputs[b].n_tokens) { + int id = inputs[b].ids[t]; + if (id < 0 || id >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + id, b, t); + free(padded_ids); free(pool_mask_data); + free(attn_mask_data); free(len_data); + return -1; + } + padded_ids[idx] = id; + pool_mask_data[idx] = 1.0f; + if (has_padding) attn_mask_data[idx] = 0.0f; + } else { + padded_ids[idx] = 0; + pool_mask_data[idx] = 0.0f; + if (has_padding) attn_mask_data[idx] = -1.0e9f; + } + } + } + + int ids_shape[] = {batch, max_seq}; + int pool_mask_shape[] = {batch, max_seq, 1}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + int len_shape[] = {batch, 1}; + + mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + mlx_array pool_mask = mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32); + mlx_array attn_mask = has_padding + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array lengths = mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32); + free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + + if (!arr_ok(ids) || !arr_ok(pool_mask) || (has_padding && !arr_ok(attn_mask)) || + !arr_ok(lengths)) { + mlx_array_free(ids); mlx_array_free(pool_mask); + if (has_padding) mlx_array_free(attn_mask); + mlx_array_free(lengths); + return -1; + } + + float scale = 1.0f / sqrtf((float)head_dim); + mlx_array null_arr = (mlx_array){0}; - /* 1. Embedding lookup */ - int ids_shape[] = {seq}; - mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32); - mlx_array x = mlx_array_new(); + /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */ + mlx_array x = mlx_array_new(); mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); mlx_array_free(ids); - /* 2. Transformer layers */ + /* 2. Transformer layers. */ for (int layer = 0; layer < c->n_layers; layer++) { mlx_layer_t *l = &ctx->layers[layer]; - /* Input RMSNorm */ mlx_array xn = mlx_array_new(); mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); - /* QKV projections */ mlx_array q_flat = linear(xn, l->wq, S); mlx_array k_flat = linear(xn, l->wk, S); mlx_array v_flat = linear(xn, l->wv, S); mlx_array_free(xn); - /* Reshape to [seq, n_heads, head_dim] */ - int q_shape[] = {seq, n_heads, head_dim}; - int k_shape[] = {seq, n_kv_heads, head_dim}; + int q_shape[] = {batch, max_seq, n_heads, head_dim}; + int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; mlx_array q = mlx_array_new(); mlx_array k = mlx_array_new(); mlx_array v = mlx_array_new(); - mlx_reshape(&q, q_flat, q_shape, 3, S); - mlx_reshape(&k, k_flat, k_shape, 3, S); - mlx_reshape(&v, v_flat, k_shape, 3, S); + mlx_reshape(&q, q_flat, q_shape, 4, S); + mlx_reshape(&k, k_flat, k_shape, 4, S); + mlx_reshape(&v, v_flat, k_shape, 4, S); mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); - /* Per-head Q/K RMSNorm */ mlx_array qn = mlx_array_new(); mlx_array kn = mlx_array_new(); mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); mlx_array_free(q); mlx_array_free(k); - /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */ + /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ int perm[] = {0, 2, 1, 3}; - mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new(); - mlx_expand_dims(&qe, qn, 0, S); - mlx_expand_dims(&ke, kn, 0, S); - mlx_expand_dims(&ve, v, 0, S); - mlx_array_free(qn); mlx_array_free(kn); - mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); - mlx_transpose_axes(&qt, qe, perm, 4, S); - mlx_transpose_axes(&kt, ke, perm, 4, S); - mlx_transpose_axes(&vt, ve, perm, 4, S); - mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve); - mlx_array_free(v); + mlx_transpose_axes(&qt, qn, perm, 4, S); + mlx_transpose_axes(&kt, kn, perm, 4, S); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); - /* RoPE (NeoX: traditional=false) */ mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; - mlx_array null_arr = (mlx_array){0}; mlx_array qr = mlx_array_new(), kr = mlx_array_new(); mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_array_free(qt); mlx_array_free(kt); - /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */ mlx_array attn = mlx_array_new(); mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, "", null_arr, null_arr, S); + &attn, qr, kr, vt, scale, "", + has_padding ? attn_mask : null_arr, null_arr, S); mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); - /* Transpose back and reshape to [seq, q_dim] */ mlx_array attn_t = mlx_array_new(); mlx_transpose_axes(&attn_t, attn, perm, 4, S); mlx_array_free(attn); - mlx_array attn_sq = mlx_array_new(); - mlx_squeeze_axis(&attn_sq, attn_t, 0, S); - mlx_array_free(attn_t); - - int flat_shape[] = {seq, q_dim}; + int flat_shape[] = {batch, max_seq, q_dim}; mlx_array attn_flat = mlx_array_new(); - mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S); - mlx_array_free(attn_sq); + mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); + mlx_array_free(attn_t); - /* Output projection + residual */ mlx_array proj = linear(attn_flat, l->wo, S); mlx_array_free(attn_flat); mlx_array x2 = mlx_array_new(); @@ -344,11 +392,9 @@ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) mlx_array_free(x); mlx_array_free(proj); x = x2; - /* Post-attention RMSNorm */ mlx_array xn2 = mlx_array_new(); mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); - /* SwiGLU MLP */ mlx_array gate = linear(xn2, l->gate_proj, S); mlx_array up = linear(xn2, l->up_proj, S); mlx_array_free(xn2); @@ -372,32 +418,64 @@ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) x = x3; } - /* 3. Final RMSNorm */ + /* 3. Final RMSNorm. */ mlx_array x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); mlx_array_free(x); - /* 4. Mean pool over seq dim */ - mlx_array emb = mlx_array_new(); - mlx_mean_axis(&emb, x_normed, 0, false, S); + /* 4. Masked mean pool over T. */ + mlx_array masked = mlx_array_new(); + mlx_multiply(&masked, x_normed, pool_mask, S); mlx_array_free(x_normed); + mlx_array_free(pool_mask); - /* 5. L2 normalize */ - int norm_axes[] = {0}; + mlx_array emb_sum = mlx_array_new(); + mlx_sum_axis(&emb_sum, masked, 1, false, S); + mlx_array_free(masked); + + mlx_array emb = mlx_array_new(); + mlx_divide(&emb, emb_sum, lengths, S); + mlx_array_free(emb_sum); mlx_array_free(lengths); + + /* 5. L2 normalize each row. */ + int norm_axes[] = {1}; mlx_array nv = mlx_array_new(); mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S); mlx_array emb_n = mlx_array_new(); mlx_divide(&emb_n, emb, nv, S); mlx_array_free(emb); mlx_array_free(nv); + if (has_padding) mlx_array_free(attn_mask); - /* 6. Eval and copy to CPU */ + /* 6. Eval and copy [B, hidden] to CPU. */ mlx_array_eval(emb_n); const float *data = mlx_array_data_float32(emb_n); - float *out = NULL; + int rc = -1; if (data) { - out = (float *)malloc(hidden * sizeof(float)); - if (out) memcpy(out, data, hidden * sizeof(float)); + memcpy(out_embeddings, data, (size_t)batch * hidden * sizeof(float)); + rc = 0; } mlx_array_free(emb_n); + return rc; +} + +int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); +} + +float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + int hidden = ctx->config.hidden_size; + float *out = (float *)malloc((size_t)hidden * sizeof(float)); + if (!out) return NULL; + + if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } return out; } diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index a1544ef..e528367 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -28,6 +28,23 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx); */ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); +/* + * Compute one embedding into caller-provided out[hidden_size]. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding); + +/* + * Compute a true padded dense batch on MLX. + * + * out_embeddings is caller-provided [batch, hidden_size]. + * Padded tokens are masked out of attention keys and mean pooling. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings); + /* Get the config (so main.c can read hidden_size etc.) */ const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index 0ea3cc6..bc93f8a 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -1135,6 +1135,101 @@ void qwen_causal_attention(float *out, const float *Q, const float *K, const flo head_dim, scale, q_offset, 0, n_heads); } +static void qwen_bidirectional_gqa_attention_packed_heads( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, int head_start, int head_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + + for (int h = head_start; h < head_end; h++) { + int kv_h = h / heads_per_kv; + + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + int end = offsets[b + 1]; + + for (int i = start; i < end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } + } + } +} + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int chunk = (t->n_heads + n_threads - 1) / n_threads; + int h0 = tid * chunk; + int h1 = h0 + chunk; + if (h1 > t->n_heads) h1 = t->n_heads; + if (h0 >= h1) return; + + qwen_bidirectional_gqa_attention_packed_heads( + t->out, t->Q, t->K, t->V, t->offsets, t->batch, + t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h0, h1); +} + +void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale) { + if (tp.n_threads > 1 && n_heads >= 2 && batch > 0) { + packed_gqa_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, + .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale + }; + parallel_for(packed_gqa_attn_worker, &task); + return; + } + + qwen_bidirectional_gqa_attention_packed_heads( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, 0, n_heads); +} + /* ======================================================================== * Position Embeddings * ======================================================================== */ diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index c59aeff..ce5e738 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -124,6 +124,22 @@ void qwen_causal_attention(float *out, const float *Q, const float *K, const flo int seq_q, int seq_k, int n_heads, int n_kv_heads, int head_dim, float scale, int q_offset); +/* + * Bidirectional GQA attention over a packed/ragged batch. + * + * Q: [total_seq, n_heads * head_dim] + * K,V: [total_seq, n_kv_heads * head_dim] + * offsets: [batch + 1], token spans for each sequence. + * + * Attention is block-diagonal: tokens only attend within their own + * offsets[b]..offsets[b+1] span. + */ +void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale); + /* ======================================================================== * Position Embeddings * ======================================================================== */ |
