aboutsummaryrefslogtreecommitdiff
path: root/pplx_embed.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-20 05:26:35 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:17:30 +0200
commit7a88087ba0b5b1b388234f3d2a907673c63bd8cb (patch)
tree3a656367ab23d65d625c9cbbb3014ed109a2ebd2 /pplx_embed.c
parent47859fbfdc02329090c59357cbb826b42c946bf7 (diff)
downloadffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.tar.gz
ffwd-7a88087ba0b5b1b388234f3d2a907673c63bd8cb.zip
all: support batched embedding attention
Diffstat (limited to 'pplx_embed.c')
-rw-r--r--pplx_embed.c261
1 files changed, 180 insertions, 81 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index dcd1acd..942cfa9 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -316,28 +316,54 @@ void pplx_free(pplx_ctx_t *ctx)
}
/* ========================================================================
- * Forward pass (returns full token-level embeddings)
+ * Packed forward pass
* ======================================================================== */
-float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+static int build_offsets(const pplx_input_t *inputs, int batch,
+ int *offsets, int *total_out, int *max_out)
{
- if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+ if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
+ return -1;
+
+ int total = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return -1;
+ if (total > INT32_MAX - inputs[b].n_tokens)
+ return -1;
+ offsets[b] = total;
+ total += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
+ }
+ offsets[batch] = total;
+ *total_out = total;
+ *max_out = max_seq;
+ return 0;
+}
+
+static int forward_packed_inplace(pplx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, const int *offsets,
+ int total_seq, int max_seq)
+{
+ if (!ctx || !inputs || batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0)
+ return -1;
const pplx_config_t *cfg = &ctx->config;
const pplx_weights_t *w = &ctx->weights;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
int n_kv_heads = cfg->n_kv_heads;
- int head_dim = cfg->head_dim;
- int q_dim = cfg->q_dim;
- int kv_dim = cfg->kv_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->rms_norm_eps;
- int seq = n_tokens;
+ int head_dim = cfg->head_dim;
+ int q_dim = cfg->q_dim;
+ int kv_dim = cfg->kv_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->rms_norm_eps;
- if (ensure_buffers(ctx, seq) != 0) return NULL;
- if (ensure_rope_cache(ctx, seq) != 0) return NULL;
+ if (ensure_buffers(ctx, total_seq) != 0) return -1;
+ if (ensure_rope_cache(ctx, max_seq) != 0) return -1;
const float *rope_cos = ctx->rope_cos;
const float *rope_sin = ctx->rope_sin;
@@ -353,106 +379,179 @@ float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
float *ffn_up = ctx->ffn_up;
float *ffn_out = ctx->ffn_out;
- /* 1. Token embedding lookup */
- for (int i = 0; i < seq; i++) {
- int id = token_ids[i];
- if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "pplx: invalid token id %d at position %d\n", id, i);
- return NULL;
+ /* 1. Token embedding lookup into packed [total_seq, hidden]. */
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ for (int i = 0; i < inputs[b].n_tokens; i++) {
+ int id = inputs[b].ids[i];
+ if (id < 0 || id >= cfg->vocab_size) {
+ fprintf(stderr, "pplx: invalid token id %d at input %d position %d\n",
+ id, b, i);
+ return -1;
+ }
+ memcpy(x + (size_t)(start + i) * hidden,
+ w->embed_tokens + (size_t)id * hidden,
+ (size_t)hidden * sizeof(float));
}
- memcpy(x + i * hidden,
- w->embed_tokens + (size_t)id * hidden,
- hidden * sizeof(float));
}
float scale = 1.0f / sqrtf((float)head_dim);
- /* 2. Transformer layers */
+ /* 2. Transformer layers. Linear/MLP ops run over all packed tokens;
+ * attention is explicitly block-diagonal by sequence offsets. */
for (int layer = 0; layer < cfg->n_layers; layer++) {
const pplx_layer_t *l = &w->layers[layer];
- /* Input RMSNorm */
- qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps);
+ qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- /* QKV projections */
- qwen_linear_nobias(q_buf, x_norm, l->wq, seq, hidden, q_dim);
- qwen_linear_nobias(k_buf, x_norm, l->wk, seq, hidden, kv_dim);
- qwen_linear_nobias(v_buf, x_norm, l->wv, seq, hidden, kv_dim);
+ qwen_linear_nobias(q_buf, x_norm, l->wq, total_seq, hidden, q_dim);
+ qwen_linear_nobias(k_buf, x_norm, l->wk, total_seq, hidden, kv_dim);
+ qwen_linear_nobias(v_buf, x_norm, l->wv, total_seq, hidden, kv_dim);
- /* Per-head Q/K RMSNorm */
- qwen_rms_norm_per_head(q_buf, l->q_norm, seq, n_heads, head_dim, eps);
- qwen_rms_norm_per_head(k_buf, l->k_norm, seq, n_kv_heads, head_dim, eps);
+ qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
- /* RoPE */
- qwen_apply_rope_neox(q_buf, rope_cos, rope_sin, seq, n_heads, head_dim);
- qwen_apply_rope_neox(k_buf, rope_cos, rope_sin, seq, n_kv_heads, head_dim);
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ int len = offsets[b + 1] - start;
+ qwen_apply_rope_neox(q_buf + (size_t)start * q_dim,
+ rope_cos, rope_sin, len, n_heads, head_dim);
+ qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim,
+ rope_cos, rope_sin, len, n_kv_heads, head_dim);
+ }
- /* Bidirectional GQA attention (q_offset = seq-1 removes causal mask) */
- qwen_causal_attention(attn_out, q_buf, k_buf, v_buf,
- seq, seq, n_heads, n_kv_heads,
- head_dim, scale, seq - 1);
+ qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf,
+ offsets, batch, n_heads,
+ n_kv_heads, head_dim, scale);
- /* Output projection + residual */
- qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden);
- qwen_add_inplace(x, proj_out, seq * hidden);
+ qwen_linear_nobias(proj_out, attn_out, l->wo, total_seq, q_dim, hidden);
+ qwen_add_inplace(x, proj_out, total_seq * hidden);
- /* Post-attention RMSNorm */
- qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps);
+ qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- /* SwiGLU MLP */
- qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter);
- qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter);
- qwen_silu(ffn_gate, seq * inter);
- qwen_mul_inplace(ffn_gate, ffn_up, seq * inter);
- qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden);
- qwen_add_inplace(x, ffn_out, seq * hidden);
+ qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, total_seq, hidden, inter);
+ qwen_linear_nobias(ffn_up, x_norm, l->up_proj, total_seq, hidden, inter);
+ qwen_silu(ffn_gate, total_seq * inter);
+ qwen_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
+ qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, total_seq, inter, hidden);
+ qwen_add_inplace(x, ffn_out, total_seq * hidden);
}
- /* 3. Final RMSNorm (in-place) */
- qwen_rms_norm(x, x, w->norm, seq, hidden, eps);
+ qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps);
+ return 0;
+}
- /* Copy result out (caller frees) */
- size_t out_size = (size_t)seq * hidden * sizeof(float);
- float *out = (float *)malloc(out_size);
- if (out) memcpy(out, x, out_size);
- return out;
+static void pool_embeddings(const pplx_ctx_t *ctx, const int *offsets,
+ int batch, float *out_embeddings)
+{
+ int hidden = ctx->config.hidden_size;
+ const float *x = ctx->x;
+
+ for (int b = 0; b < batch; b++) {
+ int start = offsets[b];
+ int end = offsets[b + 1];
+ int len = end - start;
+ float *emb = out_embeddings + (size_t)b * hidden;
+
+ memset(emb, 0, (size_t)hidden * sizeof(float));
+ for (int i = start; i < end; i++) {
+ const float *row = x + (size_t)i * hidden;
+ for (int d = 0; d < hidden; d++)
+ emb[d] += row[d];
+ }
+
+ float inv_len = 1.0f / (float)len;
+ for (int d = 0; d < hidden; d++)
+ emb[d] *= inv_len;
+
+ float norm_sq = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ norm_sq += emb[d] * emb[d];
+ float nv = sqrtf(norm_sq);
+ if (nv > 1e-8f) {
+ float inv = 1.0f / nv;
+ for (int d = 0; d < hidden; d++)
+ emb[d] *= inv;
+ }
+ }
}
/* ========================================================================
- * Embed (forward + mean pool + L2 norm)
+ * Public forward/embed APIs
* ======================================================================== */
-float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+int pplx_forward_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_states)
{
- float *all = pplx_forward(ctx, token_ids, n_tokens);
- if (!all) return NULL;
+ if (!ctx || !token_ids || n_tokens <= 0 || !out_states) return -1;
- int hidden = ctx->config.hidden_size;
- int seq = n_tokens;
+ pplx_input_t input = { token_ids, n_tokens };
+ int offsets[2] = { 0, n_tokens };
+ if (forward_packed_inplace(ctx, &input, 1, offsets, n_tokens, n_tokens) != 0)
+ return -1;
- /* Mean pool */
- float *emb = (float *)calloc(hidden, sizeof(float));
- if (!emb) { free(all); return NULL; }
+ size_t out_size = (size_t)n_tokens * ctx->config.hidden_size * sizeof(float);
+ memcpy(out_states, ctx->x, out_size);
+ return 0;
+}
- for (int i = 0; i < seq; i++) {
- const float *row = all + i * hidden;
- for (int d = 0; d < hidden; d++)
- emb[d] += row[d];
+float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ size_t n = (size_t)n_tokens * ctx->config.hidden_size;
+ float *out = (float *)malloc(n * sizeof(float));
+ if (!out) return NULL;
+
+ if (pplx_forward_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
}
- free(all);
+ return out;
+}
- float inv_seq = 1.0f / (float)seq;
- for (int d = 0; d < hidden; d++) emb[d] *= inv_seq;
+int pplx_embed_batch(pplx_ctx_t *ctx, const pplx_input_t *inputs, int batch,
+ float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+
+ int *offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!offsets) return -1;
- /* L2 normalize */
- float norm_sq = 0.0f;
- for (int d = 0; d < hidden; d++) norm_sq += emb[d] * emb[d];
- float nv = sqrtf(norm_sq);
- if (nv > 1e-8f) {
- float inv = 1.0f / nv;
- for (int d = 0; d < hidden; d++) emb[d] *= inv;
+ int total_seq = 0;
+ int max_seq = 0;
+ if (build_offsets(inputs, batch, offsets, &total_seq, &max_seq) != 0) {
+ free(offsets);
+ return -1;
}
+ int rc = forward_packed_inplace(ctx, inputs, batch, offsets, total_seq, max_seq);
+ if (rc == 0)
+ pool_embeddings(ctx, offsets, batch, out_embeddings);
+
+ free(offsets);
+ return rc;
+}
+
+int pplx_embed_into(pplx_ctx_t *ctx, const int *token_ids, int n_tokens,
+ float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ int hidden = ctx->config.hidden_size;
+ float *emb = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!emb) return NULL;
+
+ if (pplx_embed_into(ctx, token_ids, n_tokens, emb) != 0) {
+ free(emb);
+ return NULL;
+ }
return emb;
}