aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-22 00:09:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:00 +0200
commitf62e7176e6321f21c3a34cfe2064b07c47e19fc3 (patch)
treec7475061ac164c08eab0475807c403524720d32a
parent216d0547db595ab1d149be4cd1dfc4c336abdedc (diff)
downloadffwd-f62e7176e6321f21c3a34cfe2064b07c47e19fc3.tar.gz
ffwd-f62e7176e6321f21c3a34cfe2064b07c47e19fc3.zip
cpu: fuse CPU embedding hot paths
-rw-r--r--pplx_embed.c34
-rw-r--r--qwen_asr_kernels.c26
-rw-r--r--qwen_asr_kernels.h2
3 files changed, 51 insertions, 11 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index a117b10..19cacf1 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -550,7 +550,8 @@ static int forward_packed_inplace(const pplx_model_t *model,
pplx_workspace_t *ws,
const pplx_input_t *inputs,
int batch, const int *offsets,
- int total_seq, int max_seq)
+ int total_seq, int max_seq,
+ int apply_final_norm)
{
if (!model || !ws || ws->model != model || !inputs ||
batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0)
@@ -637,20 +638,25 @@ static int forward_packed_inplace(const pplx_model_t *model,
qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, total_seq, hidden, inter);
qwen_linear_nobias(ffn_up, x_norm, l->up_proj, total_seq, hidden, inter);
- qwen_silu(ffn_gate, total_seq * inter);
- qwen_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
+ qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, total_seq, inter, hidden);
qwen_add_inplace(x, ffn_out, total_seq * hidden);
}
- qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps);
+ if (apply_final_norm)
+ qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps);
return 0;
}
-static void pool_embeddings(const pplx_model_t *model, const pplx_workspace_t *ws,
- const int *offsets, int batch, float *out_embeddings)
+static void pool_normalized_embeddings(const pplx_model_t *model,
+ const pplx_workspace_t *ws,
+ const int *offsets, int batch,
+ float *out_embeddings)
{
- int hidden = model->config.hidden_size;
+ const pplx_config_t *cfg = &model->config;
+ int hidden = cfg->hidden_size;
+ float eps = cfg->rms_norm_eps;
+ const float *norm_weight = model->weights.norm;
const float *x = ws->x;
for (int b = 0; b < batch; b++) {
@@ -662,8 +668,14 @@ static void pool_embeddings(const pplx_model_t *model, const pplx_workspace_t *w
memset(emb, 0, (size_t)hidden * sizeof(float));
for (int i = start; i < end; i++) {
const float *row = x + (size_t)i * hidden;
+
+ float sum_sq = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ sum_sq += row[d] * row[d];
+
+ float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps);
for (int d = 0; d < hidden; d++)
- emb[d] += row[d];
+ emb[d] += row[d] * rms_inv * norm_weight[d];
}
float inv_len = 1.0f / (float)len;
@@ -701,7 +713,7 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws,
pplx_input_t input = { token_ids, n_tokens };
int offsets[2] = { 0, n_tokens };
if (forward_packed_inplace(model, ws, &input, 1, offsets,
- n_tokens, n_tokens) != 0)
+ n_tokens, n_tokens, 1) != 0)
return -1;
memcpy(out_states, ws->x, bytes);
@@ -747,9 +759,9 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws,
}
int rc = forward_packed_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq);
+ total_seq, max_seq, 0);
if (rc == 0)
- pool_embeddings(model, ws, offsets, batch, out_embeddings);
+ pool_normalized_embeddings(model, ws, offsets, batch, out_embeddings);
free(offsets);
return rc;
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index 70ead7b..ec2ce9e 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -887,6 +887,32 @@ void qwen_gelu(float *x, int n) {
}
}
+void qwen_silu_mul_inplace(float *gate, const float *up, int n) {
+#if defined(__APPLE__) && defined(USE_BLAS)
+ enum { CHUNK = 4096 };
+ float exp_neg[CHUNK];
+
+ for (int base = 0; base < n; base += CHUNK) {
+ int len = n - base;
+ if (len > CHUNK) len = CHUNK;
+
+ for (int i = 0; i < len; i++)
+ exp_neg[i] = -gate[base + i];
+ vvexpf(exp_neg, exp_neg, &len);
+
+ for (int i = 0; i < len; i++) {
+ float g = gate[base + i];
+ gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i];
+ }
+ }
+#else
+ for (int i = 0; i < n; i++) {
+ float g = gate[i];
+ gate[i] = (g / (1.0f + expf(-g))) * up[i];
+ }
+#endif
+}
+
typedef struct {
float *out;
const float *gate_up;
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index ce5e738..7e0a0f0 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -93,6 +93,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight,
void qwen_silu(float *x, int n);
void qwen_gelu(float *x, int n);
void qwen_softmax(float *x, int rows, int cols);
+/* gate = SiLU(gate) * up */
+void qwen_silu_mul_inplace(float *gate, const float *up, int n);
/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */
void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate);