diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-22 00:09:18 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:00 +0200 |
| commit | f62e7176e6321f21c3a34cfe2064b07c47e19fc3 (patch) | |
| tree | c7475061ac164c08eab0475807c403524720d32a | |
| parent | 216d0547db595ab1d149be4cd1dfc4c336abdedc (diff) | |
| download | ffwd-f62e7176e6321f21c3a34cfe2064b07c47e19fc3.tar.gz ffwd-f62e7176e6321f21c3a34cfe2064b07c47e19fc3.zip | |
cpu: fuse CPU embedding hot paths
| -rw-r--r-- | pplx_embed.c | 34 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 26 | ||||
| -rw-r--r-- | qwen_asr_kernels.h | 2 |
3 files changed, 51 insertions, 11 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index a117b10..19cacf1 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -550,7 +550,8 @@ static int forward_packed_inplace(const pplx_model_t *model, pplx_workspace_t *ws, const pplx_input_t *inputs, int batch, const int *offsets, - int total_seq, int max_seq) + int total_seq, int max_seq, + int apply_final_norm) { if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0) @@ -637,20 +638,25 @@ static int forward_packed_inplace(const pplx_model_t *model, qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, total_seq, hidden, inter); qwen_linear_nobias(ffn_up, x_norm, l->up_proj, total_seq, hidden, inter); - qwen_silu(ffn_gate, total_seq * inter); - qwen_mul_inplace(ffn_gate, ffn_up, total_seq * inter); + qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, total_seq, inter, hidden); qwen_add_inplace(x, ffn_out, total_seq * hidden); } - qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); + if (apply_final_norm) + qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); return 0; } -static void pool_embeddings(const pplx_model_t *model, const pplx_workspace_t *ws, - const int *offsets, int batch, float *out_embeddings) +static void pool_normalized_embeddings(const pplx_model_t *model, + const pplx_workspace_t *ws, + const int *offsets, int batch, + float *out_embeddings) { - int hidden = model->config.hidden_size; + const pplx_config_t *cfg = &model->config; + int hidden = cfg->hidden_size; + float eps = cfg->rms_norm_eps; + const float *norm_weight = model->weights.norm; const float *x = ws->x; for (int b = 0; b < batch; b++) { @@ -662,8 +668,14 @@ static void pool_embeddings(const pplx_model_t *model, const pplx_workspace_t *w memset(emb, 0, (size_t)hidden * sizeof(float)); for (int i = start; i < end; i++) { const float *row = x + (size_t)i * hidden; + + float sum_sq = 0.0f; + for (int d = 0; d < hidden; d++) + sum_sq += row[d] * row[d]; + + float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); for (int d = 0; d < hidden; d++) - emb[d] += row[d]; + emb[d] += row[d] * rms_inv * norm_weight[d]; } float inv_len = 1.0f / (float)len; @@ -701,7 +713,7 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, pplx_input_t input = { token_ids, n_tokens }; int offsets[2] = { 0, n_tokens }; if (forward_packed_inplace(model, ws, &input, 1, offsets, - n_tokens, n_tokens) != 0) + n_tokens, n_tokens, 1) != 0) return -1; memcpy(out_states, ws->x, bytes); @@ -747,9 +759,9 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, } int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq); + total_seq, max_seq, 0); if (rc == 0) - pool_embeddings(model, ws, offsets, batch, out_embeddings); + pool_normalized_embeddings(model, ws, offsets, batch, out_embeddings); free(offsets); return rc; diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index 70ead7b..ec2ce9e 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -887,6 +887,32 @@ void qwen_gelu(float *x, int n) { } } +void qwen_silu_mul_inplace(float *gate, const float *up, int n) { +#if defined(__APPLE__) && defined(USE_BLAS) + enum { CHUNK = 4096 }; + float exp_neg[CHUNK]; + + for (int base = 0; base < n; base += CHUNK) { + int len = n - base; + if (len > CHUNK) len = CHUNK; + + for (int i = 0; i < len; i++) + exp_neg[i] = -gate[base + i]; + vvexpf(exp_neg, exp_neg, &len); + + for (int i = 0; i < len; i++) { + float g = gate[base + i]; + gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i]; + } + } +#else + for (int i = 0; i < n; i++) { + float g = gate[i]; + gate[i] = (g / (1.0f + expf(-g))) * up[i]; + } +#endif +} + typedef struct { float *out; const float *gate_up; diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index ce5e738..7e0a0f0 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -93,6 +93,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight, void qwen_silu(float *x, int n); void qwen_gelu(float *x, int n); void qwen_softmax(float *x, int rows, int cols); +/* gate = SiLU(gate) * up */ +void qwen_silu_mul_inplace(float *gate, const float *up, int n); /* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */ void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate); |
