aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-30 05:26:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:59 +0200
commit385bed06e606f9cbacb719c9569cc77e4000f12c (patch)
tree89da34bae34d89f467e4a6e3cae01354d710dd69
parentaba76ae05435492b8ac3fc477ce13356fda6ffde (diff)
downloadffwd-385bed06e606f9cbacb719c9569cc77e4000f12c.tar.gz
ffwd-385bed06e606f9cbacb719c9569cc77e4000f12c.zip
server: align embedding quantization with perplexity
-rw-r--r--pplx_embed.c28
-rw-r--r--pplx_embed.h13
-rw-r--r--pplx_embed_mlx.c14
-rw-r--r--pplx_server.c24
4 files changed, 28 insertions, 51 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index e4ea605..9c02fa9 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -889,10 +889,10 @@ static int forward_packed_inplace(const pplx_model_t *model,
return 0;
}
-static void pool_normalized_embeddings(const pplx_model_t *model,
- const pplx_workspace_t *ws,
- const int *offsets, int batch,
- float *out_embeddings)
+static void pool_embeddings(const pplx_model_t *model,
+ const pplx_workspace_t *ws,
+ const int *offsets, int batch,
+ float *out_embeddings)
{
const pplx_config_t *cfg = &model->config;
int hidden = cfg->hidden_size;
@@ -923,15 +923,6 @@ static void pool_normalized_embeddings(const pplx_model_t *model,
for (int d = 0; d < hidden; d++)
emb[d] *= inv_len;
- float norm_sq = 0.0f;
- for (int d = 0; d < hidden; d++)
- norm_sq += emb[d] * emb[d];
- float nv = sqrtf(norm_sq);
- if (nv > 1e-8f) {
- float inv = 1.0f / nv;
- for (int d = 0; d < hidden; d++)
- emb[d] *= inv;
- }
}
}
@@ -988,17 +979,8 @@ int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens,
}
float inv_len = 1.0f / (float)len;
- float norm_sq = 0.0f;
for (int d = 0; d < hidden; d++) {
emb[d] *= inv_len;
- norm_sq += emb[d] * emb[d];
- }
-
- float nv = sqrtf(norm_sq);
- if (nv > 1e-8f) {
- float inv = 1.0f / nv;
- for (int d = 0; d < hidden; d++)
- emb[d] *= inv;
}
}
return 0;
@@ -1058,7 +1040,7 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws,
int rc = forward_packed_inplace(model, ws, inputs, batch, offsets,
total_seq, max_seq, 0);
if (rc == 0)
- pool_normalized_embeddings(model, ws, offsets, batch, out_embeddings);
+ pool_embeddings(model, ws, offsets, batch, out_embeddings);
return rc;
}
diff --git a/pplx_embed.h b/pplx_embed.h
index ba46639..276c67f 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -94,7 +94,10 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws);
* 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU)
* 3. Final RMSNorm
* 4. Mean pooling over all positions
- * 5. L2 normalization
+ *
+ * Perplexity embeddings are intentionally not L2-normalized. Use cosine
+ * similarity for comparisons, or normalize explicitly before storing in vector
+ * databases that only support inner product.
*
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
@@ -142,11 +145,11 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws,
float *out_states);
/*
- * Pool normalized embeddings from final hidden states.
+ * Pool embeddings from final hidden states.
*
* states is [n_tokens, hidden_size], normally produced by
- * pplx_model_forward_into(). Each span is mean pooled and L2 normalized into
- * out_embeddings[n_spans, hidden_size].
+ * pplx_model_forward_into(). Each span is mean pooled into
+ * out_embeddings[n_spans, hidden_size] without L2 normalization.
*/
int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens,
const pplx_span_t *spans, int n_spans,
@@ -162,7 +165,7 @@ int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws,
float *out_embeddings);
/*
- * Cosine similarity between two L2-normalized vectors.
+ * Cosine similarity between two vectors.
*/
float pplx_cosine_similarity(const float *a, const float *b, int dim);
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 305d235..dd17d17 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -452,19 +452,13 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
mlx_divide(&emb, emb_sum, lengths, S);
mlx_array_free(emb_sum); mlx_array_free(lengths);
- /* 5. L2 normalize each row. */
- int norm_axes[] = {1};
- mlx_array nv = mlx_array_new();
- mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S);
- mlx_array emb_n = mlx_array_new();
- mlx_divide(&emb_n, emb, nv, S);
- mlx_array_free(emb); mlx_array_free(nv);
if (has_padding) mlx_array_free(attn_mask);
- /* 6. Eval and copy [B, hidden] to CPU. */
+ /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are
+ * intentionally unnormalized; callers should use cosine similarity. */
mlx_array emb_f32 = mlx_array_new();
- mlx_astype(&emb_f32, emb_n, MLX_FLOAT32, S);
- mlx_array_free(emb_n);
+ mlx_astype(&emb_f32, emb, MLX_FLOAT32, S);
+ mlx_array_free(emb);
mlx_array_eval(emb_f32);
const float *data = mlx_array_data_float32(emb_f32);
diff --git a/pplx_server.c b/pplx_server.c
index 0258ae2..08a8dd2 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -875,6 +875,13 @@ static char *base64_encode(const unsigned char *src, size_t n) {
return out;
}
+static signed char quantize_int8_tanh(float x) {
+ int v = (int)lrintf(tanhf(x) * 127.0f);
+ if (v > 127) v = 127;
+ if (v < -128) v = -128;
+ return (signed char)v;
+}
+
static char *encode_embedding(const float *emb, int dims, const char *encoding) {
if (!strcmp(encoding, "base64_binary")) {
size_t bytes = (size_t)dims / 8;
@@ -889,18 +896,8 @@ static char *encode_embedding(const float *emb, int dims, const char *encoding)
}
signed char *q = xmalloc((size_t)dims);
- float max_abs = 0.0f;
- for (int i = 0; i < dims; i++) {
- float a = fabsf(emb[i]);
- if (a > max_abs) max_abs = a;
- }
- float scale = max_abs > 1e-12f ? 127.0f / max_abs : 0.0f;
- for (int i = 0; i < dims; i++) {
- int v = (int)lrintf(emb[i] * scale);
- if (v > 127) v = 127;
- if (v < -127) v = -127;
- q[i] = (signed char)v;
- }
+ for (int i = 0; i < dims; i++)
+ q[i] = quantize_int8_tanh(emb[i]);
char *out = base64_encode((const unsigned char *)q, (size_t)dims);
free(q);
return out;
@@ -979,7 +976,8 @@ static void append_float_embedding_object(sbuf *b, int index,
sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index);
for (int i = 0; i < dims; i++) {
if (i) sbuf_putc(b, ',');
- sbuf_printf(b, "%.9g", (double)emb[i]);
+ float v = (float)quantize_int8_tanh(emb[i]) / 128.0f;
+ sbuf_printf(b, "%.9g", (double)v);
}
sbuf_puts(b, "]}");
}