diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-30 05:26:44 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:59 +0200 |
| commit | 385bed06e606f9cbacb719c9569cc77e4000f12c (patch) | |
| tree | 89da34bae34d89f467e4a6e3cae01354d710dd69 | |
| parent | aba76ae05435492b8ac3fc477ce13356fda6ffde (diff) | |
| download | ffwd-385bed06e606f9cbacb719c9569cc77e4000f12c.tar.gz ffwd-385bed06e606f9cbacb719c9569cc77e4000f12c.zip | |
server: align embedding quantization with perplexity
| -rw-r--r-- | pplx_embed.c | 28 | ||||
| -rw-r--r-- | pplx_embed.h | 13 | ||||
| -rw-r--r-- | pplx_embed_mlx.c | 14 | ||||
| -rw-r--r-- | pplx_server.c | 24 |
4 files changed, 28 insertions, 51 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index e4ea605..9c02fa9 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -889,10 +889,10 @@ static int forward_packed_inplace(const pplx_model_t *model, return 0; } -static void pool_normalized_embeddings(const pplx_model_t *model, - const pplx_workspace_t *ws, - const int *offsets, int batch, - float *out_embeddings) +static void pool_embeddings(const pplx_model_t *model, + const pplx_workspace_t *ws, + const int *offsets, int batch, + float *out_embeddings) { const pplx_config_t *cfg = &model->config; int hidden = cfg->hidden_size; @@ -923,15 +923,6 @@ static void pool_normalized_embeddings(const pplx_model_t *model, for (int d = 0; d < hidden; d++) emb[d] *= inv_len; - float norm_sq = 0.0f; - for (int d = 0; d < hidden; d++) - norm_sq += emb[d] * emb[d]; - float nv = sqrtf(norm_sq); - if (nv > 1e-8f) { - float inv = 1.0f / nv; - for (int d = 0; d < hidden; d++) - emb[d] *= inv; - } } } @@ -988,17 +979,8 @@ int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, } float inv_len = 1.0f / (float)len; - float norm_sq = 0.0f; for (int d = 0; d < hidden; d++) { emb[d] *= inv_len; - norm_sq += emb[d] * emb[d]; - } - - float nv = sqrtf(norm_sq); - if (nv > 1e-8f) { - float inv = 1.0f / nv; - for (int d = 0; d < hidden; d++) - emb[d] *= inv; } } return 0; @@ -1058,7 +1040,7 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); if (rc == 0) - pool_normalized_embeddings(model, ws, offsets, batch, out_embeddings); + pool_embeddings(model, ws, offsets, batch, out_embeddings); return rc; } diff --git a/pplx_embed.h b/pplx_embed.h index ba46639..276c67f 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -94,7 +94,10 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws); * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU) * 3. Final RMSNorm * 4. Mean pooling over all positions - * 5. L2 normalization + * + * Perplexity embeddings are intentionally not L2-normalized. Use cosine + * similarity for comparisons, or normalize explicitly before storing in vector + * databases that only support inner product. * * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ @@ -142,11 +145,11 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, float *out_states); /* - * Pool normalized embeddings from final hidden states. + * Pool embeddings from final hidden states. * * states is [n_tokens, hidden_size], normally produced by - * pplx_model_forward_into(). Each span is mean pooled and L2 normalized into - * out_embeddings[n_spans, hidden_size]. + * pplx_model_forward_into(). Each span is mean pooled into + * out_embeddings[n_spans, hidden_size] without L2 normalization. */ int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, const pplx_span_t *spans, int n_spans, @@ -162,7 +165,7 @@ int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, float *out_embeddings); /* - * Cosine similarity between two L2-normalized vectors. + * Cosine similarity between two vectors. */ float pplx_cosine_similarity(const float *a, const float *b, int dim); diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 305d235..dd17d17 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -452,19 +452,13 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, mlx_divide(&emb, emb_sum, lengths, S); mlx_array_free(emb_sum); mlx_array_free(lengths); - /* 5. L2 normalize each row. */ - int norm_axes[] = {1}; - mlx_array nv = mlx_array_new(); - mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S); - mlx_array emb_n = mlx_array_new(); - mlx_divide(&emb_n, emb, nv, S); - mlx_array_free(emb); mlx_array_free(nv); if (has_padding) mlx_array_free(attn_mask); - /* 6. Eval and copy [B, hidden] to CPU. */ + /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are + * intentionally unnormalized; callers should use cosine similarity. */ mlx_array emb_f32 = mlx_array_new(); - mlx_astype(&emb_f32, emb_n, MLX_FLOAT32, S); - mlx_array_free(emb_n); + mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); + mlx_array_free(emb); mlx_array_eval(emb_f32); const float *data = mlx_array_data_float32(emb_f32); diff --git a/pplx_server.c b/pplx_server.c index 0258ae2..08a8dd2 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -875,6 +875,13 @@ static char *base64_encode(const unsigned char *src, size_t n) { return out; } +static signed char quantize_int8_tanh(float x) { + int v = (int)lrintf(tanhf(x) * 127.0f); + if (v > 127) v = 127; + if (v < -128) v = -128; + return (signed char)v; +} + static char *encode_embedding(const float *emb, int dims, const char *encoding) { if (!strcmp(encoding, "base64_binary")) { size_t bytes = (size_t)dims / 8; @@ -889,18 +896,8 @@ static char *encode_embedding(const float *emb, int dims, const char *encoding) } signed char *q = xmalloc((size_t)dims); - float max_abs = 0.0f; - for (int i = 0; i < dims; i++) { - float a = fabsf(emb[i]); - if (a > max_abs) max_abs = a; - } - float scale = max_abs > 1e-12f ? 127.0f / max_abs : 0.0f; - for (int i = 0; i < dims; i++) { - int v = (int)lrintf(emb[i] * scale); - if (v > 127) v = 127; - if (v < -127) v = -127; - q[i] = (signed char)v; - } + for (int i = 0; i < dims; i++) + q[i] = quantize_int8_tanh(emb[i]); char *out = base64_encode((const unsigned char *)q, (size_t)dims); free(q); return out; @@ -979,7 +976,8 @@ static void append_float_embedding_object(sbuf *b, int index, sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); for (int i = 0; i < dims; i++) { if (i) sbuf_putc(b, ','); - sbuf_printf(b, "%.9g", (double)emb[i]); + float v = (float)quantize_int8_tanh(emb[i]) / 128.0f; + sbuf_printf(b, "%.9g", (double)v); } sbuf_puts(b, "]}"); } |
