diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 18:18:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 18:18:00 +0200 |
| commit | 3889d3f43c9d52efe97ce9422ecfcabc775cc661 (patch) | |
| tree | d5125e3a1b851a3da9969374d718212d6f2b6af1 /src | |
| parent | 3a88e6f18243b399979fe03ea6e1d51d931cabfe (diff) | |
| download | ffwd-3889d3f43c9d52efe97ce9422ecfcabc775cc661.tar.gz ffwd-3889d3f43c9d52efe97ce9422ecfcabc775cc661.zip | |
server: extract output encoding and shared request structs into server_encode.c/server_internal.h
Diffstat (limited to 'src')
| -rw-r--r-- | src/server.c | 196 | ||||
| -rw-r--r-- | src/server_encode.c | 156 | ||||
| -rw-r--r-- | src/server_internal.h | 71 |
3 files changed, 227 insertions, 196 deletions
diff --git a/src/server.c b/src/server.c index beab27f..9c439f5 100644 --- a/src/server.c +++ b/src/server.c @@ -779,107 +779,6 @@ static void job_set_timing_header(job *j) { j->extra_headers = xstrdup(buf); } -static signed char quantize_int8_tanh(float x) { - int v = (int)lrintf(tanhf(x) * 127.0f); - if (v > 127) - v = 127; - if (v < -128) - v = -128; - return (signed char)v; -} - -static char *encode_embedding(const float *emb, int dims, const char *encoding) { - if (!strcmp(encoding, "base64")) { - /* OpenAI/DashScope (Qwen3): base64 of the raw little-endian float32 - * vector. Copy through a byte buffer instead of reinterpret-casting the - * float pointer; x86_64 and aarch64 are little-endian, so the bytes - * match the wire format with no swap. */ - size_t nbytes = (size_t)dims * sizeof(float); - unsigned char *buf = xmalloc(nbytes); - memcpy(buf, emb, nbytes); - char *out = base64_encode(buf, nbytes); - free(buf); - return out; - } - if (!strcmp(encoding, "base64_binary")) { - size_t bytes = (size_t)dims / 8; - unsigned char *bits = xcalloc(bytes, 1); - for (int i = 0; i < dims; i++) { - if (emb[i] >= 0.0f) - bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); - } - char *out = base64_encode(bits, bytes); - free(bits); - return out; - } - - signed char *q = xmalloc((size_t)dims); - for (int i = 0; i < dims; i++) - q[i] = quantize_int8_tanh(emb[i]); - char *out = base64_encode((const unsigned char *)q, (size_t)dims); - free(q); - return out; -} - -typedef struct { - int *ids; - int n_tokens; -} token_buf; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - int n_inputs; - token_buf *tokens; - embed_input_t *inputs; - int total_tokens; - int ready; -} embedding_request; - -typedef struct { - int *ids; - int n_tokens; - embed_span_t *spans; - int n_spans; -} contextual_doc; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - contextual_doc *docs; - int n_docs; - int total_chunks; - int total_tokens; - int ready; -} contextual_request; - -typedef struct { - int *ids; - int n_tokens; - int *keep; - int n_keep; -} late_tokens; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - late_tokens query; - late_tokens *documents; - int n_documents; - int top_n; - int return_documents; - int query_tokens; - int document_tokens; - int ready; -} rerank_request; - static void free_token_bufs(token_buf *t, int n) { if (!t) return; @@ -1096,79 +995,6 @@ static int inference_batch_accepts_input( return next_tokens <= max_batch_tokens - packed_tokens; } -static void append_embedding_object(sbuf *b, int index, const char *embedding) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); - sbuf_puts(b, embedding); - sbuf_puts(b, "\"}"); -} - -/* Emit one embedding into the response, following the model's API family. - * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope - * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every - * other encoding renders a base64 string via encode_embedding(). */ -static void append_embedding_value( - sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { - if (!strcmp(encoding, "float")) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); - for (int i = 0; i < dims; i++) { - if (i) - sbuf_putc(b, ','); - float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f; - sbuf_printf(b, "%.9g", (double)v); - } - sbuf_puts(b, "]}"); - return; - } - char *encoded = encode_embedding(emb, dims, encoding); - append_embedding_object(b, index, encoded); - free(encoded); -} - -static void set_response_from_buf(job *j, sbuf *b) { - j->status = 200; - j->content_type = "application/json"; - j->response = b->ptr; - j->response_len = b->len; - memset(b, 0, sizeof(*b)); -} - -static int render_embedding_response(embedding_request *r, const float *embs) { - const int full_dim = r->model->info->dim; - const float *render_embs = embs; - float *truncated = NULL; - if (r->model->renormalize_truncated && r->dims < full_dim) { - truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated)); - for (int i = 0; i < r->n_inputs; i++) { - float *dst = truncated + (size_t)i * r->dims; - memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); - if (embed_l2_normalize(dst, r->dims) != 0) { - free(truncated); - return -1; - } - } - render_embs = truncated; - } - int render_stride = truncated ? r->dims : full_dim; - embedding_api_t api = r->model->info->api; - - sbuf b = {0}; - sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); - for (int i = 0; i < r->n_inputs; i++) { - if (i) - sbuf_putc(&b, ','); - append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, - api); - } - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); - set_response_from_buf(r->j, &b); - - free(truncated); - return 0; -} - static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); @@ -1275,28 +1101,6 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) free(items); } -static void render_contextual_response(contextual_request *r, const float *embs) { - sbuf b = {0}; - sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); - int span_offset = 0; - for (int di = 0; di < r->n_docs; di++) { - if (di) - sbuf_putc(&b, ','); - sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); - for (int ci = 0; ci < r->docs[di].n_spans; ci++) { - if (ci) - sbuf_putc(&b, ','); - const float *emb = embs + (size_t)span_offset * r->model->info->dim; - append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api); - span_offset++; - } - sbuf_puts(&b, "]}"); - } - sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); - set_response_from_buf(r->j, &b); -} - static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); diff --git a/src/server_encode.c b/src/server_encode.c new file mode 100644 index 0000000..7e86a74 --- /dev/null +++ b/src/server_encode.c @@ -0,0 +1,156 @@ +#include "server_internal.h" +#include "server_util.h" +#include "sbuf.h" +#include "base64.h" +#include "embed.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +/* ======================================================================== + * Embedding output encoding + * + * Turns a pooled float vector into the wire form the model's API family wants: + * base64 of int8 (tanh-quantized, Perplexity default), base64 of a binary sign + * bitmap, base64 of little-endian float32, or a plain JSON float array. The + * render_* functions assemble the full OpenAI-style response object from a + * prepared request and its computed embeddings. + * ======================================================================== */ + +signed char quantize_int8_tanh(float x) { + int v = (int)lrintf(tanhf(x) * 127.0f); + if (v > 127) + v = 127; + if (v < -128) + v = -128; + return (signed char)v; +} + +char *encode_embedding(const float *emb, int dims, const char *encoding) { + if (!strcmp(encoding, "base64")) { + /* OpenAI/DashScope (Qwen3): base64 of the raw little-endian float32 + * vector. Copy through a byte buffer instead of reinterpret-casting the + * float pointer; x86_64 and aarch64 are little-endian, so the bytes + * match the wire format with no swap. */ + size_t nbytes = (size_t)dims * sizeof(float); + unsigned char *buf = xmalloc(nbytes); + memcpy(buf, emb, nbytes); + char *out = base64_encode(buf, nbytes); + free(buf); + return out; + } + if (!strcmp(encoding, "base64_binary")) { + size_t bytes = (size_t)dims / 8; + unsigned char *bits = xcalloc(bytes, 1); + for (int i = 0; i < dims; i++) { + if (emb[i] >= 0.0f) + bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); + } + char *out = base64_encode(bits, bytes); + free(bits); + return out; + } + + signed char *q = xmalloc((size_t)dims); + for (int i = 0; i < dims; i++) + q[i] = quantize_int8_tanh(emb[i]); + char *out = base64_encode((const unsigned char *)q, (size_t)dims); + free(q); + return out; +} + +static void append_embedding_object(sbuf *b, int index, const char *embedding) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); + sbuf_puts(b, embedding); + sbuf_puts(b, "\"}"); +} + +/* Emit one embedding into the response, following the model's API family. + * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope + * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every + * other encoding renders a base64 string via encode_embedding(). */ +void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { + if (!strcmp(encoding, "float")) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); + for (int i = 0; i < dims; i++) { + if (i) + sbuf_putc(b, ','); + float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f; + sbuf_printf(b, "%.9g", (double)v); + } + sbuf_puts(b, "]}"); + return; + } + char *encoded = encode_embedding(emb, dims, encoding); + append_embedding_object(b, index, encoded); + free(encoded); +} + +void set_response_from_buf(job *j, sbuf *b) { + j->status = 200; + j->content_type = "application/json"; + j->response = b->ptr; + j->response_len = b->len; + memset(b, 0, sizeof(*b)); +} + +int render_embedding_response(embedding_request *r, const float *embs) { + const int full_dim = r->model->info->dim; + const float *render_embs = embs; + float *truncated = NULL; + if (r->model->renormalize_truncated && r->dims < full_dim) { + truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated)); + for (int i = 0; i < r->n_inputs; i++) { + float *dst = truncated + (size_t)i * r->dims; + memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); + if (embed_l2_normalize(dst, r->dims) != 0) { + free(truncated); + return -1; + } + } + render_embs = truncated; + } + int render_stride = truncated ? r->dims : full_dim; + embedding_api_t api = r->model->info->api; + + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + for (int i = 0; i < r->n_inputs; i++) { + if (i) + sbuf_putc(&b, ','); + append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, + api); + } + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); + + free(truncated); + return 0; +} + +void render_contextual_response(contextual_request *r, const float *embs) { + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + int span_offset = 0; + for (int di = 0; di < r->n_docs; di++) { + if (di) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); + for (int ci = 0; ci < r->docs[di].n_spans; ci++) { + if (ci) + sbuf_putc(&b, ','); + const float *emb = embs + (size_t)span_offset * r->model->info->dim; + append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api); + span_offset++; + } + sbuf_puts(&b, "]}"); + } + sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); +} diff --git a/src/server_internal.h b/src/server_internal.h index 9de8ca8..b98890a 100644 --- a/src/server_internal.h +++ b/src/server_internal.h @@ -234,6 +234,68 @@ struct job { job *next; }; +/* Parsed request state, shared by the prepare (handlers), execute (schedule), + * and render (encode) stages. Each carries its source cJSON root (freed when + * the request is freed), the resolved model, and the tokenized inputs. */ +typedef struct { + int *ids; + int n_tokens; +} token_buf; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + int n_inputs; + token_buf *tokens; + embed_input_t *inputs; + int total_tokens; + int ready; +} embedding_request; + +typedef struct { + int *ids; + int n_tokens; + embed_span_t *spans; + int n_spans; +} contextual_doc; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + contextual_doc *docs; + int n_docs; + int total_chunks; + int total_tokens; + int ready; +} contextual_request; + +typedef struct { + int *ids; + int n_tokens; + int *keep; + int n_keep; +} late_tokens; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + late_tokens query; + late_tokens *documents; + int n_documents; + int top_n; + int return_documents; + int query_tokens; + int document_tokens; + int ready; +} rerank_request; + /* ---- server_json.c: request validation and error bodies ---- */ void append_json_string(sbuf *b, const char *s); char *json_error_body(const char *message, const char *type, size_t *len); @@ -247,4 +309,13 @@ int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct); int dimensions_from_root( cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); +/* ---- server_encode.c: embedding output encoding ---- */ +signed char quantize_int8_tanh(float x); +char *encode_embedding(const float *emb, int dims, const char *encoding); +void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api); +void set_response_from_buf(job *j, sbuf *b); +int render_embedding_response(embedding_request *r, const float *embs); +void render_contextual_response(contextual_request *r, const float *embs); + #endif /* EMBED_SERVER_INTERNAL_H */ |
