aboutsummaryrefslogtreecommitdiff
path: root/tools/server
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server')
-rw-r--r--tools/server/Makefile49
-rw-r--r--tools/server/base64.c47
-rw-r--r--tools/server/base64.h10
-rw-r--r--tools/server/encode.c152
-rw-r--r--tools/server/handlers.c800
-rw-r--r--tools/server/http.c399
-rw-r--r--tools/server/json.c217
-rw-r--r--tools/server/main.c342
-rw-r--r--tools/server/models.c342
-rw-r--r--tools/server/sbuf.c74
-rw-r--r--tools/server/sbuf.h24
-rw-r--r--tools/server/schedule.c341
-rw-r--r--tools/server/server.c414
-rw-r--r--tools/server/server.h57
-rw-r--r--tools/server/server_internal.h314
-rw-r--r--tools/server/test_base64.c51
-rw-r--r--tools/server/test_encode.c90
-rw-r--r--tools/server/test_json.c143
-rw-r--r--tools/server/test_models.c26
-rw-r--r--tools/server/test_sbuf.c35
-rw-r--r--tools/server/test_schedule.c73
-rw-r--r--tools/server/test_util.h26
-rw-r--r--tools/server/util.c82
-rw-r--r--tools/server/util.h36
24 files changed, 4144 insertions, 0 deletions
diff --git a/tools/server/Makefile b/tools/server/Makefile
new file mode 100644
index 0000000..4a11b0d
--- /dev/null
+++ b/tools/server/Makefile
@@ -0,0 +1,49 @@
+ROOT := ../..
+COMPONENT := server
+include $(ROOT)/build.mk
+
+LIBFFWD := $(ROOT)/libffwd
+AE := $(ROOT)/deps/ae
+
+BIN := $(OUTDIR)/ffwd-server
+SRCS := base64.c encode.c handlers.c http.c json.c main.c models.c sbuf.c \
+ schedule.c server.c util.c
+OBJS := $(call objs,$(SRCS))
+DEPS := $(call deps,$(OBJS))
+
+TEST_BINS := test_base64 test_sbuf test_encode test_json test_schedule test_models
+TEST_OBJS := $(filter-out $(OBJDIR)/main.o,$(OBJS))
+
+LIBS := $(OUTDIR)/libffwd.a $(OUTDIR)/libae.a
+LDLIBS := $(CJSON_LDFLAGS) $(BACKEND_LDFLAGS) -lm
+
+CFLAGS += -std=gnu11 -pthread
+CPPFLAGS += -I. -I$(LIBFFWD) -I$(AE) $(CJSON_CFLAGS)
+
+$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS)
+
+.PHONY: all clean test
+
+all: $(BIN)
+
+$(eval $(call link_program,$(BIN),$(OBJS),$(LIBS),$(LDLIBS)))
+
+test: $(TEST_BINS)
+ @rc=0; for t in $(TEST_BINS); do ./$$t || rc=1; done; \
+ if [ $$rc -eq 0 ]; then echo "server ok"; else echo "server not ok"; exit 1; fi
+
+$(TEST_BINS): %: %.c test_util.h $(TEST_OBJS) $(LIBS)
+ $(CC) $(CFLAGS) $(CPPFLAGS) $< $(TEST_OBJS) $(LIBS) $(LDLIBS) $(LDFLAGS_OPT) -o $@
+
+$(OUTDIR)/libffwd.a:
+ $(MAKE) -C $(LIBFFWD)
+
+$(OUTDIR)/libae.a:
+ $(MAKE) -C $(AE)
+
+clean:
+ $(RM) -r $(OBJDIR) $(BIN) $(BIN).dSYM \
+ $(TEST_BINS) $(addsuffix .dSYM,$(TEST_BINS))
+ $(MAKE) -C $(AE) clean
+
+-include $(DEPS)
diff --git a/tools/server/base64.c b/tools/server/base64.c
new file mode 100644
index 0000000..87351d2
--- /dev/null
+++ b/tools/server/base64.c
@@ -0,0 +1,47 @@
+#include "base64.h"
+#include "util.h"
+
+#include <stdint.h>
+
+char *base64_encode(const unsigned char *src, size_t n) {
+ static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ size_t max_groups = (SIZE_MAX - 1) / 4;
+ if (n > max_groups * 3)
+ die_oom();
+
+ size_t out_len = ((n + 2) / 3) * 4;
+ char *out = xmalloc(out_len + 1);
+ const unsigned char *p = src;
+ const unsigned char *end = src + (n / 3) * 3;
+ char *q = out;
+
+ while (p < end) {
+ uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | (uint32_t)p[2];
+ q[0] = tbl[(v >> 18) & 63];
+ q[1] = tbl[(v >> 12) & 63];
+ q[2] = tbl[(v >> 6) & 63];
+ q[3] = tbl[v & 63];
+ p += 3;
+ q += 4;
+ }
+
+ size_t rem = n - (size_t)(p - src);
+ if (rem == 1) {
+ uint32_t v = (uint32_t)p[0] << 16;
+ q[0] = tbl[(v >> 18) & 63];
+ q[1] = tbl[(v >> 12) & 63];
+ q[2] = '=';
+ q[3] = '=';
+ q += 4;
+ } else if (rem == 2) {
+ uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8);
+ q[0] = tbl[(v >> 18) & 63];
+ q[1] = tbl[(v >> 12) & 63];
+ q[2] = tbl[(v >> 6) & 63];
+ q[3] = '=';
+ q += 4;
+ }
+
+ *q = '\0';
+ return out;
+}
diff --git a/tools/server/base64.h b/tools/server/base64.h
new file mode 100644
index 0000000..88d88b3
--- /dev/null
+++ b/tools/server/base64.h
@@ -0,0 +1,10 @@
+#ifndef FFWD_BASE64_H
+#define FFWD_BASE64_H
+
+#include <stddef.h>
+
+/* Standard base64 (with '=' padding) of n bytes. Returns a NUL-terminated heap
+ * string the caller frees; die-on-OOM, so it never returns NULL. */
+char *base64_encode(const unsigned char *src, size_t n);
+
+#endif /* FFWD_BASE64_H */
diff --git a/tools/server/encode.c b/tools/server/encode.c
new file mode 100644
index 0000000..faa58a0
--- /dev/null
+++ b/tools/server/encode.c
@@ -0,0 +1,152 @@
+#include "server_internal.h"
+#include "util.h"
+#include "sbuf.h"
+#include "base64.h"
+#include "ffwd.h"
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ========================================================================
+ * Embedding output encoding
+ *
+ * Turns a pooled float vector into the requested wire form. OpenAI-compatible
+ * models emit true float32 arrays by default. Perplexity-compatible models
+ * emit tanh-quantized int8 by default, and their "float" view is the decoded
+ * int8 value for SDK compatibility.
+ * ======================================================================== */
+
+signed char quantize_int8_tanh(float x) {
+ int v = (int)lrintf(tanhf(x) * 127.0f);
+ if (v > 127)
+ v = 127;
+ if (v < -128)
+ v = -128;
+ return (signed char)v;
+}
+
+char *encode_embedding(const float *emb, int dims, const char *encoding) {
+ if (!strcmp(encoding, "base64")) {
+ /* OpenAI/DashScope (Qwen3): base64 of the raw little-endian float32
+ * vector. Copy through a byte buffer instead of reinterpret-casting the
+ * float pointer; x86_64 and aarch64 are little-endian, so the bytes
+ * match the wire format with no swap. */
+ size_t nbytes = (size_t)dims * sizeof(float);
+ unsigned char *buf = xmalloc(nbytes);
+ memcpy(buf, emb, nbytes);
+ char *out = base64_encode(buf, nbytes);
+ free(buf);
+ return out;
+ }
+ if (!strcmp(encoding, "base64_binary")) {
+ size_t bytes = (size_t)dims / 8;
+ unsigned char *bits = xcalloc(bytes, 1);
+ for (int i = 0; i < dims; i++) {
+ if (emb[i] >= 0.0f)
+ bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7));
+ }
+ char *out = base64_encode(bits, bytes);
+ free(bits);
+ return out;
+ }
+
+ signed char *q = xmalloc((size_t)dims);
+ for (int i = 0; i < dims; i++)
+ q[i] = quantize_int8_tanh(emb[i]);
+ char *out = base64_encode((const unsigned char *)q, (size_t)dims);
+ free(q);
+ return out;
+}
+
+static void append_embedding_object(sbuf *b, int index, const char *embedding) {
+ sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index);
+ sbuf_puts(b, embedding);
+ sbuf_puts(b, "\"}");
+}
+
+void append_embedding_value(
+ sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) {
+ if (!strcmp(encoding, "float")) {
+ sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index);
+ for (int i = 0; i < dims; i++) {
+ if (i)
+ sbuf_putc(b, ',');
+ float v =
+ api == FFWD_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i];
+ sbuf_printf(b, "%.9g", (double)v);
+ }
+ sbuf_puts(b, "]}");
+ return;
+ }
+ char *encoded = encode_embedding(emb, dims, encoding);
+ append_embedding_object(b, index, encoded);
+ free(encoded);
+}
+
+void set_response_from_buf(job *j, sbuf *b) {
+ j->status = 200;
+ j->content_type = "application/json";
+ j->response = b->ptr;
+ j->response_len = b->len;
+ memset(b, 0, sizeof(*b));
+}
+
+int render_embedding_response(embedding_request *r, const float *embs) {
+ const int full_dim = r->model->info->dim;
+ const float *render_embs = embs;
+ float *truncated = NULL;
+ if (r->model->renormalize_truncated && r->dims < full_dim) {
+ truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated));
+ for (int i = 0; i < r->n_inputs; i++) {
+ float *dst = truncated + (size_t)i * r->dims;
+ memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst));
+ if (ffwd_l2_normalize(dst, r->dims) != 0) {
+ free(truncated);
+ return -1;
+ }
+ }
+ render_embs = truncated;
+ }
+ int render_stride = truncated ? r->dims : full_dim;
+ embedding_api_t api = r->model->info->api;
+
+ sbuf b = {0};
+ sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
+ for (int i = 0; i < r->n_inputs; i++) {
+ if (i)
+ sbuf_putc(&b, ',');
+ append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding,
+ api);
+ }
+ sbuf_printf(&b,
+ "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
+ set_response_from_buf(r->j, &b);
+
+ free(truncated);
+ return 0;
+}
+
+void render_contextual_response(contextual_request *r, const float *embs) {
+ sbuf b = {0};
+ sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
+ int span_offset = 0;
+ for (int di = 0; di < r->n_docs; di++) {
+ if (di)
+ sbuf_putc(&b, ',');
+ sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di);
+ for (int ci = 0; ci < r->docs[di].n_spans; ci++) {
+ if (ci)
+ sbuf_putc(&b, ',');
+ const float *emb = embs + (size_t)span_offset * r->model->info->dim;
+ append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api);
+ span_offset++;
+ }
+ sbuf_puts(&b, "]}");
+ }
+ sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
+ set_response_from_buf(r->j, &b);
+}
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
new file mode 100644
index 0000000..6720784
--- /dev/null
+++ b/tools/server/handlers.c
@@ -0,0 +1,800 @@
+#include "server_internal.h"
+#include "util.h"
+#include "ffwd.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ========================================================================
+ * Endpoint handlers: prepare, batch, execute, render
+ *
+ * One prepare_* per endpoint validates the request JSON and tokenizes its
+ * inputs into a *_request; the worker (schedule.c) groups compatible
+ * ready requests and calls the matching execute_*_request_list, which runs the
+ * batched forward pass through model_ffwd_* and renders each response via
+ * encode.c. Rerank runs MaxSim over late-interaction token vectors.
+ * ======================================================================== */
+
+int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+typedef struct {
+ ffwd_input_t input;
+ int output_index;
+} embedding_batch_item;
+
+static int embedding_batch_item_cmp(const void *a, const void *b) {
+ const embedding_batch_item *ia = a;
+ const embedding_batch_item *ib = b;
+ if (ia->input.n_tokens < ib->input.n_tokens)
+ return -1;
+ if (ia->input.n_tokens > ib->input.n_tokens)
+ return 1;
+ return ia->output_index - ib->output_index;
+}
+
+int embedding_request_fits_group(const embedding_request *r,
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_inputs == 0)
+ return 1;
+ return r->n_inputs <= max_batch - group_inputs &&
+ r->total_tokens <= max_batch_tokens - group_tokens;
+}
+
+void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_inputs = 0;
+ for (int i = 0; i < n_reqs; i++)
+ total_inputs += reqs[i]->n_inputs;
+
+ embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items));
+ float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float));
+
+ int pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ for (int k = 0; k < reqs[i]->n_inputs; k++) {
+ items[pos].input = reqs[i]->inputs[k];
+ items[pos].output_index = pos;
+ pos++;
+ }
+ }
+ qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp);
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs;
+ if (max_batch > total_inputs)
+ max_batch = total_inputs;
+ int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
+ ffwd_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
+
+ int failed = 0;
+ uint64_t infer_ns = 0;
+ for (int start = 0; start < total_inputs;) {
+ int cur = 0;
+ int tokens = 0;
+ while (start + cur < total_inputs && cur < max_batch) {
+ int n_tokens = items[start + cur].input.n_tokens;
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens))
+ break;
+ inputs[cur] = items[start + cur].input;
+ tokens += n_tokens;
+ cur++;
+ }
+ uint64_t t0 = nstime();
+ int ffwd_rc = model_ffwd_batch(m, inputs, cur, batch_embs);
+ infer_ns += nstime() - t0;
+ if (ffwd_rc != 0) {
+ failed = 1;
+ break;
+ }
+ for (int i = 0; i < cur; i++)
+ memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim,
+ (size_t)dim * sizeof(float));
+ start += cur;
+ }
+
+ pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ reqs[i]->j->infer_ns += infer_ns;
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
+ } else {
+ uint64_t t0 = nstime();
+ if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
+ job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
+ reqs[i]->j->encode_ns += nstime() - t0;
+ }
+ pos += reqs[i]->n_inputs;
+ }
+
+ free(batch_embs);
+ free(inputs);
+ free(embs);
+ free(items);
+}
+
+int contextual_request_compatible(const contextual_request *a, const contextual_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+int contextual_request_fits_group(const contextual_request *r,
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_docs == 0)
+ return 1;
+ return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens;
+}
+
+typedef struct {
+ ffwd_context_input_t input;
+ int output_span_index;
+ int order;
+} contextual_batch_item;
+
+static int contextual_batch_item_cmp(const void *a, const void *b) {
+ const contextual_batch_item *ia = a;
+ const contextual_batch_item *ib = b;
+ if (ia->input.input.n_tokens < ib->input.input.n_tokens)
+ return -1;
+ if (ia->input.input.n_tokens > ib->input.input.n_tokens)
+ return 1;
+ return ia->order - ib->order;
+}
+
+void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_docs = 0;
+ int total_chunks = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ if (total_docs > INT_MAX - reqs[i]->n_docs ||
+ total_chunks > INT_MAX - reqs[i]->total_chunks) {
+ for (int k = 0; k < n_reqs; k++)
+ job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error");
+ return;
+ }
+ total_docs += reqs[i]->n_docs;
+ total_chunks += reqs[i]->total_chunks;
+ }
+ if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
+ for (int i = 0; i < n_reqs; i++)
+ job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error");
+ return;
+ }
+
+ contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items));
+ float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
+ int pos = 0;
+ int span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ for (int d = 0; d < reqs[i]->n_docs; d++) {
+ contextual_doc *doc = &reqs[i]->docs[d];
+ items[pos].input.input.ids = doc->ids;
+ items[pos].input.input.n_tokens = doc->n_tokens;
+ items[pos].input.spans = doc->spans;
+ items[pos].input.n_spans = doc->n_spans;
+ items[pos].output_span_index = span_offset;
+ items[pos].order = pos;
+ span_offset += doc->n_spans;
+ pos++;
+ }
+ }
+ qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs;
+ if (max_batch > total_docs)
+ max_batch = total_docs;
+ int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
+ ffwd_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ float *batch_embs = xmalloc(sizeof(*batch_embs));
+ size_t batch_emb_cap = 0;
+
+ int failed = 0;
+ uint64_t infer_ns = 0;
+ for (int start = 0; start < total_docs;) {
+ int cur = 0;
+ int tokens = 0;
+ int chunks = 0;
+ while (start + cur < total_docs && cur < max_batch) {
+ const ffwd_context_input_t *input = &items[start + cur].input;
+ if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
+ max_batch_tokens))
+ break;
+ inputs[cur] = *input;
+ tokens += input->input.n_tokens;
+ chunks += input->n_spans;
+ cur++;
+ }
+ size_t needed = (size_t)chunks * dim;
+ if (needed > batch_emb_cap) {
+ batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs));
+ batch_emb_cap = needed;
+ }
+ uint64_t t0 = nstime();
+ int ffwd_rc = model_ffwd_spans_batch(m, inputs, cur, batch_embs);
+ infer_ns += nstime() - t0;
+ if (ffwd_rc != 0) {
+ failed = 1;
+ break;
+ }
+ int batch_span = 0;
+ for (int i = 0; i < cur; i++) {
+ int n_spans = inputs[i].n_spans;
+ memcpy(embs + (size_t)items[start + i].output_span_index * dim,
+ batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs));
+ batch_span += n_spans;
+ }
+ start += cur;
+ }
+
+ span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ reqs[i]->j->infer_ns += infer_ns;
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error");
+ } else {
+ uint64_t t0 = nstime();
+ render_contextual_response(reqs[i], embs + (size_t)span_offset * dim);
+ reqs[i]->j->encode_ns += nstime() - t0;
+ }
+ span_offset += reqs[i]->total_chunks;
+ }
+
+ free(batch_embs);
+ free(inputs);
+ free(embs);
+ free(items);
+}
+
+static int validate_common(cJSON *root,
+ cJSON *detail,
+ model_kind expected_kind,
+ loaded_model **out_model,
+ int *out_dims,
+ const char **out_encoding,
+ http_server *s) {
+ cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
+ const char *model_id = NULL;
+ loaded_model *m = NULL;
+ if (!model_item) {
+ ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
+ } else if (!cJSON_IsString(model_item) || !model_item->valuestring) {
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
+ } else {
+ model_id = model_item->valuestring;
+ m = loaded_model_for_label(s, model_id);
+ if (!m) {
+ ve_add(detail, "[\"body\",\"model\"]",
+ "value is not a valid enum member for this endpoint", "enum");
+ } else if (m->info->kind != expected_kind) {
+ ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum");
+ m = NULL;
+ }
+ }
+ *out_model = m;
+ const char *enc = encoding_from_root(root, detail, m ? m->info->api : FFWD_API_OPENAI);
+ int min_dim = m ? m->info->min_dim : 128;
+ int max_dim = m ? m->info->dim : 2560;
+ int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
+ *out_encoding = enc;
+ *out_dims = dims;
+ return 0;
+}
+
+void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ loaded_model *m = NULL;
+ int dims = 0;
+ const char *encoding = NULL;
+ validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
+
+ /* The query instruction, when requested, applies to every input. */
+ const char *query_instruct = m && text_type_is_query(root, detail, m->info->query_instruct)
+ ? m->info->query_instruct
+ : NULL;
+
+ cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
+ int n_inputs = 0;
+ if (!input) {
+ ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
+ } else if (cJSON_IsString(input)) {
+ if (!input->valuestring || input->valuestring[0] == '\0')
+ ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty");
+ n_inputs = 1;
+ } else if (cJSON_IsArray(input)) {
+ n_inputs = cJSON_GetArraySize(input);
+ if (n_inputs < 1)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item",
+ "value_error.list.min_items");
+ else if (n_inputs > FFWD_API_MAX_STANDARD_INPUTS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items",
+ "value_error.list.max_items");
+ cJSON *item;
+ int i = 0;
+ cJSON_ArrayForEach(item, input) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i);
+ if (!cJSON_IsString(item)) {
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
+ } else if (!item->valuestring || item->valuestring[0] == '\0') {
+ ve_add(detail, loc, "input item must not be empty", "value_error.empty");
+ }
+ i++;
+ }
+ } else {
+ ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings",
+ "type_error");
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m && input) {
+ out->n_inputs = n_inputs;
+ out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens));
+ out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs));
+
+ int idx = 0;
+ if (input && cJSON_IsString(input) && input->valuestring) {
+ if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) {
+ ve_add(detail, "[\"body\",\"input\"]", "tokenization failed",
+ "value_error.tokenization");
+ } else {
+ out->inputs[0].ids = out->tokens[0].ids;
+ out->inputs[0].n_tokens = out->tokens[0].n_tokens;
+ out->total_tokens = out->tokens[0].n_tokens;
+ if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ } else {
+ cJSON *item;
+ cJSON_ArrayForEach(item, input) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
+ if (!cJSON_IsString(item) || !item->valuestring) {
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
+ idx++;
+ continue;
+ }
+ if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ } else {
+ out->inputs[idx].ids = out->tokens[idx].ids;
+ out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
+ out->total_tokens += out->tokens[idx].n_tokens;
+ if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
+ ve_add(detail, loc, "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ idx++;
+ }
+ }
+
+ if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ embedding_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ loaded_model *m = NULL;
+ int dims = 0;
+ const char *encoding = NULL;
+ validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
+
+ cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
+ int n_docs = 0;
+ int total_chunks = 0;
+ if (!input) {
+ ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
+ } else if (!cJSON_IsArray(input)) {
+ ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays",
+ "type_error.array");
+ } else {
+ n_docs = cJSON_GetArraySize(input);
+ if (n_docs < 1)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document",
+ "value_error.list.min_items");
+ else if (n_docs > FFWD_API_MAX_CONTEXT_DOCS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents",
+ "value_error.list.max_items");
+ cJSON *doc_arr;
+ int di = 0;
+ cJSON_ArrayForEach(doc_arr, input) {
+ if (!cJSON_IsArray(doc_arr)) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document must be an array of strings", "type_error.array");
+ di++;
+ continue;
+ }
+ int doc_chunks = cJSON_GetArraySize(doc_arr);
+ if (doc_chunks > FFWD_API_MAX_CONTEXT_CHUNKS - total_chunks)
+ total_chunks = FFWD_API_MAX_CONTEXT_CHUNKS + 1;
+ else
+ total_chunks += doc_chunks;
+ if (doc_chunks < 1) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document must contain at least 1 chunk",
+ "value_error.list.min_items");
+ }
+ cJSON *chunk;
+ int ci = 0;
+ cJSON_ArrayForEach(chunk, doc_arr) {
+ char loc[80];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
+ if (!cJSON_IsString(chunk)) {
+ ve_add(detail, loc, "chunk must be a string", "type_error.string");
+ } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') {
+ ve_add(detail, loc, "chunk must not be empty", "value_error.empty");
+ }
+ ci++;
+ }
+ di++;
+ }
+ if (total_chunks > FFWD_API_MAX_CONTEXT_CHUNKS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
+ "value_error.list.max_items");
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m) {
+ out->n_docs = n_docs;
+ out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
+ int64_t request_tokens = 0;
+ cJSON *doc_arr;
+ int di = 0;
+ cJSON_ArrayForEach(doc_arr, input) {
+ contextual_doc *doc = &out->docs[di];
+ int n_chunks = cJSON_GetArraySize(doc_arr);
+ token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
+ doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
+ doc->n_spans = n_chunks;
+ int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
+ int doc_valid = 1;
+
+ cJSON *chunk;
+ int ci = 0;
+ cJSON_ArrayForEach(chunk, doc_arr) {
+ char loc[80];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
+ if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ doc_valid = 0;
+ } else {
+ doc_tokens += chunk_tokens[ci].n_tokens;
+ }
+ ci++;
+ }
+
+ if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document exceeds 32768 token limit",
+ "value_error.context_length");
+ doc_valid = 0;
+ }
+ request_tokens += doc_tokens;
+
+ if (doc_valid) {
+ doc->n_tokens = (int)doc_tokens;
+ doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids));
+ int sep_id = ffwd_tok_context_separator_id(m->tok);
+ int pos = 0;
+ for (ci = 0; ci < n_chunks; ci++) {
+ doc->spans[ci].start = pos;
+ doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens;
+ /* n_tokens > 0 implies ids != NULL (tokenize_one sets both
+ * together), and the guard avoids a memcpy(dst, NULL, 0) for
+ * an empty chunk that the static analyzer reports as UB. */
+ if (chunk_tokens[ci].n_tokens > 0)
+ memcpy(doc->ids + pos, chunk_tokens[ci].ids,
+ (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids));
+ pos += chunk_tokens[ci].n_tokens;
+ if (ci + 1 < n_chunks) {
+ doc->ids[pos++] = sep_id;
+ }
+ }
+ }
+ free_token_bufs(chunk_tokens, n_chunks);
+ free(chunk_tokens);
+ di++;
+ }
+ if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) {
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ } else {
+ out->total_tokens = (int)request_tokens;
+ out->total_chunks = total_chunks;
+ }
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ contextual_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+static int
+validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) {
+ cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model");
+ if (!item) {
+ ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
+ return 0;
+ }
+ if (!cJSON_IsString(item) || !item->valuestring) {
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
+ return 0;
+ }
+ loaded_model *m = loaded_model_for_label(s, item->valuestring);
+ if (!m) {
+ ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
+ "enum");
+ return 0;
+ }
+ if (m->info->kind != MODEL_KIND_LATE) {
+ ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum");
+ return 0;
+ }
+ *out_model = m;
+ return 0;
+}
+
+void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ validate_rerank_model(root, detail, s, &out->model);
+ cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query");
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents");
+ /* query_str is set only once query passes every check; the tokenize step
+ * below runs only when detail is empty, so it is non-NULL there. Hoisting it
+ * also keeps the validated value explicit for the static analyzer. */
+ const char *query_str = NULL;
+ if (!query) {
+ ve_add(detail, "[\"body\",\"query\"]", "field required", "missing");
+ } else if (!cJSON_IsString(query)) {
+ ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string");
+ } else if (!query->valuestring || !query->valuestring[0]) {
+ ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty");
+ } else {
+ query_str = query->valuestring;
+ }
+
+ int n_documents = 0;
+ if (!documents) {
+ ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing");
+ } else if (!cJSON_IsArray(documents)) {
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings",
+ "type_error.array");
+ } else {
+ n_documents = cJSON_GetArraySize(documents);
+ if (n_documents < 1)
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
+ "value_error.list.min_items");
+ else if (n_documents > FFWD_API_MAX_RERANK_DOCUMENTS)
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
+ "value_error.list.max_items");
+ cJSON *document;
+ int i = 0;
+ cJSON_ArrayForEach(document, documents) {
+ char loc[72];
+ snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++);
+ if (!cJSON_IsString(document))
+ ve_add(detail, loc, "document must be a string", "type_error.string");
+ else if (!document->valuestring || !document->valuestring[0])
+ ve_add(detail, loc, "document must not be empty", "value_error.empty");
+ }
+ }
+
+ cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n");
+ cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k");
+ if (top_n && top_k) {
+ ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one",
+ "value_error.conflict");
+ }
+ cJSON *top = top_n ? top_n : top_k;
+ out->top_n = n_documents;
+ if (top) {
+ if (!cjson_is_integer(top)) {
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be an integer", "type_error.integer");
+ } else {
+ out->top_n = top->valueint;
+ if (out->top_n < 1 || out->top_n > n_documents)
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be between 1 and the number of documents", "value_error.range");
+ }
+ }
+
+ cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents");
+ if (return_documents) {
+ if (!cJSON_IsBool(return_documents))
+ ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean",
+ "type_error.bool");
+ else
+ out->return_documents = cJSON_IsTrue(return_documents);
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && out->model) {
+ out->n_documents = n_documents;
+ out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
+ if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) {
+ ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization");
+ } else {
+ out->query_tokens = out->query.n_tokens;
+ if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS)
+ ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+
+ cJSON *document;
+ int i = 0;
+ cJSON_ArrayForEach(document, documents) {
+ char loc[72];
+ snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i);
+ if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) !=
+ 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ } else {
+ int n = out->documents[i].n_tokens;
+ if (n > FFWD_API_MAX_ITEM_TOKENS)
+ ve_add(detail, loc, "document exceeds 32768 token limit",
+ "value_error.context_length");
+ if (out->document_tokens <= INT_MAX - n)
+ out->document_tokens += n;
+ else
+ out->document_tokens = INT_MAX;
+ }
+ i++;
+ }
+ if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) {
+ ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ }
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ rerank_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+typedef struct {
+ int index;
+ float score;
+} rerank_result;
+
+static int rerank_result_cmp(const void *a, const void *b) {
+ const rerank_result *ra = a;
+ const rerank_result *rb = b;
+ if (ra->score > rb->score)
+ return -1;
+ if (ra->score < rb->score)
+ return 1;
+ return ra->index - rb->index;
+}
+
+void execute_rerank_request(rerank_request *r) {
+ if (!r || !r->model) {
+ if (r && r->j)
+ job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
+ return;
+ }
+ float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
+
+ /* Flatten the per-document fields into the arrays the backend rerank takes;
+ * it encodes the query and every candidate in one packed forward and scores
+ * MaxSim, so N short documents collapse to a single batch. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+ ffwd_rerank_input_t in = {
+ .query_ids = r->query.ids,
+ .query_n_tokens = r->query.n_tokens,
+ .query_n_keep = r->query.n_keep,
+ .doc_ids = doc_ids,
+ .doc_n_tokens = n_tokens,
+ .doc_keep = keep,
+ .doc_n_keep = n_keep,
+ .n_docs = r->n_documents,
+ };
+ uint64_t t0 = nstime();
+ int rc = ffwd_rerank(r->model->backend, &in, scores);
+ r->j->infer_ns += nstime() - t0;
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
+ if (rc != 0) {
+ free(scores);
+ job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
+ return;
+ }
+
+ rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked));
+ for (int i = 0; i < r->n_documents; i++) {
+ ranked[i].index = i;
+ ranked[i].score = scores[i];
+ }
+ qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp);
+
+ t0 = nstime();
+ sbuf b = {0};
+ sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id);
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents");
+ for (int i = 0; i < r->top_n; i++) {
+ if (i)
+ sbuf_putc(&b, ',');
+ sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index,
+ (double)ranked[i].score);
+ if (r->return_documents) {
+ cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index);
+ sbuf_puts(&b, ",\"document\":");
+ append_json_string(&b, document->valuestring);
+ }
+ sbuf_putc(&b, '}');
+ }
+ sbuf_printf(&b,
+ "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens);
+ set_response_from_buf(r->j, &b);
+ r->j->encode_ns += nstime() - t0;
+ free(ranked);
+ free(scores);
+}
diff --git a/tools/server/http.c b/tools/server/http.c
new file mode 100644
index 0000000..a1d347f
--- /dev/null
+++ b/tools/server/http.c
@@ -0,0 +1,399 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * HTTP/1.1 connection lifecycle, header parsing, and response framing
+ *
+ * One client per connection, refcounted: the event loop holds a reference
+ * while the socket is open and the worker thread holds one while a job derived
+ * from the connection is in flight, so a request that completes after the peer
+ * disconnects frees the client safely. read_cb accumulates bytes until a full
+ * request is framed, then hands off to dispatch_request (server.c); write_cb
+ * drains the response and either closes or arms the next pipelined request.
+ * ======================================================================== */
+
+int set_nonblock(int fd) {
+ int flags = fcntl(fd, F_GETFL, 0);
+ if (flags < 0)
+ return -1;
+ return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+}
+
+static void http_request_free(http_request *r) {
+ free(r->body);
+ free(r->auth);
+ memset(r, 0, sizeof(*r));
+}
+
+static void client_free(client *c) {
+ if (!c)
+ return;
+ http_request_free(&c->req);
+ sbuf_free(&c->in);
+ sbuf_free(&c->out);
+ free(c);
+}
+
+void client_incref(client *c) {
+ pthread_mutex_lock(&c->srv->mu);
+ c->refcount++;
+ pthread_mutex_unlock(&c->srv->mu);
+}
+
+static void client_decref(client *c) {
+ int free_now = 0;
+ pthread_mutex_lock(&c->srv->mu);
+ if (--c->refcount == 0)
+ free_now = 1;
+ pthread_mutex_unlock(&c->srv->mu);
+ if (free_now)
+ client_free(c);
+}
+
+void client_decref_n(client *c, int n) {
+ int free_now = 0;
+ if (n <= 0)
+ return;
+ pthread_mutex_lock(&c->srv->mu);
+ c->refcount -= n;
+ if (c->refcount <= 0)
+ free_now = 1;
+ pthread_mutex_unlock(&c->srv->mu);
+ if (free_now)
+ client_free(c);
+}
+
+static void client_unlink(client *c) {
+ http_server *s = c->srv;
+ if (!c->linked)
+ return;
+ if (c->prev)
+ c->prev->next = c->next;
+ else
+ s->clients = c->next;
+ if (c->next)
+ c->next->prev = c->prev;
+ c->prev = c->next = NULL;
+ c->linked = 0;
+ if (s->n_clients > 0)
+ s->n_clients--;
+}
+
+int close_client_unlink(client *c) {
+ if (!c || c->cancelled)
+ return 0;
+ c->cancelled = 1;
+ if (c->fd >= 0) {
+ aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
+ aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE);
+ close(c->fd);
+ c->fd = -1;
+ }
+ client_unlink(c);
+ return 1;
+}
+
+void close_client(client *c) {
+ if (!close_client_unlink(c))
+ return;
+ client_decref(c);
+}
+
+void append_http_response_ex(client *c,
+ int status,
+ const char *ctype,
+ const char *extra_headers,
+ const char *body,
+ size_t body_len) {
+ const char *reason = "OK";
+ if (status == 204)
+ reason = "No Content";
+ else if (status == 400)
+ reason = "Bad Request";
+ else if (status == 401)
+ reason = "Unauthorized";
+ else if (status == 404)
+ reason = "Not Found";
+ else if (status == 405)
+ reason = "Method Not Allowed";
+ else if (status == 422)
+ reason = "Unprocessable Entity";
+ else if (status == 503)
+ reason = "Service Unavailable";
+ else if (status >= 500)
+ reason = "Internal Server Error";
+
+ sbuf_printf(&c->out,
+ "HTTP/1.1 %d %s\r\n"
+ "Content-Length: %zu\r\n"
+ "Connection: %s\r\n",
+ status, reason, body_len, c->close_after_write ? "close" : "keep-alive");
+ if (ctype)
+ sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype);
+
+ if (extra_headers)
+ sbuf_puts(&c->out, extra_headers);
+ sbuf_puts(&c->out, "\r\n");
+ if (body_len)
+ sbuf_append(&c->out, body, body_len);
+}
+
+void append_http_response(
+ client *c, int status, const char *ctype, const char *body, size_t body_len) {
+ append_http_response_ex(c, status, ctype, NULL, body, body_len);
+}
+
+static ssize_t find_header_end(const char *p, size_t n) {
+ for (size_t i = 3; i < n; i++) {
+ if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n')
+ return (ssize_t)(i + 1);
+ }
+ for (size_t i = 1; i < n; i++) {
+ if (p[i - 1] == '\n' && p[i] == '\n')
+ return (ssize_t)(i + 1);
+ }
+ return -1;
+}
+
+static char *header_value_dup(const char *h, size_t n, const char *name) {
+ size_t name_len = strlen(name);
+ const char *p = h, *end = h + n;
+ while (p < end) {
+ const char *line = p;
+ while (p < end && *p != '\n')
+ p++;
+ size_t len = (size_t)(p - line);
+ if (len && line[len - 1] == '\r')
+ len--;
+ if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') {
+ const char *v = line + name_len + 1;
+ while (v < line + len && isspace((unsigned char)*v))
+ v++;
+ const char *e = line + len;
+ while (e > v && isspace((unsigned char)e[-1]))
+ e--;
+ return xstrndup(v, (size_t)(e - v));
+ }
+ if (p < end)
+ p++;
+ }
+ return NULL;
+}
+
+static int header_has_token(const char *value, const char *token) {
+ size_t token_len = strlen(token);
+ const char *p = value;
+ while (p && *p) {
+ while (*p == ',' || isspace((unsigned char)*p))
+ p++;
+ const char *start = p;
+ while (*p && *p != ',')
+ p++;
+ const char *end = p;
+ while (end > start && isspace((unsigned char)end[-1]))
+ end--;
+ if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len))
+ return 1;
+ }
+ return 0;
+}
+
+static int parse_request_headers(client *c) {
+ char line[512];
+ size_t i = 0;
+ while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) {
+ line[i] = c->in.ptr[i];
+ i++;
+ }
+ line[i] = '\0';
+ int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version);
+ if (fields < 2)
+ return -1;
+ if (fields < 3)
+ snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0");
+ char *q = strchr(c->req.path, '?');
+ if (q)
+ *q = '\0';
+
+ c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0;
+ char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection");
+ if (conn) {
+ if (header_has_token(conn, "close"))
+ c->close_after_write = 1;
+ else if (header_has_token(conn, "keep-alive"))
+ c->close_after_write = 0;
+ free(conn);
+ }
+
+ char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length");
+ if (cl) {
+ char *end = NULL;
+ errno = 0;
+ unsigned long long v = strtoull(cl, &end, 10);
+ if (errno || !end || *end || v > FFWD_HTTP_MAX_BODY) {
+ free(cl);
+ return -1;
+ }
+ c->content_length = (size_t)v;
+ free(cl);
+ }
+ c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization");
+ return 0;
+}
+
+static int complete_request(client *c) {
+ if (!c->header_done) {
+ ssize_t hend = find_header_end(c->in.ptr, c->in.len);
+ if (hend < 0) {
+ if (c->in.len > FFWD_HTTP_MAX_HEADER)
+ return -1;
+ return 0;
+ }
+ c->header_done = 1;
+ c->header_len = (size_t)hend;
+ if (parse_request_headers(c) != 0)
+ return -1;
+ }
+ if (c->content_length > FFWD_HTTP_MAX_BODY)
+ return -1;
+ if (c->in.len < c->header_len + c->content_length)
+ return 0;
+ c->req.body_len = c->content_length;
+ c->req.body = xmalloc(c->content_length + 1);
+ memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length);
+ c->req.body[c->content_length] = '\0';
+ return 1;
+}
+
+static void send_bad_http_request(client *c) {
+ size_t len;
+ char *body = json_error_body("bad HTTP request", "invalid_request_error", &len);
+ c->close_after_write = 1;
+ append_http_response(c, 400, "application/json", body, len);
+ free(body);
+ aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+}
+
+static void reset_client_for_next_request(client *c) {
+ size_t consumed = c->header_len + c->content_length;
+ if (consumed > c->in.len)
+ consumed = c->in.len;
+ size_t remain = c->in.len - consumed;
+ if (remain)
+ memmove(c->in.ptr, c->in.ptr + consumed, remain);
+ c->in.len = remain;
+ if (c->in.ptr)
+ c->in.ptr[remain] = '\0';
+
+ http_request_free(&c->req);
+ sbuf_clear(&c->out);
+ c->sent = 0;
+ c->header_done = 0;
+ c->header_len = 0;
+ c->content_length = 0;
+ c->close_after_write = 0;
+}
+
+static void arm_next_request(client *c) {
+ reset_client_for_next_request(c);
+ if (c->cancelled || c->fd < 0)
+ return;
+
+ if (c->in.len) {
+ int done = complete_request(c);
+ if (done < 0) {
+ send_bad_http_request(c);
+ return;
+ }
+ if (done > 0) {
+ dispatch_request(c);
+ return;
+ }
+ }
+
+ if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK)
+ close_client(c);
+}
+
+static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)loop;
+ (void)mask;
+ client *c = clientData;
+ while (c->sent < c->out.len) {
+ ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent);
+ if (n > 0) {
+ c->sent += (size_t)n;
+ c->last_active_ms = mstime();
+ continue;
+ }
+ if (n < 0 && errno == EINTR)
+ continue;
+ if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
+ return;
+ close_client(c);
+ return;
+ }
+ aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE);
+ if (c->close_after_write)
+ close_client(c);
+ else
+ arm_next_request(c);
+}
+
+int queue_write(client *c) {
+ if (!c || c->cancelled || c->fd < 0)
+ return -1;
+ return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c);
+}
+
+bool route_is_inference(const char *method, const char *path) {
+ return !strcmp(method, "POST") &&
+ (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") ||
+ !strcmp(path, "/v1/rerank"));
+}
+
+void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)loop;
+ (void)mask;
+ client *c = clientData;
+ char tmp[FFWD_HTTP_IO_BUF];
+ for (;;) {
+ ssize_t n = read(fd, tmp, sizeof(tmp));
+ if (n > 0) {
+ sbuf_append(&c->in, tmp, (size_t)n);
+ c->last_active_ms = mstime();
+ int done = complete_request(c);
+ if (done < 0) {
+ send_bad_http_request(c);
+ return;
+ }
+ if (done > 0) {
+ dispatch_request(c);
+ return;
+ }
+ continue;
+ }
+ if (n == 0) {
+ close_client(c);
+ return;
+ }
+ if (errno == EINTR)
+ continue;
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
+ close_client(c);
+ return;
+ }
+}
diff --git a/tools/server/json.c b/tools/server/json.c
new file mode 100644
index 0000000..4812164
--- /dev/null
+++ b/tools/server/json.c
@@ -0,0 +1,217 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <cjson/cJSON.h>
+#include <ctype.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ========================================================================
+ * Request validation and error bodies
+ *
+ * Two JSON shapes are produced here: the OpenAI-style {"error":{message,type}}
+ * body (append_json_error / json_error_body / job_set_error) for transport and
+ * auth failures, and the Perplexity-style {"detail":[...]} 422 body built from
+ * ve_add entries for field validation. The parse_* / *_from_root helpers read
+ * and validate the request object; append_json_string escapes a value for
+ * inclusion in a response. */
+
+static void append_json_error(sbuf *b, const char *message, const char *type) {
+ sbuf_puts(b, "{\"error\":{\"message\":");
+ append_json_string(b, message ? message : "");
+ sbuf_puts(b, ",\"type\":");
+ append_json_string(b, type ? type : "server_error");
+ sbuf_puts(b, "}}");
+}
+
+void append_json_string(sbuf *b, const char *s) {
+ sbuf_putc(b, '"');
+ for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) {
+ switch (*p) {
+ case '"':
+ sbuf_puts(b, "\\\"");
+ break;
+ case '\\':
+ sbuf_puts(b, "\\\\");
+ break;
+ case '\b':
+ sbuf_puts(b, "\\b");
+ break;
+ case '\f':
+ sbuf_puts(b, "\\f");
+ break;
+ case '\n':
+ sbuf_puts(b, "\\n");
+ break;
+ case '\r':
+ sbuf_puts(b, "\\r");
+ break;
+ case '\t':
+ sbuf_puts(b, "\\t");
+ break;
+ default:
+ if (*p < 0x20)
+ sbuf_printf(b, "\\u%04x", (unsigned)*p);
+ else
+ sbuf_putc(b, (char)*p);
+ }
+ }
+ sbuf_putc(b, '"');
+}
+
+char *json_error_body(const char *message, const char *type, size_t *len) {
+ sbuf b = {0};
+ append_json_error(&b, message, type);
+ *len = b.len;
+ return b.ptr;
+}
+
+void job_set_error(job *j, int status, const char *message, const char *type) {
+ free(j->response);
+ j->status = status;
+ j->content_type = "application/json";
+ j->response = json_error_body(message, type, &j->response_len);
+}
+
+bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) {
+ cJSON *obj = cJSON_CreateObject();
+ cJSON *loc = cJSON_Parse(loc_json);
+ if (!obj || !loc) {
+ cJSON_Delete(obj);
+ cJSON_Delete(loc);
+ return false;
+ }
+ cJSON_AddItemToObject(obj, "loc", loc);
+ cJSON_AddStringToObject(obj, "msg", msg ? msg : "");
+ cJSON_AddStringToObject(obj, "type", type ? type : "");
+ cJSON_AddItemToArray(detail, obj);
+ return true;
+}
+
+void job_set_422(job *j, cJSON *detail) {
+ cJSON *root = cJSON_CreateObject();
+ cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1));
+ char *s = cJSON_PrintUnformatted(root);
+ cJSON_Delete(root);
+ if (!s) {
+ job_set_error(j, 500, "failed to render validation error", "server_error");
+ return;
+ }
+ free(j->response);
+ j->status = 422;
+ j->content_type = "application/json";
+ j->response = xstrdup(s);
+ j->response_len = strlen(j->response);
+ cJSON_free(s);
+}
+
+cJSON *parse_json_body(job *j, cJSON *detail) {
+ if (!j->body || j->body_len == 0) {
+ ve_add(detail, "[\"body\"]", "body required", "missing");
+ return NULL;
+ }
+ const char *end = NULL;
+ cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false);
+ if (!root) {
+ ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
+ return NULL;
+ }
+ const char *body_end = j->body + j->body_len;
+ while (end && end < body_end && isspace((unsigned char)*end))
+ end++;
+ if (!end || end != body_end) {
+ ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
+ cJSON_Delete(root);
+ return NULL;
+ }
+ if (!cJSON_IsObject(root)) {
+ ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object");
+ cJSON_Delete(root);
+ return NULL;
+ }
+ return root;
+}
+
+bool cjson_is_integer(cJSON *item) {
+ if (!cJSON_IsNumber(item))
+ return false;
+ double d = item->valuedouble;
+ int i = item->valueint;
+ return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i;
+}
+
+const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
+ const char *dflt = api == FFWD_API_PERPLEXITY ? "base64_int8" : "float";
+ cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
+ if (!encoding)
+ return dflt;
+ if (!cJSON_IsString(encoding) || !encoding->valuestring) {
+ ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string",
+ "type_error.string");
+ return dflt;
+ }
+ const char *v = encoding->valuestring;
+ int ok = api == FFWD_API_PERPLEXITY
+ ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || !strcmp(v, "float"))
+ : (!strcmp(v, "float") || !strcmp(v, "base64"));
+ if (!ok) {
+ ve_add(detail, "[\"body\",\"encoding_format\"]",
+ api == FFWD_API_PERPLEXITY
+ ? "value is not a valid enum member; permitted: "
+ "'base64_int8', 'base64_binary', 'float'"
+ : "value is not a valid enum member; permitted: 'float', 'base64'",
+ "enum");
+ return dflt;
+ }
+ return v;
+}
+
+/* Parse the `text_type` hint. Returns 1 when the model's query instruction
+ * should be prepended, 0 otherwise (document / absent). Models without a
+ * published query instruction reject the field. */
+int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) {
+ cJSON *tt = cJSON_GetObjectItemCaseSensitive(root, "text_type");
+ if (!tt)
+ return 0;
+ if (!query_instruct) {
+ ve_add(detail, "[\"body\",\"text_type\"]",
+ "text_type is only supported for instruction embedding models", "extra_fields");
+ return 0;
+ }
+ if (!cJSON_IsString(tt) || !tt->valuestring) {
+ ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string", "type_error.string");
+ return 0;
+ }
+ const char *v = tt->valuestring;
+ if (!strcmp(v, "query"))
+ return 1;
+ if (!strcmp(v, "document"))
+ return 0;
+ ve_add(detail, "[\"body\",\"text_type\"]",
+ "value is not a valid enum member; permitted: 'query', 'document'", "enum");
+ return 0;
+}
+
+int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
+ int dims = max_dim;
+ cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
+ if (dimensions) {
+ if (!cjson_is_integer(dimensions)) {
+ ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer",
+ "type_error.integer");
+ } else {
+ dims = dimensions->valueint;
+ if (dims < min_dim || dims > max_dim) {
+ char msg[96];
+ snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, max_dim);
+ ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
+ }
+ }
+ }
+ if (!strcmp(encoding, "base64_binary") && dims % 8 != 0)
+ ve_add(detail, "[\"body\",\"dimensions\"]",
+ "dimensions must be divisible by 8 for base64_binary", "value_error.divisible");
+ return dims;
+}
diff --git a/tools/server/main.c b/tools/server/main.c
new file mode 100644
index 0000000..1cbdc3b
--- /dev/null
+++ b/tools/server/main.c
@@ -0,0 +1,342 @@
+/* main.c - ffwd-server entry point.
+ *
+ * The frontend for the ffwd-server binary: argument parsing, model-spec
+ * handling, and main(). The server logic lives in server.c; this file only
+ * builds the ffwd_server_config_t and calls ffwd_run_server. It is split
+ * from server.c so the test suite can include server.c (for its internal
+ * functions) without a conflicting main() and without a compile-out guard. */
+#include "build.h"
+#include "server.h"
+#include "server_internal.h"
+#include "ffwd.h"
+#include "internal.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void print_usage(const char *prog) {
+ fprintf(stderr,
+ "Usage: %s --model LABEL=DIR [options]\n"
+ "\n"
+ "Options:\n"
+ " --model LABEL=DIR[:key=val]\n"
+ " Standard embedding model (repeatable)\n"
+ " --contextual-model LABEL=DIR[:key=val]\n"
+ " Contextual embedding model (repeatable)\n"
+ " --late-model LABEL=DIR[:key=val]\n"
+ " Late-interaction rerank model (repeatable)\n"
+ " keys: api=openai|perplexity, min_dim=N,\n"
+ " query=TEXT (last key; \\n escapes newline)\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 32)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N First-arrival micro-batch deadline in us\n"
+ " (default: %d)\n",
+ prog, ffwd_default_batch_wait_us());
+ ffwd_server_help(stderr);
+ fprintf(stderr,
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -V, --version Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model docs=./model --port 8000\n"
+ " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n",
+ prog, prog);
+}
+
+typedef struct {
+ ffwd_server_model_spec_t *v;
+ int n;
+ int cap;
+} model_specs_t;
+
+static char *dup_range(const char *p, size_t n) {
+ char *s = (char *)malloc(n + 1);
+ if (!s)
+ return NULL;
+ memcpy(s, p, n);
+ s[n] = '\0';
+ return s;
+}
+
+static char *unescape_model_option(const char *s) {
+ size_t n = strlen(s);
+ char *out = (char *)malloc(n + 1);
+ if (!out)
+ return NULL;
+ size_t w = 0;
+ for (size_t r = 0; r < n; r++) {
+ if (s[r] == '\\' && r + 1 < n) {
+ r++;
+ if (s[r] == 'n')
+ out[w++] = '\n';
+ else if (s[r] == 't')
+ out[w++] = '\t';
+ else if (s[r] == 'r')
+ out[w++] = '\r';
+ else
+ out[w++] = s[r];
+ } else {
+ out[w++] = s[r];
+ }
+ }
+ out[w] = '\0';
+ return out;
+}
+
+static int parse_model_option(ffwd_server_model_spec_t *spec, const char *opt, const char *flag) {
+ const char *eq = strchr(opt, '=');
+ if (!eq || eq == opt || !eq[1]) {
+ fprintf(stderr, "%s option expects key=value: %s\n", flag, opt);
+ return -1;
+ }
+ size_t key_len = (size_t)(eq - opt);
+ const char *val = eq + 1;
+ if (key_len == 3 && strncmp(opt, "api", key_len) == 0) {
+ if (!strcmp(val, "openai")) {
+ spec->api = FFWD_SERVER_API_OPENAI;
+ } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) {
+ spec->api = FFWD_SERVER_API_PERPLEXITY;
+ } else if (!strcmp(val, "default")) {
+ spec->api = FFWD_SERVER_API_DEFAULT;
+ } else {
+ fprintf(stderr, "%s api must be openai or perplexity\n", flag);
+ return -1;
+ }
+ return 0;
+ }
+ if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) {
+ char *end = NULL;
+ long v = strtol(val, &end, 10);
+ if (!end || *end || v <= 0 || v > INT_MAX) {
+ fprintf(stderr, "%s min_dim must be a positive integer\n", flag);
+ return -1;
+ }
+ spec->min_dim = (int)v;
+ return 0;
+ }
+ fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt);
+ return -1;
+}
+
+static int append_model_spec(model_specs_t *specs,
+ const char *arg,
+ ffwd_server_model_kind_t kind,
+ const char *flag) {
+ const char *eq = strchr(arg, '=');
+ if (!eq || eq == arg || !eq[1]) {
+ fprintf(stderr, "%s expects LABEL=DIR\n", flag);
+ return -1;
+ }
+ if (specs->n == specs->cap) {
+ int new_cap = specs->cap ? specs->cap * 2 : 4;
+ void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
+ if (!p)
+ return -1;
+ specs->v = p;
+ specs->cap = new_cap;
+ }
+ size_t id_len = (size_t)(eq - arg);
+ const char *path_start = eq + 1;
+ const char *opts = strchr(path_start, ':');
+ size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start);
+ if (path_len == 0) {
+ fprintf(stderr, "%s expects a non-empty DIR\n", flag);
+ return -1;
+ }
+ ffwd_server_model_spec_t spec;
+ memset(&spec, 0, sizeof(spec));
+ spec.id = dup_range(arg, id_len);
+ spec.path = dup_range(path_start, path_len);
+ spec.kind = kind;
+ if (!spec.id || !spec.path) {
+ free((char *)spec.id);
+ free((char *)spec.path);
+ return -1;
+ }
+ const char *p = opts ? opts + 1 : NULL;
+ while (p && *p) {
+ if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) {
+ const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15);
+ spec.query_instruct = unescape_model_option(val);
+ if (!spec.query_instruct)
+ goto fail;
+ break;
+ }
+ const char *next = strchr(p, ':');
+ size_t opt_len = next ? (size_t)(next - p) : strlen(p);
+ char *opt = dup_range(p, opt_len);
+ if (!opt)
+ goto fail;
+ int rc = parse_model_option(&spec, opt, flag);
+ free(opt);
+ if (rc != 0)
+ goto fail;
+ p = next ? next + 1 : NULL;
+ }
+ specs->v[specs->n] = spec;
+ specs->n++;
+ return 0;
+
+fail:
+ free((char *)spec.id);
+ free((char *)spec.path);
+ free((char *)spec.query_instruct);
+ return -1;
+}
+
+static void free_model_specs(model_specs_t *specs) {
+ if (!specs)
+ return;
+ for (int i = 0; i < specs->n; i++) {
+ free((char *)specs->v[i].id);
+ free((char *)specs->v[i].path);
+ free((char *)specs->v[i].query_instruct);
+ }
+ free(specs->v);
+}
+
+int main(int argc, char *argv[]) {
+ int verbose = 0;
+ int batch_size = 0;
+ int max_batch_tokens = 0;
+ int batch_wait_us = -1;
+ /* Backend tuning flags parse into opts regardless of backend; the linked
+ * backend uses what it supports. */
+ ffwd_options_t opts = {.gpu_quant_group_size = 64};
+ const char *host = "127.0.0.1";
+ const char *api_key = NULL;
+ int port = 8000;
+ model_specs_t model_specs = {0};
+
+ const char *prog = ffwd_prog_name(argv[0]);
+ int arg = 1;
+ while (arg < argc && argv[arg][0] == '-') {
+ const char *f = argv[arg];
+ if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
+ ffwd_print_version(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else if (!strcmp(f, "--build-info")) {
+ ffwd_print_build_info(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else if (!strcmp(f, "--model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_STANDARD, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--contextual-model")) {
+ if (arg + 1 >= argc || append_model_spec(&model_specs, argv[++arg],
+ FFWD_SERVER_MODEL_CONTEXTUAL, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--late-model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_LATE, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--gpu-quant-bits")) {
+ opts.gpu_quant_bits = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--gpu-quant-group-size")) {
+ opts.gpu_quant_group_size = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--memory-utilization")) {
+ opts.memory_utilization = atof(argv[++arg]);
+ if (opts.memory_utilization <= 0.0) {
+ fprintf(stderr, "--memory-utilization must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--gpu-gemm-mode")) {
+ opts.gpu_gemm_mode = argv[++arg];
+ } else if (!strcmp(f, "--gpu-weight-dtype")) {
+ opts.gpu_weight_dtype = argv[++arg];
+ } else if (!strcmp(f, "--host")) {
+ host = argv[++arg];
+ } else if (!strcmp(f, "--port")) {
+ port = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--api-key")) {
+ api_key = argv[++arg];
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--max-batch-tokens")) {
+ max_batch_tokens = atoi(argv[++arg]);
+ if (max_batch_tokens <= 0) {
+ fprintf(stderr, "--max-batch-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ opts.n_threads = atoi(argv[++arg]);
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else {
+ fprintf(stderr, "unknown option: %s\n", f);
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ arg++;
+ }
+
+ ffwd_verbose = verbose;
+
+ if (arg < argc) {
+ fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (model_specs.n == 0) {
+ fprintf(stderr, "at least one --model LABEL=DIR is required\n");
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ char err[256];
+ if (ffwd_init(&opts, err, sizeof(err)) != 0) {
+ fprintf(stderr, "%s\n", err);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
+ ffwd_server_config_t scfg = {
+ .models = model_specs.v,
+ .n_models = model_specs.n,
+ .host = host,
+ .port = port,
+ .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE,
+ .max_batch_tokens = max_batch_tokens,
+ .batch_wait_us = batch_wait_us,
+ .gpu_quantize_bits = opts.gpu_quant_bits,
+ .gpu_quantize_group_size = opts.gpu_quant_group_size,
+ .memory_utilization = opts.memory_utilization,
+ .api_key = api_key,
+ };
+ int rc = ffwd_run_server(&scfg);
+ free_model_specs(&model_specs);
+ return rc;
+}
diff --git a/tools/server/models.c b/tools/server/models.c
new file mode 100644
index 0000000..8fdc590
--- /dev/null
+++ b/tools/server/models.c
@@ -0,0 +1,342 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * Model load/lifecycle, and tokenize/inference dispatch
+ *
+ * Each loaded model owns resolved serving metadata for the operator-chosen
+ * label. Transformer behavior comes from the model files through ffwd_config;
+ * server-only facts absent from those files, such as output API family and
+ * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_*
+ * helpers are the single dispatch point the request handlers call, hiding the
+ * compiled backend choice behind one signature.
+ * ======================================================================== */
+
+loaded_model *loaded_model_for_label(http_server *s, const char *label) {
+ if (!label)
+ return NULL;
+ for (int i = 0; i < s->n_models; i++) {
+ if (s->models[i].info && !strcmp(label, s->models[i].info->id))
+ return &s->models[i];
+ }
+ return NULL;
+}
+
+static model_kind model_kind_from_public(ffwd_server_model_kind_t kind) {
+ switch (kind) {
+ case FFWD_SERVER_MODEL_CONTEXTUAL:
+ return MODEL_KIND_CONTEXTUAL;
+ case FFWD_SERVER_MODEL_LATE:
+ return MODEL_KIND_LATE;
+ case FFWD_SERVER_MODEL_STANDARD:
+ default:
+ return MODEL_KIND_STANDARD;
+ }
+}
+
+static embedding_api_t embedding_api_from_public(ffwd_server_embedding_api_t api) {
+ return api == FFWD_SERVER_API_PERPLEXITY ? FFWD_API_PERPLEXITY : FFWD_API_OPENAI;
+}
+
+static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ server_log("ffwd-server: model path too long");
+ return -1;
+ }
+ FILE *f = fopen(path, "rb");
+ if (!f && errno == ENOENT)
+ return 0;
+ if (!f) {
+ server_log("ffwd-server: failed to open %s", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ free(buf);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+ *out = buf;
+ return 1;
+}
+
+static int read_query_instruct_from_files(const char *model_dir, char **out) {
+ char *buf = NULL;
+ int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
+ if (rc <= 0)
+ return rc;
+ cJSON *root = cJSON_Parse(buf);
+ free(buf);
+ if (!root) {
+ server_log("ffwd-server: invalid config_sentence_transformers.json: %s", model_dir);
+ return -1;
+ }
+ cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
+ cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
+ if (query && !cJSON_IsString(query)) {
+ cJSON_Delete(root);
+ server_log("ffwd-server: prompts.query must be a string: %s", model_dir);
+ return -1;
+ }
+ if (query && query->valuestring && query->valuestring[0]) {
+ *out = xstrdup(query->valuestring);
+ cJSON_Delete(root);
+ return *out ? 1 : -1;
+ }
+ cJSON_Delete(root);
+ return 0;
+}
+
+static int init_model_info(model_info **out, const ffwd_server_model_spec_t *spec) {
+ model_info *info = xcalloc(1, sizeof(*info));
+ info->id = xstrdup(spec->id);
+ info->kind = model_kind_from_public(spec->kind);
+ info->api = embedding_api_from_public(spec->api);
+ info->min_dim = spec->min_dim;
+ if (spec->query_instruct) {
+ info->query_instruct = xstrdup(spec->query_instruct);
+ } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ if (!info->id || (spec->query_instruct && !info->query_instruct)) {
+ free(info->query_instruct);
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ *out = info;
+ return 0;
+}
+
+static void free_model_info(model_info *info) {
+ if (!info)
+ return;
+ free(info->id);
+ free(info->query_instruct);
+ free(info);
+}
+
+void free_token_bufs(token_buf *t, int n) {
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
+}
+
+void embedding_request_free(embedding_request *r) {
+ if (!r)
+ return;
+ free_token_bufs(r->tokens, r->n_inputs);
+ free(r->tokens);
+ free(r->inputs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void contextual_request_free(contextual_request *r) {
+ if (!r)
+ return;
+ for (int i = 0; i < r->n_docs; i++) {
+ free(r->docs[i].ids);
+ free(r->docs[i].spans);
+ }
+ free(r->docs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void late_tokens_free(late_tokens *t) {
+ if (!t)
+ return;
+ free(t->ids);
+ free(t->keep);
+ memset(t, 0, sizeof(*t));
+}
+
+void rerank_request_free(rerank_request *r) {
+ if (!r)
+ return;
+ late_tokens_free(&r->query);
+ for (int i = 0; i < r->n_documents; i++)
+ late_tokens_free(&r->documents[i]);
+ free(r->documents);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only
+ * add the server's per-stage timing onto the job; the request handlers call
+ * them unchanged. */
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
+ uint64_t t0 = nstime();
+ out->ids = ffwd_tokenize(m->tok, text, query_instruct, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ if (!out->ids) {
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ return 0;
+}
+
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
+ return tokenize_input(m, j, text, NULL, out);
+}
+
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
+ uint64_t t0 = nstime();
+ int rc = ffwd_tokenize_late(m->tok, text, is_query, out);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ return rc;
+}
+
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_encode_batch(m->backend, inputs, batch, out);
+}
+
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_encode_spans_batch(m->backend, inputs, batch, out);
+}
+
+static int model_uses_dense_batches(const loaded_model *m) {
+ return ffwd_uses_dense_batches(m->backend);
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size <= 0) {
+ if (m && m->info)
+ server_log("ffwd-server: model %s has invalid config", m->info->id);
+ return -1;
+ }
+ if (m->info->kind == MODEL_KIND_LATE) {
+ if (token_dim <= 0) {
+ server_log("ffwd-server: late model %s is missing token projection metadata",
+ m->info->id);
+ return -1;
+ }
+ } else if (token_dim != 0) {
+ server_log("ffwd-server: non-late model %s unexpectedly has token projection metadata",
+ m->info->id);
+ return -1;
+ }
+
+ int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
+ if (min_dim <= 0 || min_dim > config->hidden_size) {
+ server_log("ffwd-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, min_dim,
+ config->hidden_size);
+ return -1;
+ }
+ m->info->dim = config->hidden_size;
+ m->info->min_dim = min_dim;
+ m->info->token_dim = token_dim;
+ m->info->attention_mode = config->attention_mode;
+ m->info->pooling_mode = config->pooling_mode;
+ m->info->normalize_embeddings = config->normalize_embeddings;
+ m->renormalize_truncated =
+ config->normalize_embeddings && config->pooling_mode == FFWD_POOL_LAST_TOKEN;
+ return 0;
+}
+
+/* Apply the loaded model's config to its serving metadata and validate it. CPU
+ * runs this in load_one_model; a GPU build runs it on the inference thread once
+ * ffwd_activate has created the context. */
+int finalize_loaded_model(loaded_model *m) {
+ const ffwd_config_t *config = ffwd_config(m->backend);
+ int token_dim = ffwd_token_dim(m->backend);
+ ffwd_tok_set_append_terminal(m->tok, config->append_terminal_token);
+ return configure_loaded_model(m, config, token_dim);
+}
+
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec) {
+ /* Grow the model list by one entry. */
+ loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
+ if (!nm)
+ return -1;
+ s->models = nm;
+ loaded_model *m = &s->models[s->n_models];
+ memset(m, 0, sizeof(*m));
+ s->n_models++;
+
+ if (init_model_info(&m->info, spec) != 0)
+ return -1;
+ const char *path = spec->path;
+ m->path = xstrdup(path);
+ char err[256];
+ /* The tokenizer (file probe, special-token resolution, [CLS]/[SEP] vs Qwen
+ * terminal-token layout) is owned by libffwd now. */
+ m->tok = ffwd_tok_open(path, m->info->kind == MODEL_KIND_LATE, err, sizeof(err));
+ if (!m->tok) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+
+ m->backend =
+ ffwd_open(path, m->info->kind == MODEL_KIND_LATE, &s->backend_opts, err, sizeof(err));
+ if (!m->backend) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+ /* CPU loads the model in open, so its config is ready to finalize now. A GPU
+ * build defers the load - and this finalize - to the inference thread. */
+ if (ffwd_config(m->backend))
+ return finalize_loaded_model(m);
+ return 0;
+}
+
+void free_models(http_server *s) {
+ for (int i = 0; i < s->n_models; i++) {
+ loaded_model *m = &s->models[i];
+ free_model_info(m->info);
+ free(m->path);
+ ffwd_tok_free(m->tok);
+ ffwd_free(m->backend);
+ }
+ free(s->models);
+ s->models = NULL;
+ s->n_models = 0;
+}
diff --git a/tools/server/sbuf.c b/tools/server/sbuf.c
new file mode 100644
index 0000000..b341c31
--- /dev/null
+++ b/tools/server/sbuf.c
@@ -0,0 +1,74 @@
+#include "sbuf.h"
+#include "util.h"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+void sbuf_reserve(sbuf *b, size_t add) {
+ if (add > SIZE_MAX - b->len - 1)
+ die_oom();
+ size_t need = b->len + add + 1;
+ if (need <= b->cap)
+ return;
+ size_t cap = b->cap ? b->cap : 256;
+ while (cap < need) {
+ if (cap > SIZE_MAX / 2) {
+ cap = need;
+ break;
+ }
+ cap *= 2;
+ }
+ b->ptr = xrealloc(b->ptr, cap);
+ b->cap = cap;
+}
+
+void sbuf_append(sbuf *b, const void *p, size_t n) {
+ sbuf_reserve(b, n);
+ memcpy(b->ptr + b->len, p, n);
+ b->len += n;
+ b->ptr[b->len] = '\0';
+}
+
+void sbuf_putc(sbuf *b, char c) {
+ sbuf_reserve(b, 1);
+ b->ptr[b->len++] = c;
+ b->ptr[b->len] = '\0';
+}
+
+void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); }
+
+void sbuf_clear(sbuf *b) {
+ b->len = 0;
+ if (b->ptr)
+ b->ptr[0] = '\0';
+}
+
+void sbuf_printf(sbuf *b, const char *fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ for (;;) {
+ size_t avail = b->cap > b->len ? b->cap - b->len : 0;
+ va_list aq;
+ va_copy(aq, ap);
+ int n = vsnprintf(avail ? b->ptr + b->len : NULL, avail, fmt, aq);
+ va_end(aq);
+ if (n < 0) {
+ va_end(ap);
+ die_oom();
+ }
+ if ((size_t)n < avail) {
+ b->len += (size_t)n;
+ va_end(ap);
+ return;
+ }
+ sbuf_reserve(b, (size_t)n);
+ }
+}
+
+void sbuf_free(sbuf *b) {
+ free(b->ptr);
+ memset(b, 0, sizeof(*b));
+}
diff --git a/tools/server/sbuf.h b/tools/server/sbuf.h
new file mode 100644
index 0000000..3116090
--- /dev/null
+++ b/tools/server/sbuf.h
@@ -0,0 +1,24 @@
+#ifndef FFWD_SBUF_H
+#define FFWD_SBUF_H
+
+#include <stddef.h>
+
+/* Growable, always-NUL-terminated byte buffer used to build HTTP responses and
+ * JSON bodies. Growth is die-on-OOM (see server_util.h), so the mutators never
+ * fail; callers do not check return values. */
+
+typedef struct {
+ char *ptr;
+ size_t len;
+ size_t cap;
+} sbuf;
+
+void sbuf_reserve(sbuf *b, size_t add);
+void sbuf_append(sbuf *b, const void *p, size_t n);
+void sbuf_putc(sbuf *b, char c);
+void sbuf_puts(sbuf *b, const char *s);
+void sbuf_clear(sbuf *b);
+void sbuf_printf(sbuf *b, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
+void sbuf_free(sbuf *b);
+
+#endif /* FFWD_SBUF_H */
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
new file mode 100644
index 0000000..c13e0cf
--- /dev/null
+++ b/tools/server/schedule.c
@@ -0,0 +1,341 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * Job queue, micro-batching, and completion
+ *
+ * dispatch_request (http.c) enqueues one job per framed request; the
+ * inference worker thread (worker_main, server.c) drains them with
+ * collect_job_batch (a short first-arrival wait groups concurrent arrivals) and
+ * runs process_job_group, which routes each job to its endpoint handler and
+ * batches compatible ones. Finished jobs go on the done queue and wake the event
+ * loop through the completion pipe; completion_cb writes each response back on
+ * the loop thread, so sockets are only ever touched there.
+ * ======================================================================== */
+
+static void enqueue_done(job *j) {
+ http_server *s = j->srv;
+ pthread_mutex_lock(&s->mu);
+ if (s->done_tail)
+ s->done_tail->next = j;
+ else
+ s->done_head = j;
+ s->done_tail = j;
+ pthread_mutex_unlock(&s->mu);
+ const unsigned char byte = 1;
+ if (write(s->completion_pipe[1], &byte, 1) < 0) {
+ /* best-effort wakeup; the event loop also drains the done queue */
+ }
+}
+
+static job *pop_job_locked(http_server *s) {
+ job *j = s->job_head;
+ if (j) {
+ s->job_head = j->next;
+ if (!s->job_head)
+ s->job_tail = NULL;
+ j->next = NULL;
+ }
+ return j;
+}
+
+static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) {
+ uint64_t now_ns = nstime();
+ if (now_ns >= deadline_ns)
+ return ETIMEDOUT;
+
+ uint64_t remaining_ns = deadline_ns - now_ns;
+ struct timespec abs;
+ if (clock_gettime(CLOCK_REALTIME, &abs) != 0) {
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ abs.tv_sec = tv.tv_sec;
+ abs.tv_nsec = (long)tv.tv_usec * 1000L;
+ }
+ abs.tv_sec += (time_t)(remaining_ns / 1000000000u);
+ abs.tv_nsec += (long)(remaining_ns % 1000000000u);
+ if (abs.tv_nsec >= 1000000000L) {
+ abs.tv_sec++;
+ abs.tv_nsec -= 1000000000L;
+ }
+ return pthread_cond_timedwait(cv, mu, &abs);
+}
+
+int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
+ if (max_jobs <= 0)
+ return 0;
+
+ pthread_mutex_lock(&s->mu);
+ while (!s->job_head && !s->stopping)
+ pthread_cond_wait(&s->cv, &s->mu);
+
+ job *first = pop_job_locked(s);
+ if (!first) {
+ pthread_mutex_unlock(&s->mu);
+ return 0;
+ }
+
+ int n = 1;
+ jobs[0] = first;
+ int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") ||
+ !strcmp(first->path, "/v1/contextualizedembeddings"));
+ if (!batchable) {
+ pthread_mutex_unlock(&s->mu);
+ return n;
+ }
+
+ /* Exact item and token budgets are applied after worker-side tokenization;
+ * this queue stage only defines which arrival window may be grouped. */
+ uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
+ uint64_t deadline_ns =
+ first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
+ while (n < max_jobs) {
+ while (!s->job_head && !s->stopping && wait_ns > 0) {
+ int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
+ if (rc == ETIMEDOUT)
+ break;
+ }
+ if (!s->job_head)
+ break;
+ if (wait_ns > 0 && s->job_head->created_ns > deadline_ns)
+ break;
+ jobs[n++] = pop_job_locked(s);
+ if (wait_ns == 0 && !s->job_head)
+ break;
+ }
+ pthread_mutex_unlock(&s->mu);
+ return n;
+}
+
+void enqueue_job(job *j) {
+ http_server *s = j->srv;
+ pthread_mutex_lock(&s->mu);
+ if (s->job_tail)
+ s->job_tail->next = j;
+ else
+ s->job_head = j;
+ s->job_tail = j;
+ pthread_cond_signal(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+}
+
+static job *pop_done(http_server *s) {
+ pthread_mutex_lock(&s->mu);
+ job *j = s->done_head;
+ if (j) {
+ s->done_head = j->next;
+ if (!s->done_head)
+ s->done_tail = NULL;
+ j->next = NULL;
+ }
+ pthread_mutex_unlock(&s->mu);
+ return j;
+}
+
+static void job_free(job *j) {
+ if (!j)
+ return;
+ free(j->body);
+ free(j->auth);
+ free(j->extra_headers);
+ free(j->response);
+ free(j);
+}
+
+void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)loop;
+ (void)mask;
+ http_server *s = clientData;
+ unsigned char tmp[128];
+ while (read(fd, tmp, sizeof(tmp)) > 0) {
+ }
+
+ job *j;
+ while ((j = pop_done(s)) != NULL) {
+ client *c = j->c;
+ int refs_to_drop = 1; /* job reference */
+ if (!c->cancelled) {
+ append_http_response_ex(c, j->status, j->content_type, j->extra_headers,
+ j->response ? j->response : "",
+ j->response ? j->response_len : 0);
+ if (queue_write(c) != AE_OK)
+ refs_to_drop += close_client_unlink(c);
+ }
+ client_decref_n(c, refs_to_drop);
+ job_free(j);
+ }
+}
+
+static void job_set_timing_header(job *j) {
+ if (!j || j->extra_headers || !j->created_ns)
+ return;
+ uint64_t done_ns = nstime();
+ uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0;
+ uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0;
+
+ char buf[384];
+ int n = snprintf(buf, sizeof(buf),
+ "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
+ "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
+ "worker;dur=%.3f\r\n",
+ ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns),
+ ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
+ if (n > 0 && (size_t)n < sizeof(buf))
+ j->extra_headers = xstrdup(buf);
+}
+
+static int parse_job_root(job *j, cJSON **out_root) {
+ cJSON *detail = cJSON_CreateArray();
+ cJSON *root = parse_json_body(j, detail);
+ if (!root) {
+ job_set_422(j, detail);
+ cJSON_Delete(detail);
+ *out_root = NULL;
+ return -1;
+ }
+ cJSON_Delete(detail);
+ *out_root = root;
+ return 0;
+}
+
+static void process_unknown_job(job *j) {
+ cJSON *root = NULL;
+ if (!j->started_ns)
+ j->started_ns = nstime();
+ uint64_t t0 = nstime();
+ int rc = parse_job_root(j, &root);
+ j->parse_ns += nstime() - t0;
+ if (rc != 0)
+ return;
+ job_set_error(j, 404, "unknown endpoint", "invalid_request_error");
+ cJSON_Delete(root);
+}
+
+void process_job_group(http_server *s, job **jobs, int n_jobs) {
+ embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
+ contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
+ rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
+ int *kind = xcalloc((size_t)n_jobs, sizeof(*kind));
+ int *done = xcalloc((size_t)n_jobs, sizeof(*done));
+ embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
+ contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group));
+
+ for (int i = 0; i < n_jobs; i++) {
+ cJSON *root = NULL;
+ if (!jobs[i]->started_ns)
+ jobs[i]->started_ns = nstime();
+ if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
+ kind[i] = 1;
+ } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
+ kind[i] = 2;
+ } else if (!strcmp(jobs[i]->path, "/v1/rerank")) {
+ kind[i] = 3;
+ } else {
+ continue;
+ }
+ uint64_t t0 = nstime();
+ int parse_rc = parse_job_root(jobs[i], &root);
+ jobs[i]->parse_ns += nstime() - t0;
+ if (parse_rc != 0)
+ continue;
+ if (kind[i] == 1)
+ prepare_embedding_request(jobs[i], root, s, &std_reqs[i]);
+ else if (kind[i] == 2)
+ prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]);
+ else
+ prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]);
+ }
+
+ for (int i = 0; i < n_jobs; i++) {
+ if (done[i])
+ continue;
+
+ if (!kind[i]) {
+ process_unknown_job(jobs[i]);
+ job_set_timing_header(jobs[i]);
+ enqueue_done(jobs[i]);
+ done[i] = 1;
+ continue;
+ }
+
+ if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) ||
+ (kind[i] == 3 && !rerank_reqs[i].ready)) {
+ job_set_timing_header(jobs[i]);
+ enqueue_done(jobs[i]);
+ done[i] = 1;
+ continue;
+ }
+
+ if (kind[i] == 3) {
+ execute_rerank_request(&rerank_reqs[i]);
+ job_set_timing_header(jobs[i]);
+ enqueue_done(jobs[i]);
+ done[i] = 1;
+ } else if (kind[i] == 1) {
+ int group_n = 1;
+ int group_inputs = std_reqs[i].n_inputs;
+ int group_tokens = std_reqs[i].total_tokens;
+ std_group[0] = &std_reqs[i];
+ for (int k = i + 1; k < n_jobs; k++) {
+ if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
+ embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
+ embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
+ std_group[group_n++] = &std_reqs[k];
+ group_inputs += std_reqs[k].n_inputs;
+ group_tokens += std_reqs[k].total_tokens;
+ }
+ }
+ execute_embedding_request_list(std_group, group_n);
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(std_group[k] - std_reqs);
+ job_set_timing_header(jobs[idx]);
+ enqueue_done(jobs[idx]);
+ done[idx] = 1;
+ }
+ } else {
+ int group_n = 1;
+ int group_docs = ctx_reqs[i].n_docs;
+ int group_tokens = ctx_reqs[i].total_tokens;
+ ctx_group[0] = &ctx_reqs[i];
+ for (int k = i + 1; k < n_jobs; k++) {
+ if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
+ contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
+ contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
+ ctx_group[group_n++] = &ctx_reqs[k];
+ group_docs += ctx_reqs[k].n_docs;
+ group_tokens += ctx_reqs[k].total_tokens;
+ }
+ }
+ execute_contextual_request_list(ctx_group, group_n);
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(ctx_group[k] - ctx_reqs);
+ job_set_timing_header(jobs[idx]);
+ enqueue_done(jobs[idx]);
+ done[idx] = 1;
+ }
+ }
+ }
+
+ for (int i = 0; i < n_jobs; i++) {
+ embedding_request_free(&std_reqs[i]);
+ contextual_request_free(&ctx_reqs[i]);
+ rerank_request_free(&rerank_reqs[i]);
+ }
+ free(ctx_group);
+ free(std_group);
+ free(done);
+ free(kind);
+ free(rerank_reqs);
+ free(ctx_reqs);
+ free(std_reqs);
+}
diff --git a/tools/server/server.c b/tools/server/server.c
new file mode 100644
index 0000000..abff2c2
--- /dev/null
+++ b/tools/server/server.c
@@ -0,0 +1,414 @@
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+
+#include "server.h"
+#include "server_internal.h"
+#include "util.h"
+#include "sbuf.h"
+#include "ffwd.h"
+#include "ae.h"
+#include "anet.h"
+
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <netinet/in.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+# include <sys/sysctl.h>
+#endif
+
+#include <cjson/cJSON.h>
+
+static volatile sig_atomic_t g_stop_requested = 0;
+static volatile sig_atomic_t g_signal_wfd = -1;
+
+static void stop_signal_handler(int sig) {
+ (void)sig;
+ if (g_stop_requested)
+ _exit(130);
+ int saved = errno;
+ g_stop_requested = 1;
+ int fd = (int)g_signal_wfd;
+ if (fd >= 0) {
+ const unsigned char byte = 1;
+ if (write(fd, &byte, 1) < 0) {
+ /* best-effort stop wakeup; a failed self-pipe write is ignored */
+ }
+ }
+ errno = saved;
+}
+
+static bool auth_ok(http_server *s, const char *auth) {
+ if (!s->api_key || !s->api_key[0])
+ return true;
+ if (!auth)
+ return false;
+ const char prefix[] = "Bearer ";
+ return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
+ !strcmp(auth + sizeof(prefix) - 1, s->api_key);
+}
+
+void dispatch_request(client *c) {
+ aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
+
+ if (!strcmp(c->req.method, "OPTIONS")) {
+ append_http_response(c, 204, NULL, "", 0);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ if (!route_is_inference(c->req.method, c->req.path)) {
+ size_t len;
+ char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
+ append_http_response(c, 404, "application/json", body, len);
+ free(body);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ if (!auth_ok(c->srv, c->req.auth)) {
+ size_t len;
+ char *body = json_error_body("missing or invalid bearer token", "authentication_error", &len);
+ append_http_response(c, 401, "application/json", body, len);
+ free(body);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ job *j = xcalloc(1, sizeof(*j));
+ j->srv = c->srv;
+ j->c = c;
+ snprintf(j->method, sizeof(j->method), "%s", c->req.method);
+ snprintf(j->path, sizeof(j->path), "%s", c->req.path);
+ j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len);
+ j->body_len = c->req.body_len;
+ j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
+ j->created_ns = nstime();
+ client_incref(c);
+ enqueue_job(j);
+}
+
+static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)loop;
+ (void)mask;
+ http_server *s = clientData;
+ char cip[64];
+ int cport = 0;
+ for (;;) {
+ int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
+ if (cfd == ANET_ERR) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
+ if (errno == EINTR)
+ continue;
+ return;
+ }
+ set_nonblock(cfd);
+ client *c = xcalloc(1, sizeof(*c));
+ c->srv = s;
+ c->fd = cfd;
+ c->refcount = 1;
+ c->last_active_ms = mstime();
+ c->linked = 1;
+ c->next = s->clients;
+ if (s->clients)
+ s->clients->prev = c;
+ s->clients = c;
+ s->n_clients++;
+ if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
+ close_client(c);
+ }
+}
+
+static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
+ (void)loop;
+ (void)id;
+ http_server *s = clientData;
+ uint64_t now = mstime();
+ client *c = s->clients;
+ while (c) {
+ client *next = c->next;
+ /* refcount > 1 means a request from this client is queued or being
+ * processed by the worker; long batch queues (e.g. 4B long-document
+ * batches) must not be reaped as idle. */
+ if (!c->cancelled && c->refcount <= 1 &&
+ now - c->last_active_ms > FFWD_HTTP_CLIENT_TIMEOUT_MS)
+ close_client(c);
+ c = next;
+ }
+ return FFWD_HTTP_SWEEP_MS;
+}
+
+static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)fd;
+ (void)mask;
+ http_server *s = clientData;
+ unsigned char tmp[128];
+ while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
+ }
+ aeStop(loop);
+}
+
+static void *worker_main(void *arg) {
+ http_server *s = arg;
+ /* Create each model's context on this thread (a GPU build defers the load
+ * here so the context lives on the thread that uses it) and finalize the
+ * config a GPU build could not finalize at open time. CPU is loaded and
+ * finalized already, marked by info->dim being set. */
+ int rc = 0;
+ char load_err[256];
+ for (int i = 0; i < s->n_models; i++) {
+ loaded_model *m = &s->models[i];
+ if (ffwd_activate(m->backend, load_err, sizeof(load_err)) != 0) {
+ server_log("ffwd-server: failed to load model on worker: %s: %s", m->path, load_err);
+ rc = 1;
+ break;
+ }
+ if (m->info->dim == 0 && finalize_loaded_model(m) != 0) {
+ rc = 1;
+ break;
+ }
+ }
+ pthread_mutex_lock(&s->mu);
+ s->worker_init_rc = rc;
+ s->worker_ready = 1;
+ if (rc)
+ s->stopping = 1;
+ pthread_cond_broadcast(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+ if (rc) {
+ for (int i = 0; i < s->n_models; i++)
+ ffwd_worker_free(s->models[i].backend);
+ return NULL;
+ }
+
+ for (;;) {
+ job *jobs[FFWD_SERVER_MICROBATCH_MAX_JOBS];
+ int n_jobs = collect_job_batch(s, jobs, FFWD_SERVER_MICROBATCH_MAX_JOBS);
+ if (n_jobs == 0)
+ break;
+ process_job_group(s, jobs, n_jobs);
+ }
+ for (int i = 0; i < s->n_models; i++)
+ ffwd_worker_free(s->models[i].backend);
+ return NULL;
+}
+
+static int listen_on(const char *host, int port) {
+ char portbuf[32];
+ snprintf(portbuf, sizeof(portbuf), "%d", port);
+ char err[ANET_ERR_LEN];
+ int fd = anetTcpServer(err, port, (char *)host, FFWD_HTTP_BACKLOG);
+ (void)portbuf;
+ if (fd == ANET_ERR) {
+ server_log("ffwd-server: listen failed on %s:%d: %s", host, port, err);
+ return -1;
+ }
+ set_nonblock(fd);
+ return fd;
+}
+
+static int validate_model_specs(const ffwd_server_config_t *cfg) {
+ for (int i = 0; i < cfg->n_models; i++) {
+ const ffwd_server_model_spec_t *spec = &cfg->models[i];
+ if (!spec->id || !spec->id[0]) {
+ fprintf(stderr, "ffwd-server: model label must not be empty\n");
+ return -1;
+ }
+ if (!spec->path || !spec->path[0]) {
+ fprintf(stderr, "ffwd-server: model %s has an empty path\n", spec->id);
+ return -1;
+ }
+ if (spec->kind < FFWD_SERVER_MODEL_STANDARD || spec->kind > FFWD_SERVER_MODEL_LATE) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid kind\n", spec->id);
+ return -1;
+ }
+ if (spec->api < FFWD_SERVER_API_DEFAULT || spec->api > FFWD_SERVER_API_PERPLEXITY) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid api\n", spec->id);
+ return -1;
+ }
+ if (spec->min_dim < 0) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid min_dim\n", spec->id);
+ return -1;
+ }
+ for (int j = 0; j < i; j++) {
+ if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) {
+ fprintf(stderr, "ffwd-server: duplicate model label: %s\n", spec->id);
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+
+/* Hold the model paths in one array so the backend preflight can size the set
+ * against host memory in a single pass. */
+static int run_memory_preflight(const ffwd_server_config_t *cfg, const ffwd_options_t *opts) {
+ const char **paths = (const char **)malloc((size_t)cfg->n_models * sizeof(*paths));
+ if (!paths)
+ return -1;
+ for (int i = 0; i < cfg->n_models; i++)
+ paths[i] = cfg->models[i].path;
+ char err[256] = {0};
+ int rc = ffwd_preflight(paths, cfg->n_models, opts, err, sizeof(err));
+ free(paths);
+ if (rc != 0 && err[0])
+ server_log("ffwd-server: %s", err);
+ return rc;
+}
+
+int ffwd_run_server(const ffwd_server_config_t *cfg) {
+ if (!cfg || !cfg->models || cfg->n_models <= 0) {
+ fprintf(stderr, "ffwd-server: at least one model label and path is required\n");
+ return 1;
+ }
+ ffwd_options_t backend_opts = {
+ .gpu_quant_bits = cfg->gpu_quantize_bits,
+ .gpu_quant_group_size = cfg->gpu_quantize_group_size > 0 ? cfg->gpu_quantize_group_size : 64,
+ .memory_utilization = cfg->memory_utilization,
+ };
+ if (validate_model_specs(cfg) != 0 || run_memory_preflight(cfg, &backend_opts) != 0)
+ return 1;
+
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
+ s.port = cfg->port > 0 ? cfg->port : 8000;
+ s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE;
+ s.max_batch_tokens =
+ cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
+ s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
+ s.backend_opts = backend_opts;
+ if (cfg->api_key && cfg->api_key[0])
+ s.api_key = xstrdup(cfg->api_key);
+ else {
+ const char *env_key = getenv("FFWD_API_KEY");
+ if (env_key && env_key[0])
+ s.api_key = xstrdup(env_key);
+ }
+
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ for (int i = 0; i < cfg->n_models; i++) {
+ if (load_one_model(&s, &cfg->models[i]) != 0) {
+ free_models(&s);
+ return 1;
+ }
+ }
+
+ if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) {
+ perror("pipe");
+ free_models(&s);
+ return 1;
+ }
+ set_nonblock(s.completion_pipe[0]);
+ set_nonblock(s.completion_pipe[1]);
+ set_nonblock(s.signal_pipe[0]);
+ set_nonblock(s.signal_pipe[1]);
+
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_handler = stop_signal_handler;
+ sigemptyset(&sa.sa_mask);
+ sigaction(SIGINT, &sa, NULL);
+ sigaction(SIGTERM, &sa, NULL);
+ signal(SIGPIPE, SIG_IGN);
+ g_signal_wfd = s.signal_pipe[1];
+
+ s.loop = aeCreateEventLoop(FFWD_HTTP_SETSIZE);
+ if (!s.loop) {
+ fprintf(stderr, "ffwd-server: failed to create event loop\n");
+ free_models(&s);
+ return 1;
+ }
+
+ if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) {
+ fprintf(stderr, "ffwd-server: failed to start worker\n");
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ return 1;
+ }
+ pthread_mutex_lock(&s.mu);
+ while (!s.worker_ready)
+ pthread_cond_wait(&s.cv, &s.mu);
+ int worker_init_rc = s.worker_init_rc;
+ pthread_mutex_unlock(&s.mu);
+ if (worker_init_rc) {
+ pthread_join(s.worker, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
+
+ s.listen_fd = listen_on(s.host, s.port);
+ if (s.listen_fd < 0) {
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ return 1;
+ }
+
+ aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s);
+ aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s);
+ aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s);
+ aeCreateTimeEvent(s.loop, FFWD_HTTP_SWEEP_MS, timeout_cb, &s, NULL);
+
+ server_log("ffwd-server: listening on http://%s:%d", s.host, s.port);
+ aeMain(s.loop);
+
+ server_log("ffwd-server: shutdown requested");
+ if (s.listen_fd >= 0)
+ close(s.listen_fd);
+ client *c = s.clients;
+ while (c) {
+ client *next = c->next;
+ close_client(c);
+ c = next;
+ }
+
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+
+ aeDeleteEventLoop(s.loop);
+ close(s.completion_pipe[0]);
+ close(s.completion_pipe[1]);
+ close(s.signal_pipe[0]);
+ close(s.signal_pipe[1]);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 0;
+}
diff --git a/tools/server/server.h b/tools/server/server.h
new file mode 100644
index 0000000..b8ec7e4
--- /dev/null
+++ b/tools/server/server.h
@@ -0,0 +1,57 @@
+#ifndef FFWD_SERVER_H
+#define FFWD_SERVER_H
+
+/* Public-API export annotation. The libraries are built with
+ * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
+ * from the shared library; everything else stays internal. */
+#ifndef FFWD_API
+# if defined(__GNUC__)
+# define FFWD_API __attribute__((visibility("default")))
+# else
+# define FFWD_API
+# endif
+#endif
+
+#include <stdbool.h>
+
+typedef enum {
+ FFWD_SERVER_MODEL_STANDARD = 0,
+ FFWD_SERVER_MODEL_CONTEXTUAL = 1,
+ FFWD_SERVER_MODEL_LATE = 2,
+} ffwd_server_model_kind_t;
+
+typedef enum {
+ FFWD_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
+ FFWD_SERVER_API_OPENAI = 1,
+ FFWD_SERVER_API_PERPLEXITY = 2,
+} ffwd_server_embedding_api_t;
+
+typedef struct {
+ const char *id; /* served label; requests match this string exactly */
+ const char *path;
+ ffwd_server_model_kind_t kind;
+ ffwd_server_embedding_api_t api;
+ int min_dim; /* 0 means the loaded model's full hidden size */
+ const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
+} ffwd_server_model_spec_t;
+
+typedef struct {
+ const ffwd_server_model_spec_t *models;
+ int n_models;
+ const char *host;
+ int port;
+ int batch_size;
+ int max_batch_tokens;
+ int batch_wait_us;
+ /* GPU build tuning; ignored by backends that do not use it. */
+ int gpu_quantize_bits;
+ int gpu_quantize_group_size;
+ double memory_utilization; /* fraction of physical memory the GPU preflight
+ may plan for; 0 = default 0.90, above 1.0
+ allows overcommit */
+ const char *api_key;
+} ffwd_server_config_t;
+
+FFWD_API int ffwd_run_server(const ffwd_server_config_t *cfg);
+
+#endif
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
new file mode 100644
index 0000000..91b51c5
--- /dev/null
+++ b/tools/server/server_internal.h
@@ -0,0 +1,314 @@
+#ifndef FFWD_SERVER_INTERNAL_H
+#define FFWD_SERVER_INTERNAL_H
+
+/* Declarations shared across the ffwd-server translation units (HTTP framing,
+ * JSON validation, output encoding, scheduling, model lifecycle, endpoint
+ * handlers). The server is split by concern but compiles as several objects
+ * linked into one binary; this header is what they share. Not a public API -
+ * the CLI and library do not include it. */
+
+#include "ffwd.h"
+#include "server.h"
+#include "sbuf.h"
+
+#include "ae.h"
+
+#include <cjson/cJSON.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define FFWD_HTTP_IO_BUF 8192
+#define FFWD_HTTP_MAX_HEADER (64u * 1024u)
+#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define FFWD_HTTP_MAX_PATH 255u
+#define FFWD_HTTP_BACKLOG 128
+#define FFWD_HTTP_SETSIZE 10240
+#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000
+#define FFWD_HTTP_SWEEP_MS 1000
+
+#define FFWD_API_MAX_STANDARD_INPUTS 512
+#define FFWD_API_MAX_CONTEXT_DOCS 512
+#define FFWD_API_MAX_CONTEXT_CHUNKS 16000
+#define FFWD_API_MAX_RERANK_DOCUMENTS 1000
+#define FFWD_API_MAX_ITEM_TOKENS 32768
+#define FFWD_API_MAX_TOTAL_TOKENS 120000
+/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned
+ * by the tokenizer (ffwd_tok). */
+/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
+ * short-request throughput over 8 with no long-document penalty, while 128
+ * was no faster and inflated long-document tail latency. */
+#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32
+#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
+/* Microseconds the worker waits for more requests before dispatching a batch.
+ * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
+ * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
+ * Override with --batch-wait-us. */
+#define FFWD_SERVER_BATCH_WAIT_US 0
+#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000
+#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128
+#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90
+#define FFWD_MLX_RESIDENT_MULTIPLIER 2
+
+typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
+
+typedef enum {
+ FFWD_API_OPENAI = 0,
+ FFWD_API_PERPLEXITY = 1,
+} embedding_api_t;
+
+typedef struct {
+ char *id;
+ model_kind kind;
+ int dim;
+ int min_dim;
+ int token_dim;
+ ffwd_attention_mode_t attention_mode;
+ ffwd_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
+ embedding_api_t api;
+ char *query_instruct;
+} model_info;
+
+typedef struct {
+ model_info *info;
+ char *path;
+ ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
+ ffwd_t *backend; /* compute handle */
+ int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */
+} loaded_model;
+
+typedef struct client client;
+typedef struct job job;
+
+typedef struct {
+ aeEventLoop *loop;
+ int listen_fd;
+ int completion_pipe[2];
+ int signal_pipe[2];
+ const char *host;
+ int port;
+ int batch_size;
+ int max_batch_tokens;
+ int batch_wait_us;
+ char *api_key;
+ loaded_model *models;
+ int n_models;
+ ffwd_options_t backend_opts;
+
+ pthread_t worker;
+ pthread_mutex_t mu;
+ pthread_cond_t cv;
+ int stopping;
+ int worker_ready;
+ int worker_init_rc;
+ job *job_head;
+ job *job_tail;
+ job *done_head;
+ job *done_tail;
+
+ client *clients;
+ int n_clients;
+} http_server;
+
+typedef struct {
+ char method[8];
+ char path[FFWD_HTTP_MAX_PATH + 1];
+ char version[16];
+ char *body;
+ size_t body_len;
+ char *auth;
+} http_request;
+
+struct client {
+ http_server *srv;
+ int fd;
+ int refcount;
+ int cancelled;
+ int linked;
+ uint64_t last_active_ms;
+ sbuf in;
+ sbuf out;
+ size_t sent;
+ int header_done;
+ size_t header_len;
+ size_t content_length;
+ int close_after_write;
+ http_request req;
+ client *prev;
+ client *next;
+};
+
+struct job {
+ http_server *srv;
+ client *c;
+ char method[8];
+ char path[FFWD_HTTP_MAX_PATH + 1];
+ char *body;
+ size_t body_len;
+ char *auth;
+ int status;
+ char *content_type;
+ char *extra_headers;
+ char *response;
+ size_t response_len;
+ uint64_t created_ns;
+ uint64_t started_ns;
+ uint64_t parse_ns;
+ uint64_t tokenize_ns;
+ uint64_t infer_ns;
+ uint64_t encode_ns;
+ job *next;
+};
+
+/* Parsed request state, shared by the prepare (handlers), execute (schedule),
+ * and render (encode) stages. Each carries its source cJSON root (freed when
+ * the request is freed), the resolved model, and the tokenized inputs. */
+typedef struct {
+ int *ids;
+ int n_tokens;
+} token_buf;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ int n_inputs;
+ token_buf *tokens;
+ ffwd_input_t *inputs;
+ int total_tokens;
+ int ready;
+} embedding_request;
+
+typedef struct {
+ int *ids;
+ int n_tokens;
+ ffwd_span_t *spans;
+ int n_spans;
+} contextual_doc;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ contextual_doc *docs;
+ int n_docs;
+ int total_chunks;
+ int total_tokens;
+ int ready;
+} contextual_request;
+
+/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */
+typedef ffwd_late_tokens_t late_tokens;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ late_tokens query;
+ late_tokens *documents;
+ int n_documents;
+ int top_n;
+ int return_documents;
+ int query_tokens;
+ int document_tokens;
+ int ready;
+} rerank_request;
+
+/* ---- server_json.c: request validation and error bodies ---- */
+void append_json_string(sbuf *b, const char *s);
+char *json_error_body(const char *message, const char *type, size_t *len);
+void job_set_error(job *j, int status, const char *message, const char *type);
+bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type);
+void job_set_422(job *j, cJSON *detail);
+cJSON *parse_json_body(job *j, cJSON *detail);
+bool cjson_is_integer(cJSON *item);
+const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api);
+int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct);
+int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding);
+
+/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */
+int set_nonblock(int fd);
+void client_incref(client *c);
+void client_decref_n(client *c, int n);
+int close_client_unlink(client *c);
+void close_client(client *c);
+void append_http_response_ex(client *c,
+ int status,
+ const char *ctype,
+ const char *extra_headers,
+ const char *body,
+ size_t body_len);
+void append_http_response(
+ client *c, int status, const char *ctype, const char *body, size_t body_len);
+int queue_write(client *c);
+bool route_is_inference(const char *method, const char *path);
+void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
+/* Defined in server.c (request routing): framed requests are handed here. */
+void dispatch_request(client *c);
+
+/* ---- server_encode.c: embedding output encoding ---- */
+signed char quantize_int8_tanh(float x);
+char *encode_embedding(const float *emb, int dims, const char *encoding);
+void append_embedding_value(
+ sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api);
+void set_response_from_buf(job *j, sbuf *b);
+int render_embedding_response(embedding_request *r, const float *embs);
+void render_contextual_response(contextual_request *r, const float *embs);
+
+/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */
+loaded_model *loaded_model_for_label(http_server *s, const char *label);
+void free_token_bufs(token_buf *t, int n);
+void embedding_request_free(embedding_request *r);
+void contextual_request_free(contextual_request *r);
+void late_tokens_free(late_tokens *t);
+void rerank_request_free(rerank_request *r);
+/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out);
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out);
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out);
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim);
+int finalize_loaded_model(loaded_model *m);
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec);
+void free_models(http_server *s);
+
+/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
+void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out);
+int embedding_request_compatible(const embedding_request *a, const embedding_request *b);
+int embedding_request_fits_group(const embedding_request *r,
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens);
+void execute_embedding_request_list(embedding_request **reqs, int n_reqs);
+void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out);
+int contextual_request_compatible(const contextual_request *a, const contextual_request *b);
+int contextual_request_fits_group(const contextual_request *r,
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens);
+void execute_contextual_request_list(contextual_request **reqs, int n_reqs);
+void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out);
+void execute_rerank_request(rerank_request *r);
+
+/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
+void enqueue_job(job *j);
+int collect_job_batch(http_server *s, job **jobs, int max_jobs);
+void process_job_group(http_server *s, job **jobs, int n_jobs);
+void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
+
+#endif /* FFWD_SERVER_INTERNAL_H */
diff --git a/tools/server/test_base64.c b/tools/server/test_base64.c
new file mode 100644
index 0000000..3519c0c
--- /dev/null
+++ b/tools/server/test_base64.c
@@ -0,0 +1,51 @@
+/* test_base64.c - base64_encode against RFC 4648 vectors and binary inputs. */
+
+#include "base64.h"
+#include "test_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+static void test_base64_rfc4648(void) {
+ /* RFC 4648 test vectors. */
+ static const struct {
+ const char *in, *out;
+ } cases[] = {
+ {"", ""},
+ {"f", "Zg=="},
+ {"fo", "Zm8="},
+ {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="},
+ {"fooba", "Zm9vYmE="},
+ {"foobar", "Zm9vYmFy"},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in));
+ TEST_ASSERT(strcmp(got, cases[i].out) == 0);
+ free(got);
+ }
+}
+
+static void test_base64_binary_vectors(void) {
+ static const struct {
+ unsigned char bytes[5];
+ size_t n;
+ const char *out;
+ } cases[] = {
+ {{0xff}, 1, "/w=="},
+ {{0xff, 0xee}, 2, "/+4="},
+ {{0xff, 0xee, 0xdd}, 3, "/+7d"},
+ {{0x00, 0x01, 0x02, 0x03, 0x04}, 5, "AAECAwQ="},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ char *got = base64_encode(cases[i].bytes, cases[i].n);
+ TEST_ASSERT(strcmp(got, cases[i].out) == 0);
+ free(got);
+ }
+}
+
+int main(void) {
+ test_base64_rfc4648();
+ test_base64_binary_vectors();
+ return TEST_REPORT("base64");
+}
diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c
new file mode 100644
index 0000000..a7db9c4
--- /dev/null
+++ b/tools/server/test_encode.c
@@ -0,0 +1,90 @@
+/* test_encode.c - embedding quantization and the output encodings
+ * (base64_int8, base64_binary, base64 float32) plus float rendering. */
+
+#include "base64.h"
+#include "server_internal.h"
+#include "test_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+static void test_quantize_int8_tanh(void) {
+ TEST_ASSERT(quantize_int8_tanh(0.0f) == 0);
+ /* tanh saturates to +-1, scaled by 127. */
+ TEST_ASSERT(quantize_int8_tanh(100.0f) == 127);
+ TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127);
+ /* Symmetry. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f));
+ /* round(tanh(0.5) * 127) = round(58.69) = 59. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == 59);
+}
+
+static void test_encode_embedding_int8(void) {
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ signed char expect[4] = {0, 59, -59, 127};
+ char *got = encode_embedding(emb, 4, "base64_int8");
+ char *want = base64_encode((const unsigned char *)expect, 4);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_binary(void) {
+ /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */
+ float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f};
+ unsigned char expect = 0x85; /* bits 0, 2, 7 set */
+ char *got = encode_embedding(emb, 8, "base64_binary");
+ char *want = base64_encode(&expect, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+
+ /* Zero counts as non-negative (sign bit set). */
+ float zeros[8] = {0};
+ unsigned char all = 0xff;
+ got = encode_embedding(zeros, 8, "base64_binary");
+ want = base64_encode(&all, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_base64_float32(void) {
+ /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32
+ * vector - lossless, unlike the int8 formats. */
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ /* Reference bytes via memcpy: a float->byte reinterpret cast confuses the
+ * static analyzer's uninitialized-read model. */
+ unsigned char bytes[sizeof emb];
+ memcpy(bytes, emb, sizeof emb);
+ char *want = base64_encode(bytes, sizeof emb);
+ char *got = encode_embedding(emb, 4, "base64");
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_append_embedding_value_float(void) {
+ float emb[1] = {0.5f};
+
+ /* OpenAI-compatible models render the true float32 value. */
+ sbuf b = {0};
+ append_embedding_value(&b, 0, emb, 1, "float", FFWD_API_OPENAI);
+ TEST_ASSERT(strstr(b.ptr, "\"embedding\":[0.5]") != NULL);
+ sbuf_free(&b);
+
+ /* Perplexity-compatible models render the int8-decoded view. */
+ sbuf b2 = {0};
+ append_embedding_value(&b2, 0, emb, 1, "float", FFWD_API_PERPLEXITY);
+ TEST_ASSERT(strstr(b2.ptr, "0.4609375") != NULL);
+ sbuf_free(&b2);
+}
+
+int main(void) {
+ test_quantize_int8_tanh();
+ test_encode_embedding_int8();
+ test_encode_embedding_binary();
+ test_encode_embedding_base64_float32();
+ test_append_embedding_value_float();
+ return TEST_REPORT("encode");
+}
diff --git a/tools/server/test_json.c b/tools/server/test_json.c
new file mode 100644
index 0000000..76a0f45
--- /dev/null
+++ b/tools/server/test_json.c
@@ -0,0 +1,143 @@
+/* test_json.c - request validation, JSON error bodies, and 422 detail
+ * handling (encoding_format, text_type, error escaping, job_set_422). */
+
+#include "server_internal.h"
+#include "test_util.h"
+#include "util.h" /* xstrdup */
+
+#include <stdlib.h>
+#include <string.h>
+
+static const char *TEST_QWEN3_QUERY_INSTRUCT =
+ "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:";
+
+static void test_json_error_body_escaping(void) {
+ size_t len = 0;
+ char *body = json_error_body("line\nquote\" tab\t ctrl\001 slash\\", "bad\ntype", &len);
+ TEST_ASSERT(body != NULL);
+ if (!body)
+ return;
+ TEST_ASSERT(len == strlen(body));
+ TEST_ASSERT(strchr(body, '\n') == NULL);
+ TEST_ASSERT(strchr(body, '\t') == NULL);
+ cJSON *root = cJSON_Parse(body);
+ TEST_ASSERT(root != NULL);
+ if (!root) {
+ free(body);
+ return;
+ }
+ cJSON *err = cJSON_GetObjectItemCaseSensitive(root, "error");
+ cJSON *msg = err ? cJSON_GetObjectItemCaseSensitive(err, "message") : NULL;
+ cJSON *type = err ? cJSON_GetObjectItemCaseSensitive(err, "type") : NULL;
+ if (!msg || !type || !cJSON_IsString(msg) || !cJSON_IsString(type)) {
+ TEST_ASSERT(0 && "error body fields must be strings");
+ cJSON_Delete(root);
+ free(body);
+ return;
+ }
+ TEST_ASSERT(strcmp(msg->valuestring, "line\nquote\" tab\t ctrl\001 slash\\") == 0);
+ TEST_ASSERT(strcmp(type->valuestring, "bad\ntype") == 0);
+ cJSON_Delete(root);
+ free(body);
+}
+
+static void test_job_set_422_replaces_response(void) {
+ job j;
+ memset(&j, 0, sizeof(j));
+ j.response = xstrdup("old response");
+ j.response_len = strlen(j.response);
+ j.status = 500;
+
+ cJSON *detail = cJSON_CreateArray();
+ ve_add(detail, "[\"body\",\"field\"]", "bad value", "value_error");
+ job_set_422(&j, detail);
+ TEST_ASSERT(j.status == 422);
+ TEST_ASSERT(j.response != NULL);
+ if (j.response) {
+ TEST_ASSERT(j.response_len == strlen(j.response));
+ TEST_ASSERT(strstr(j.response, "bad value") != NULL);
+ }
+
+ cJSON_Delete(detail);
+ free(j.response);
+}
+
+static void test_encoding_from_root_family(void) {
+ cJSON *detail = cJSON_CreateArray();
+
+ /* Default encoding follows the explicit serving API. */
+ cJSON *empty = cJSON_CreateObject();
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_OPENAI), "float") == 0);
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_PERPLEXITY), "base64_int8") ==
+ 0);
+ cJSON_Delete(empty);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* OpenAI rejects int8 formats; Perplexity rejects the OpenAI base64. */
+ cJSON *o = cJSON_CreateObject();
+ cJSON_AddStringToObject(o, "encoding_format", "base64_int8");
+ encoding_from_root(o, detail, FFWD_API_OPENAI);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
+ cJSON_Delete(o);
+
+ cJSON *p = cJSON_CreateObject();
+ cJSON_AddStringToObject(p, "encoding_format", "base64");
+ encoding_from_root(p, detail, FFWD_API_PERPLEXITY);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
+ cJSON_Delete(p);
+
+ /* An unknown format is rejected. */
+ cJSON *bad = cJSON_CreateObject();
+ cJSON_AddStringToObject(bad, "encoding_format", "base64_fp16");
+ encoding_from_root(bad, detail, FFWD_API_OPENAI);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 3);
+ cJSON_Delete(bad);
+
+ cJSON_Delete(detail);
+}
+
+static void test_text_type(void) {
+ cJSON *detail = cJSON_CreateArray();
+
+ /* Absent: no-op for both model types, no error. */
+ cJSON *empty = cJSON_CreateObject();
+ TEST_ASSERT(text_type_is_query(empty, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0);
+ cJSON_Delete(empty);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* Qwen3: query -> 1, document -> 0, both accepted. */
+ cJSON *q = cJSON_CreateObject();
+ cJSON_AddStringToObject(q, "text_type", "query");
+ TEST_ASSERT(text_type_is_query(q, detail, TEST_QWEN3_QUERY_INSTRUCT) == 1);
+ cJSON_Delete(q);
+ cJSON *d = cJSON_CreateObject();
+ cJSON_AddStringToObject(d, "text_type", "document");
+ TEST_ASSERT(text_type_is_query(d, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ cJSON_Delete(d);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* Invalid enum value on Qwen3 is reported. */
+ cJSON *bad = cJSON_CreateObject();
+ cJSON_AddStringToObject(bad, "text_type", "passage");
+ TEST_ASSERT(text_type_is_query(bad, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
+ cJSON_Delete(bad);
+
+ /* text_type on a model with no query instruction is rejected. */
+ cJSON *p = cJSON_CreateObject();
+ cJSON_AddStringToObject(p, "text_type", "query");
+ TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
+ cJSON_Delete(p);
+
+ cJSON_Delete(detail);
+}
+
+int main(void) {
+ test_json_error_body_escaping();
+ test_job_set_422_replaces_response();
+ test_encoding_from_root_family();
+ test_text_type();
+ return TEST_REPORT("json");
+}
diff --git a/tools/server/test_models.c b/tools/server/test_models.c
new file mode 100644
index 0000000..3dce62b
--- /dev/null
+++ b/tools/server/test_models.c
@@ -0,0 +1,26 @@
+/* test_models.c - resolving a served model by its request label. */
+
+#include "server_internal.h"
+#include "test_util.h"
+
+#include <string.h>
+
+static void test_loaded_model_lookup(void) {
+ model_info info;
+ memset(&info, 0, sizeof(info));
+ info.id = "served-label";
+ loaded_model models[1];
+ memset(models, 0, sizeof(models));
+ models[0].info = &info;
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.models = models;
+ s.n_models = 1;
+ TEST_ASSERT(loaded_model_for_label(&s, "served-label") == &models[0]);
+ TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL);
+}
+
+int main(void) {
+ test_loaded_model_lookup();
+ return TEST_REPORT("models");
+}
diff --git a/tools/server/test_sbuf.c b/tools/server/test_sbuf.c
new file mode 100644
index 0000000..65d0eba
--- /dev/null
+++ b/tools/server/test_sbuf.c
@@ -0,0 +1,35 @@
+/* test_sbuf.c - growable string buffer growth, formatting, and clearing. */
+
+#include "sbuf.h"
+#include "test_util.h"
+
+#include <string.h>
+
+static void test_sbuf_growth_and_formatting(void) {
+ sbuf b = {0};
+ sbuf_puts(&b, "ab");
+ sbuf_putc(&b, 'c');
+ TEST_ASSERT(b.len == 3);
+ TEST_ASSERT(strcmp(b.ptr, "abc") == 0);
+
+ size_t old_cap = b.cap;
+ sbuf_printf(&b, "-%04d-%s", 7, "tail");
+ TEST_ASSERT(strcmp(b.ptr, "abc-0007-tail") == 0);
+ TEST_ASSERT(b.cap == old_cap);
+
+ char big[600];
+ memset(big, 'x', sizeof(big) - 1);
+ big[sizeof(big) - 1] = '\0';
+ sbuf_puts(&b, big);
+ TEST_ASSERT(b.len == strlen("abc-0007-tail") + sizeof(big) - 1);
+ TEST_ASSERT(b.ptr[b.len] == '\0');
+ sbuf_clear(&b);
+ TEST_ASSERT(b.len == 0);
+ TEST_ASSERT(b.ptr && b.ptr[0] == '\0');
+ sbuf_free(&b);
+}
+
+int main(void) {
+ test_sbuf_growth_and_formatting();
+ return TEST_REPORT("sbuf");
+}
diff --git a/tools/server/test_schedule.c b/tools/server/test_schedule.c
new file mode 100644
index 0000000..1f799f1
--- /dev/null
+++ b/tools/server/test_schedule.c
@@ -0,0 +1,73 @@
+/* test_schedule.c - job queue micro-batching: the batch-wait deadline and the
+ * zero-wait fast path through enqueue_job / collect_job_batch. */
+
+#include "server_internal.h"
+#include "test_util.h"
+#include "util.h" /* nstime */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <string.h>
+
+static void test_collect_job_batch_deadline(void) {
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.batch_size = 32;
+ s.batch_wait_us = 1000;
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ uint64_t first_ns = nstime() - 5000000u;
+ job first = {.srv = &s, .created_ns = first_ns};
+ job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u};
+ job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u};
+ snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
+ snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings");
+ snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings");
+ enqueue_job(&first);
+ enqueue_job(&before_deadline);
+ enqueue_job(&after_deadline);
+
+ job *batch[3] = {0};
+ TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2);
+ TEST_ASSERT(batch[0] == &first);
+ TEST_ASSERT(batch[1] == &before_deadline);
+ TEST_ASSERT(s.job_head == &after_deadline);
+ TEST_ASSERT(s.job_tail == &after_deadline);
+
+ s.job_head = NULL;
+ s.job_tail = NULL;
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+}
+
+static void test_collect_job_batch_zero_wait(void) {
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.batch_size = 32;
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ job first = {.srv = &s, .created_ns = nstime()};
+ job second = {.srv = &s, .created_ns = nstime()};
+ snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
+ snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings");
+ enqueue_job(&first);
+ enqueue_job(&second);
+
+ job *batch[2] = {0};
+ TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2);
+ TEST_ASSERT(batch[0] == &first);
+ TEST_ASSERT(batch[1] == &second);
+ TEST_ASSERT(s.job_head == NULL);
+ TEST_ASSERT(s.job_tail == NULL);
+
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+}
+
+int main(void) {
+ test_collect_job_batch_deadline();
+ test_collect_job_batch_zero_wait();
+ return TEST_REPORT("schedule");
+}
diff --git a/tools/server/test_util.h b/tools/server/test_util.h
new file mode 100644
index 0000000..c6b6a13
--- /dev/null
+++ b/tools/server/test_util.h
@@ -0,0 +1,26 @@
+#ifndef FFWD_SERVER_TEST_UTIL_H
+#define FFWD_SERVER_TEST_UTIL_H
+
+/* Shared harness for the ffwd-server unit tests. TEST_ASSERT records failures
+ * (rather than aborting) so a single run reports every failure, not just the
+ * first. Each test file includes this, runs its checks, and ends main with
+ * `return TEST_REPORT("label");`, which prints a summary and yields the exit
+ * code (0 when clean, 1 otherwise). */
+
+#include <stdio.h>
+
+static int test_failures = 0;
+
+#define TEST_ASSERT(cond) \
+ do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ test_failures++; \
+ } \
+ } while (0)
+
+#define TEST_REPORT(label) \
+ (test_failures ? (fprintf(stderr, "%s: %d failure(s)\n", (label), test_failures), 1) \
+ : (printf("ok: %s\n", (label)), 0))
+
+#endif /* FFWD_SERVER_TEST_UTIL_H */
diff --git a/tools/server/util.c b/tools/server/util.c
new file mode 100644
index 0000000..1ec4a2a
--- /dev/null
+++ b/tools/server/util.c
@@ -0,0 +1,82 @@
+#include "util.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+
+void die_oom(void) {
+ fprintf(stderr, "ffwd-server: out of memory\n");
+ exit(1);
+}
+
+void *xmalloc(size_t n) {
+ void *p = malloc(n ? n : 1);
+ if (!p)
+ die_oom();
+ return p;
+}
+
+void *xcalloc(size_t n, size_t sz) {
+ void *p = calloc(n ? n : 1, sz ? sz : 1);
+ if (!p)
+ die_oom();
+ return p;
+}
+
+void *xrealloc(void *p, size_t n) {
+ p = realloc(p, n ? n : 1);
+ if (!p)
+ die_oom();
+ return p;
+}
+
+char *xstrdup(const char *s) {
+ size_t n = strlen(s);
+ char *p = xmalloc(n + 1);
+ memcpy(p, s, n + 1);
+ return p;
+}
+
+char *xstrndup(const char *s, size_t n) {
+ char *p = xmalloc(n + 1);
+ memcpy(p, s, n);
+ p[n] = '\0';
+ return p;
+}
+
+uint64_t mstime(void) {
+ struct timespec ts;
+ if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0)
+ return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u;
+}
+
+uint64_t nstime(void) {
+ struct timespec ts;
+ if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0)
+ return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec;
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u;
+}
+
+double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; }
+
+void server_log(const char *fmt, ...) {
+ char ts[32];
+ time_t now = time(NULL);
+ struct tm tmv;
+ localtime_r(&now, &tmv);
+ strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv);
+ fprintf(stderr, "%s ", ts);
+ va_list ap;
+ va_start(ap, fmt);
+ vfprintf(stderr, fmt, ap);
+ va_end(ap);
+ fputc('\n', stderr);
+}
diff --git a/tools/server/util.h b/tools/server/util.h
new file mode 100644
index 0000000..3175cd0
--- /dev/null
+++ b/tools/server/util.h
@@ -0,0 +1,36 @@
+#ifndef FFWD_SERVER_UTIL_H
+#define FFWD_SERVER_UTIL_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+/* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by
+ * the ffwd-server translation units. The x* allocators abort the process on
+ * failure rather than returning NULL, so callers never null-check them; the
+ * attributes below carry that contract across translation-unit boundaries so
+ * the static analyzer does not flag the (correctly) unchecked uses. */
+
+#if defined(__GNUC__) || defined(__clang__)
+# define SERVER_NORETURN __attribute__((noreturn))
+# define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull))
+#else
+# define SERVER_NORETURN
+# define SERVER_RETURNS_NONNULL
+#endif
+
+SERVER_NORETURN void die_oom(void);
+
+SERVER_RETURNS_NONNULL void *xmalloc(size_t n);
+SERVER_RETURNS_NONNULL void *xcalloc(size_t n, size_t sz);
+SERVER_RETURNS_NONNULL void *xrealloc(void *p, size_t n);
+SERVER_RETURNS_NONNULL char *xstrdup(const char *s);
+SERVER_RETURNS_NONNULL char *xstrndup(const char *s, size_t n);
+
+uint64_t mstime(void);
+uint64_t nstime(void);
+double ns_to_ms(uint64_t ns);
+
+/* printf-style line logger: timestamp prefix, newline appended, to stderr. */
+void server_log(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
+
+#endif /* FFWD_SERVER_UTIL_H */