diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server')
| -rw-r--r-- | tools/server/Makefile | 49 | ||||
| -rw-r--r-- | tools/server/base64.c | 47 | ||||
| -rw-r--r-- | tools/server/base64.h | 10 | ||||
| -rw-r--r-- | tools/server/encode.c | 152 | ||||
| -rw-r--r-- | tools/server/handlers.c | 800 | ||||
| -rw-r--r-- | tools/server/http.c | 399 | ||||
| -rw-r--r-- | tools/server/json.c | 217 | ||||
| -rw-r--r-- | tools/server/main.c | 342 | ||||
| -rw-r--r-- | tools/server/models.c | 342 | ||||
| -rw-r--r-- | tools/server/sbuf.c | 74 | ||||
| -rw-r--r-- | tools/server/sbuf.h | 24 | ||||
| -rw-r--r-- | tools/server/schedule.c | 341 | ||||
| -rw-r--r-- | tools/server/server.c | 414 | ||||
| -rw-r--r-- | tools/server/server.h | 57 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 314 | ||||
| -rw-r--r-- | tools/server/test_base64.c | 51 | ||||
| -rw-r--r-- | tools/server/test_encode.c | 90 | ||||
| -rw-r--r-- | tools/server/test_json.c | 143 | ||||
| -rw-r--r-- | tools/server/test_models.c | 26 | ||||
| -rw-r--r-- | tools/server/test_sbuf.c | 35 | ||||
| -rw-r--r-- | tools/server/test_schedule.c | 73 | ||||
| -rw-r--r-- | tools/server/test_util.h | 26 | ||||
| -rw-r--r-- | tools/server/util.c | 82 | ||||
| -rw-r--r-- | tools/server/util.h | 36 |
24 files changed, 4144 insertions, 0 deletions
diff --git a/tools/server/Makefile b/tools/server/Makefile new file mode 100644 index 0000000..4a11b0d --- /dev/null +++ b/tools/server/Makefile @@ -0,0 +1,49 @@ +ROOT := ../.. +COMPONENT := server +include $(ROOT)/build.mk + +LIBFFWD := $(ROOT)/libffwd +AE := $(ROOT)/deps/ae + +BIN := $(OUTDIR)/ffwd-server +SRCS := base64.c encode.c handlers.c http.c json.c main.c models.c sbuf.c \ + schedule.c server.c util.c +OBJS := $(call objs,$(SRCS)) +DEPS := $(call deps,$(OBJS)) + +TEST_BINS := test_base64 test_sbuf test_encode test_json test_schedule test_models +TEST_OBJS := $(filter-out $(OBJDIR)/main.o,$(OBJS)) + +LIBS := $(OUTDIR)/libffwd.a $(OUTDIR)/libae.a +LDLIBS := $(CJSON_LDFLAGS) $(BACKEND_LDFLAGS) -lm + +CFLAGS += -std=gnu11 -pthread +CPPFLAGS += -I. -I$(LIBFFWD) -I$(AE) $(CJSON_CFLAGS) + +$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS) + +.PHONY: all clean test + +all: $(BIN) + +$(eval $(call link_program,$(BIN),$(OBJS),$(LIBS),$(LDLIBS))) + +test: $(TEST_BINS) + @rc=0; for t in $(TEST_BINS); do ./$$t || rc=1; done; \ + if [ $$rc -eq 0 ]; then echo "server ok"; else echo "server not ok"; exit 1; fi + +$(TEST_BINS): %: %.c test_util.h $(TEST_OBJS) $(LIBS) + $(CC) $(CFLAGS) $(CPPFLAGS) $< $(TEST_OBJS) $(LIBS) $(LDLIBS) $(LDFLAGS_OPT) -o $@ + +$(OUTDIR)/libffwd.a: + $(MAKE) -C $(LIBFFWD) + +$(OUTDIR)/libae.a: + $(MAKE) -C $(AE) + +clean: + $(RM) -r $(OBJDIR) $(BIN) $(BIN).dSYM \ + $(TEST_BINS) $(addsuffix .dSYM,$(TEST_BINS)) + $(MAKE) -C $(AE) clean + +-include $(DEPS) diff --git a/tools/server/base64.c b/tools/server/base64.c new file mode 100644 index 0000000..87351d2 --- /dev/null +++ b/tools/server/base64.c @@ -0,0 +1,47 @@ +#include "base64.h" +#include "util.h" + +#include <stdint.h> + +char *base64_encode(const unsigned char *src, size_t n) { + static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t max_groups = (SIZE_MAX - 1) / 4; + if (n > max_groups * 3) + die_oom(); + + size_t out_len = ((n + 2) / 3) * 4; + char *out = xmalloc(out_len + 1); + const unsigned char *p = src; + const unsigned char *end = src + (n / 3) * 3; + char *q = out; + + while (p < end) { + uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | (uint32_t)p[2]; + q[0] = tbl[(v >> 18) & 63]; + q[1] = tbl[(v >> 12) & 63]; + q[2] = tbl[(v >> 6) & 63]; + q[3] = tbl[v & 63]; + p += 3; + q += 4; + } + + size_t rem = n - (size_t)(p - src); + if (rem == 1) { + uint32_t v = (uint32_t)p[0] << 16; + q[0] = tbl[(v >> 18) & 63]; + q[1] = tbl[(v >> 12) & 63]; + q[2] = '='; + q[3] = '='; + q += 4; + } else if (rem == 2) { + uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8); + q[0] = tbl[(v >> 18) & 63]; + q[1] = tbl[(v >> 12) & 63]; + q[2] = tbl[(v >> 6) & 63]; + q[3] = '='; + q += 4; + } + + *q = '\0'; + return out; +} diff --git a/tools/server/base64.h b/tools/server/base64.h new file mode 100644 index 0000000..88d88b3 --- /dev/null +++ b/tools/server/base64.h @@ -0,0 +1,10 @@ +#ifndef FFWD_BASE64_H +#define FFWD_BASE64_H + +#include <stddef.h> + +/* Standard base64 (with '=' padding) of n bytes. Returns a NUL-terminated heap + * string the caller frees; die-on-OOM, so it never returns NULL. */ +char *base64_encode(const unsigned char *src, size_t n); + +#endif /* FFWD_BASE64_H */ diff --git a/tools/server/encode.c b/tools/server/encode.c new file mode 100644 index 0000000..faa58a0 --- /dev/null +++ b/tools/server/encode.c @@ -0,0 +1,152 @@ +#include "server_internal.h" +#include "util.h" +#include "sbuf.h" +#include "base64.h" +#include "ffwd.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +/* ======================================================================== + * Embedding output encoding + * + * Turns a pooled float vector into the requested wire form. OpenAI-compatible + * models emit true float32 arrays by default. Perplexity-compatible models + * emit tanh-quantized int8 by default, and their "float" view is the decoded + * int8 value for SDK compatibility. + * ======================================================================== */ + +signed char quantize_int8_tanh(float x) { + int v = (int)lrintf(tanhf(x) * 127.0f); + if (v > 127) + v = 127; + if (v < -128) + v = -128; + return (signed char)v; +} + +char *encode_embedding(const float *emb, int dims, const char *encoding) { + if (!strcmp(encoding, "base64")) { + /* OpenAI/DashScope (Qwen3): base64 of the raw little-endian float32 + * vector. Copy through a byte buffer instead of reinterpret-casting the + * float pointer; x86_64 and aarch64 are little-endian, so the bytes + * match the wire format with no swap. */ + size_t nbytes = (size_t)dims * sizeof(float); + unsigned char *buf = xmalloc(nbytes); + memcpy(buf, emb, nbytes); + char *out = base64_encode(buf, nbytes); + free(buf); + return out; + } + if (!strcmp(encoding, "base64_binary")) { + size_t bytes = (size_t)dims / 8; + unsigned char *bits = xcalloc(bytes, 1); + for (int i = 0; i < dims; i++) { + if (emb[i] >= 0.0f) + bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); + } + char *out = base64_encode(bits, bytes); + free(bits); + return out; + } + + signed char *q = xmalloc((size_t)dims); + for (int i = 0; i < dims; i++) + q[i] = quantize_int8_tanh(emb[i]); + char *out = base64_encode((const unsigned char *)q, (size_t)dims); + free(q); + return out; +} + +static void append_embedding_object(sbuf *b, int index, const char *embedding) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); + sbuf_puts(b, embedding); + sbuf_puts(b, "\"}"); +} + +void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { + if (!strcmp(encoding, "float")) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); + for (int i = 0; i < dims; i++) { + if (i) + sbuf_putc(b, ','); + float v = + api == FFWD_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i]; + sbuf_printf(b, "%.9g", (double)v); + } + sbuf_puts(b, "]}"); + return; + } + char *encoded = encode_embedding(emb, dims, encoding); + append_embedding_object(b, index, encoded); + free(encoded); +} + +void set_response_from_buf(job *j, sbuf *b) { + j->status = 200; + j->content_type = "application/json"; + j->response = b->ptr; + j->response_len = b->len; + memset(b, 0, sizeof(*b)); +} + +int render_embedding_response(embedding_request *r, const float *embs) { + const int full_dim = r->model->info->dim; + const float *render_embs = embs; + float *truncated = NULL; + if (r->model->renormalize_truncated && r->dims < full_dim) { + truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated)); + for (int i = 0; i < r->n_inputs; i++) { + float *dst = truncated + (size_t)i * r->dims; + memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); + if (ffwd_l2_normalize(dst, r->dims) != 0) { + free(truncated); + return -1; + } + } + render_embs = truncated; + } + int render_stride = truncated ? r->dims : full_dim; + embedding_api_t api = r->model->info->api; + + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + for (int i = 0; i < r->n_inputs; i++) { + if (i) + sbuf_putc(&b, ','); + append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, + api); + } + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); + + free(truncated); + return 0; +} + +void render_contextual_response(contextual_request *r, const float *embs) { + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + int span_offset = 0; + for (int di = 0; di < r->n_docs; di++) { + if (di) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); + for (int ci = 0; ci < r->docs[di].n_spans; ci++) { + if (ci) + sbuf_putc(&b, ','); + const float *emb = embs + (size_t)span_offset * r->model->info->dim; + append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api); + span_offset++; + } + sbuf_puts(&b, "]}"); + } + sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); +} diff --git a/tools/server/handlers.c b/tools/server/handlers.c new file mode 100644 index 0000000..6720784 --- /dev/null +++ b/tools/server/handlers.c @@ -0,0 +1,800 @@ +#include "server_internal.h" +#include "util.h" +#include "ffwd.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* ======================================================================== + * Endpoint handlers: prepare, batch, execute, render + * + * One prepare_* per endpoint validates the request JSON and tokenizes its + * inputs into a *_request; the worker (schedule.c) groups compatible + * ready requests and calls the matching execute_*_request_list, which runs the + * batched forward pass through model_ffwd_* and renders each response via + * encode.c. Rerank runs MaxSim over late-interaction token vectors. + * ======================================================================== */ + +int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +typedef struct { + ffwd_input_t input; + int output_index; +} embedding_batch_item; + +static int embedding_batch_item_cmp(const void *a, const void *b) { + const embedding_batch_item *ia = a; + const embedding_batch_item *ib = b; + if (ia->input.n_tokens < ib->input.n_tokens) + return -1; + if (ia->input.n_tokens > ib->input.n_tokens) + return 1; + return ia->output_index - ib->output_index; +} + +int embedding_request_fits_group(const embedding_request *r, + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_inputs == 0) + return 1; + return r->n_inputs <= max_batch - group_inputs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_inputs = 0; + for (int i = 0; i < n_reqs; i++) + total_inputs += reqs[i]->n_inputs; + + embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); + float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); + + int pos = 0; + for (int i = 0; i < n_reqs; i++) { + for (int k = 0; k < reqs[i]->n_inputs; k++) { + items[pos].input = reqs[i]->inputs[k]; + items[pos].output_index = pos; + pos++; + } + } + qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) + max_batch = total_inputs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + ffwd_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_inputs;) { + int cur = 0; + int tokens = 0; + while (start + cur < total_inputs && cur < max_batch) { + int n_tokens = items[start + cur].input.n_tokens; + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) + break; + inputs[cur] = items[start + cur].input; + tokens += n_tokens; + cur++; + } + uint64_t t0 = nstime(); + int ffwd_rc = model_ffwd_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (ffwd_rc != 0) { + failed = 1; + break; + } + for (int i = 0; i < cur; i++) + memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, + (size_t)dim * sizeof(float)); + start += cur; + } + + pos = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); + } else { + uint64_t t0 = nstime(); + if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) + job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); + reqs[i]->j->encode_ns += nstime() - t0; + } + pos += reqs[i]->n_inputs; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +int contextual_request_fits_group(const contextual_request *r, + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_docs == 0) + return 1; + return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; +} + +typedef struct { + ffwd_context_input_t input; + int output_span_index; + int order; +} contextual_batch_item; + +static int contextual_batch_item_cmp(const void *a, const void *b) { + const contextual_batch_item *ia = a; + const contextual_batch_item *ib = b; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) + return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) + return 1; + return ia->order - ib->order; +} + +void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_docs = 0; + int total_chunks = 0; + for (int i = 0; i < n_reqs; i++) { + if (total_docs > INT_MAX - reqs[i]->n_docs || + total_chunks > INT_MAX - reqs[i]->total_chunks) { + for (int k = 0; k < n_reqs; k++) + job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); + return; + } + total_docs += reqs[i]->n_docs; + total_chunks += reqs[i]->total_chunks; + } + if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { + for (int i = 0; i < n_reqs; i++) + job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); + return; + } + + contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); + float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); + int pos = 0; + int span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + for (int d = 0; d < reqs[i]->n_docs; d++) { + contextual_doc *doc = &reqs[i]->docs[d]; + items[pos].input.input.ids = doc->ids; + items[pos].input.input.n_tokens = doc->n_tokens; + items[pos].input.spans = doc->spans; + items[pos].input.n_spans = doc->n_spans; + items[pos].output_span_index = span_offset; + items[pos].order = pos; + span_offset += doc->n_spans; + pos++; + } + } + qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) + max_batch = total_docs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + ffwd_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc(sizeof(*batch_embs)); + size_t batch_emb_cap = 0; + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_docs;) { + int cur = 0; + int tokens = 0; + int chunks = 0; + while (start + cur < total_docs && cur < max_batch) { + const ffwd_context_input_t *input = &items[start + cur].input; + if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, + max_batch_tokens)) + break; + inputs[cur] = *input; + tokens += input->input.n_tokens; + chunks += input->n_spans; + cur++; + } + size_t needed = (size_t)chunks * dim; + if (needed > batch_emb_cap) { + batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); + batch_emb_cap = needed; + } + uint64_t t0 = nstime(); + int ffwd_rc = model_ffwd_spans_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (ffwd_rc != 0) { + failed = 1; + break; + } + int batch_span = 0; + for (int i = 0; i < cur; i++) { + int n_spans = inputs[i].n_spans; + memcpy(embs + (size_t)items[start + i].output_span_index * dim, + batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); + batch_span += n_spans; + } + start += cur; + } + + span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); + } else { + uint64_t t0 = nstime(); + render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); + reqs[i]->j->encode_ns += nstime() - t0; + } + span_offset += reqs[i]->total_chunks; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +static int validate_common(cJSON *root, + cJSON *detail, + model_kind expected_kind, + loaded_model **out_model, + int *out_dims, + const char **out_encoding, + http_server *s) { + cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); + const char *model_id = NULL; + loaded_model *m = NULL; + if (!model_item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + } else { + model_id = model_item->valuestring; + m = loaded_model_for_label(s, model_id); + if (!m) { + ve_add(detail, "[\"body\",\"model\"]", + "value is not a valid enum member for this endpoint", "enum"); + } else if (m->info->kind != expected_kind) { + ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum"); + m = NULL; + } + } + *out_model = m; + const char *enc = encoding_from_root(root, detail, m ? m->info->api : FFWD_API_OPENAI); + int min_dim = m ? m->info->min_dim : 128; + int max_dim = m ? m->info->dim : 2560; + int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); + *out_encoding = enc; + *out_dims = dims; + return 0; +} + +void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + /* The query instruction, when requested, applies to every input. */ + const char *query_instruct = m && text_type_is_query(root, detail, m->info->query_instruct) + ? m->info->query_instruct + : NULL; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_inputs = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (cJSON_IsString(input)) { + if (!input->valuestring || input->valuestring[0] == '\0') + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); + n_inputs = 1; + } else if (cJSON_IsArray(input)) { + n_inputs = cJSON_GetArraySize(input); + if (n_inputs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", + "value_error.list.min_items"); + else if (n_inputs > FFWD_API_MAX_STANDARD_INPUTS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", + "value_error.list.max_items"); + cJSON *item; + int i = 0; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); + if (!cJSON_IsString(item)) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + } else if (!item->valuestring || item->valuestring[0] == '\0') { + ve_add(detail, loc, "input item must not be empty", "value_error.empty"); + } + i++; + } + } else { + ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", + "type_error"); + } + + if (cJSON_GetArraySize(detail) == 0 && m && input) { + out->n_inputs = n_inputs; + out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); + out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); + + int idx = 0; + if (input && cJSON_IsString(input) && input->valuestring) { + if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) { + ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", + "value_error.tokenization"); + } else { + out->inputs[0].ids = out->tokens[0].ids; + out->inputs[0].n_tokens = out->tokens[0].n_tokens; + out->total_tokens = out->tokens[0].n_tokens; + if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", + "value_error.context_length"); + } + } else { + cJSON *item; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); + if (!cJSON_IsString(item) || !item->valuestring) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + idx++; + continue; + } + if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + } else { + out->inputs[idx].ids = out->tokens[idx].ids; + out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; + out->total_tokens += out->tokens[idx].n_tokens; + if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "input exceeds 32768 token limit", + "value_error.context_length"); + } + idx++; + } + } + + if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + embedding_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_docs = 0; + int total_chunks = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (!cJSON_IsArray(input)) { + ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", + "type_error.array"); + } else { + n_docs = cJSON_GetArraySize(input); + if (n_docs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", + "value_error.list.min_items"); + else if (n_docs > FFWD_API_MAX_CONTEXT_DOCS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", + "value_error.list.max_items"); + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + if (!cJSON_IsArray(doc_arr)) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must be an array of strings", "type_error.array"); + di++; + continue; + } + int doc_chunks = cJSON_GetArraySize(doc_arr); + if (doc_chunks > FFWD_API_MAX_CONTEXT_CHUNKS - total_chunks) + total_chunks = FFWD_API_MAX_CONTEXT_CHUNKS + 1; + else + total_chunks += doc_chunks; + if (doc_chunks < 1) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must contain at least 1 chunk", + "value_error.list.min_items"); + } + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (!cJSON_IsString(chunk)) { + ve_add(detail, loc, "chunk must be a string", "type_error.string"); + } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { + ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); + } + ci++; + } + di++; + } + if (total_chunks > FFWD_API_MAX_CONTEXT_CHUNKS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", + "value_error.list.max_items"); + } + + if (cJSON_GetArraySize(detail) == 0 && m) { + out->n_docs = n_docs; + out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); + int64_t request_tokens = 0; + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + contextual_doc *doc = &out->docs[di]; + int n_chunks = cJSON_GetArraySize(doc_arr); + token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); + doc->n_spans = n_chunks; + int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; + int doc_valid = 1; + + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + doc_valid = 0; + } else { + doc_tokens += chunk_tokens[ci].n_tokens; + } + ci++; + } + + if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + doc_valid = 0; + } + request_tokens += doc_tokens; + + if (doc_valid) { + doc->n_tokens = (int)doc_tokens; + doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); + int sep_id = ffwd_tok_context_separator_id(m->tok); + int pos = 0; + for (ci = 0; ci < n_chunks; ci++) { + doc->spans[ci].start = pos; + doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; + /* n_tokens > 0 implies ids != NULL (tokenize_one sets both + * together), and the guard avoids a memcpy(dst, NULL, 0) for + * an empty chunk that the static analyzer reports as UB. */ + if (chunk_tokens[ci].n_tokens > 0) + memcpy(doc->ids + pos, chunk_tokens[ci].ids, + (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); + pos += chunk_tokens[ci].n_tokens; + if (ci + 1 < n_chunks) { + doc->ids[pos++] = sep_id; + } + } + } + free_token_bufs(chunk_tokens, n_chunks); + free(chunk_tokens); + di++; + } + if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) { + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } else { + out->total_tokens = (int)request_tokens; + out->total_chunks = total_chunks; + } + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + contextual_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +static int +validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { + cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); + if (!item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + return 0; + } + if (!cJSON_IsString(item) || !item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + return 0; + } + loaded_model *m = loaded_model_for_label(s, item->valuestring); + if (!m) { + ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", + "enum"); + return 0; + } + if (m->info->kind != MODEL_KIND_LATE) { + ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum"); + return 0; + } + *out_model = m; + return 0; +} + +void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + validate_rerank_model(root, detail, s, &out->model); + cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query"); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents"); + /* query_str is set only once query passes every check; the tokenize step + * below runs only when detail is empty, so it is non-NULL there. Hoisting it + * also keeps the validated value explicit for the static analyzer. */ + const char *query_str = NULL; + if (!query) { + ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); + } else if (!cJSON_IsString(query)) { + ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); + } else if (!query->valuestring || !query->valuestring[0]) { + ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); + } else { + query_str = query->valuestring; + } + + int n_documents = 0; + if (!documents) { + ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); + } else if (!cJSON_IsArray(documents)) { + ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", + "type_error.array"); + } else { + n_documents = cJSON_GetArraySize(documents); + if (n_documents < 1) + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", + "value_error.list.min_items"); + else if (n_documents > FFWD_API_MAX_RERANK_DOCUMENTS) + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", + "value_error.list.max_items"); + cJSON *document; + int i = 0; + cJSON_ArrayForEach(document, documents) { + char loc[72]; + snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); + if (!cJSON_IsString(document)) + ve_add(detail, loc, "document must be a string", "type_error.string"); + else if (!document->valuestring || !document->valuestring[0]) + ve_add(detail, loc, "document must not be empty", "value_error.empty"); + } + } + + cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); + cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); + if (top_n && top_k) { + ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", + "value_error.conflict"); + } + cJSON *top = top_n ? top_n : top_k; + out->top_n = n_documents; + if (top) { + if (!cjson_is_integer(top)) { + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be an integer", "type_error.integer"); + } else { + out->top_n = top->valueint; + if (out->top_n < 1 || out->top_n > n_documents) + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be between 1 and the number of documents", "value_error.range"); + } + } + + cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); + if (return_documents) { + if (!cJSON_IsBool(return_documents)) + ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", + "type_error.bool"); + else + out->return_documents = cJSON_IsTrue(return_documents); + } + + if (cJSON_GetArraySize(detail) == 0 && out->model) { + out->n_documents = n_documents; + out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); + if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) { + ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); + } else { + out->query_tokens = out->query.n_tokens; + if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS) + ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", + "value_error.context_length"); + } + + cJSON *document; + int i = 0; + cJSON_ArrayForEach(document, documents) { + char loc[72]; + snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); + if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != + 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + } else { + int n = out->documents[i].n_tokens; + if (n > FFWD_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + if (out->document_tokens <= INT_MAX - n) + out->document_tokens += n; + else + out->document_tokens = INT_MAX; + } + i++; + } + if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) { + ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + rerank_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +typedef struct { + int index; + float score; +} rerank_result; + +static int rerank_result_cmp(const void *a, const void *b) { + const rerank_result *ra = a; + const rerank_result *rb = b; + if (ra->score > rb->score) + return -1; + if (ra->score < rb->score) + return 1; + return ra->index - rb->index; +} + +void execute_rerank_request(rerank_request *r) { + if (!r || !r->model) { + if (r && r->j) + job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); + return; + } + float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); + + /* Flatten the per-document fields into the arrays the backend rerank takes; + * it encodes the query and every candidate in one packed forward and scores + * MaxSim, so N short documents collapse to a single batch. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + ffwd_rerank_input_t in = { + .query_ids = r->query.ids, + .query_n_tokens = r->query.n_tokens, + .query_n_keep = r->query.n_keep, + .doc_ids = doc_ids, + .doc_n_tokens = n_tokens, + .doc_keep = keep, + .doc_n_keep = n_keep, + .n_docs = r->n_documents, + }; + uint64_t t0 = nstime(); + int rc = ffwd_rerank(r->model->backend, &in, scores); + r->j->infer_ns += nstime() - t0; + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); + if (rc != 0) { + free(scores); + job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); + return; + } + + rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); + for (int i = 0; i < r->n_documents; i++) { + ranked[i].index = i; + ranked[i].score = scores[i]; + } + qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp); + + t0 = nstime(); + sbuf b = {0}; + sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); + for (int i = 0; i < r->top_n; i++) { + if (i) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, + (double)ranked[i].score); + if (r->return_documents) { + cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); + sbuf_puts(&b, ",\"document\":"); + append_json_string(&b, document->valuestring); + } + sbuf_putc(&b, '}'); + } + sbuf_printf(&b, + "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," + "\"total_tokens\":%d}}", + r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); + set_response_from_buf(r->j, &b); + r->j->encode_ns += nstime() - t0; + free(ranked); + free(scores); +} diff --git a/tools/server/http.c b/tools/server/http.c new file mode 100644 index 0000000..a1d347f --- /dev/null +++ b/tools/server/http.c @@ -0,0 +1,399 @@ +#include "server_internal.h" +#include "util.h" + +#include <ctype.h> +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sys/types.h> +#include <unistd.h> + +/* ======================================================================== + * HTTP/1.1 connection lifecycle, header parsing, and response framing + * + * One client per connection, refcounted: the event loop holds a reference + * while the socket is open and the worker thread holds one while a job derived + * from the connection is in flight, so a request that completes after the peer + * disconnects frees the client safely. read_cb accumulates bytes until a full + * request is framed, then hands off to dispatch_request (server.c); write_cb + * drains the response and either closes or arms the next pipelined request. + * ======================================================================== */ + +int set_nonblock(int fd) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) + return -1; + return fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static void http_request_free(http_request *r) { + free(r->body); + free(r->auth); + memset(r, 0, sizeof(*r)); +} + +static void client_free(client *c) { + if (!c) + return; + http_request_free(&c->req); + sbuf_free(&c->in); + sbuf_free(&c->out); + free(c); +} + +void client_incref(client *c) { + pthread_mutex_lock(&c->srv->mu); + c->refcount++; + pthread_mutex_unlock(&c->srv->mu); +} + +static void client_decref(client *c) { + int free_now = 0; + pthread_mutex_lock(&c->srv->mu); + if (--c->refcount == 0) + free_now = 1; + pthread_mutex_unlock(&c->srv->mu); + if (free_now) + client_free(c); +} + +void client_decref_n(client *c, int n) { + int free_now = 0; + if (n <= 0) + return; + pthread_mutex_lock(&c->srv->mu); + c->refcount -= n; + if (c->refcount <= 0) + free_now = 1; + pthread_mutex_unlock(&c->srv->mu); + if (free_now) + client_free(c); +} + +static void client_unlink(client *c) { + http_server *s = c->srv; + if (!c->linked) + return; + if (c->prev) + c->prev->next = c->next; + else + s->clients = c->next; + if (c->next) + c->next->prev = c->prev; + c->prev = c->next = NULL; + c->linked = 0; + if (s->n_clients > 0) + s->n_clients--; +} + +int close_client_unlink(client *c) { + if (!c || c->cancelled) + return 0; + c->cancelled = 1; + if (c->fd >= 0) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE); + close(c->fd); + c->fd = -1; + } + client_unlink(c); + return 1; +} + +void close_client(client *c) { + if (!close_client_unlink(c)) + return; + client_decref(c); +} + +void append_http_response_ex(client *c, + int status, + const char *ctype, + const char *extra_headers, + const char *body, + size_t body_len) { + const char *reason = "OK"; + if (status == 204) + reason = "No Content"; + else if (status == 400) + reason = "Bad Request"; + else if (status == 401) + reason = "Unauthorized"; + else if (status == 404) + reason = "Not Found"; + else if (status == 405) + reason = "Method Not Allowed"; + else if (status == 422) + reason = "Unprocessable Entity"; + else if (status == 503) + reason = "Service Unavailable"; + else if (status >= 500) + reason = "Internal Server Error"; + + sbuf_printf(&c->out, + "HTTP/1.1 %d %s\r\n" + "Content-Length: %zu\r\n" + "Connection: %s\r\n", + status, reason, body_len, c->close_after_write ? "close" : "keep-alive"); + if (ctype) + sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); + + if (extra_headers) + sbuf_puts(&c->out, extra_headers); + sbuf_puts(&c->out, "\r\n"); + if (body_len) + sbuf_append(&c->out, body, body_len); +} + +void append_http_response( + client *c, int status, const char *ctype, const char *body, size_t body_len) { + append_http_response_ex(c, status, ctype, NULL, body, body_len); +} + +static ssize_t find_header_end(const char *p, size_t n) { + for (size_t i = 3; i < n; i++) { + if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n') + return (ssize_t)(i + 1); + } + for (size_t i = 1; i < n; i++) { + if (p[i - 1] == '\n' && p[i] == '\n') + return (ssize_t)(i + 1); + } + return -1; +} + +static char *header_value_dup(const char *h, size_t n, const char *name) { + size_t name_len = strlen(name); + const char *p = h, *end = h + n; + while (p < end) { + const char *line = p; + while (p < end && *p != '\n') + p++; + size_t len = (size_t)(p - line); + if (len && line[len - 1] == '\r') + len--; + if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') { + const char *v = line + name_len + 1; + while (v < line + len && isspace((unsigned char)*v)) + v++; + const char *e = line + len; + while (e > v && isspace((unsigned char)e[-1])) + e--; + return xstrndup(v, (size_t)(e - v)); + } + if (p < end) + p++; + } + return NULL; +} + +static int header_has_token(const char *value, const char *token) { + size_t token_len = strlen(token); + const char *p = value; + while (p && *p) { + while (*p == ',' || isspace((unsigned char)*p)) + p++; + const char *start = p; + while (*p && *p != ',') + p++; + const char *end = p; + while (end > start && isspace((unsigned char)end[-1])) + end--; + if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len)) + return 1; + } + return 0; +} + +static int parse_request_headers(client *c) { + char line[512]; + size_t i = 0; + while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) { + line[i] = c->in.ptr[i]; + i++; + } + line[i] = '\0'; + int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version); + if (fields < 2) + return -1; + if (fields < 3) + snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); + char *q = strchr(c->req.path, '?'); + if (q) + *q = '\0'; + + c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; + char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); + if (conn) { + if (header_has_token(conn, "close")) + c->close_after_write = 1; + else if (header_has_token(conn, "keep-alive")) + c->close_after_write = 0; + free(conn); + } + + char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length"); + if (cl) { + char *end = NULL; + errno = 0; + unsigned long long v = strtoull(cl, &end, 10); + if (errno || !end || *end || v > FFWD_HTTP_MAX_BODY) { + free(cl); + return -1; + } + c->content_length = (size_t)v; + free(cl); + } + c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization"); + return 0; +} + +static int complete_request(client *c) { + if (!c->header_done) { + ssize_t hend = find_header_end(c->in.ptr, c->in.len); + if (hend < 0) { + if (c->in.len > FFWD_HTTP_MAX_HEADER) + return -1; + return 0; + } + c->header_done = 1; + c->header_len = (size_t)hend; + if (parse_request_headers(c) != 0) + return -1; + } + if (c->content_length > FFWD_HTTP_MAX_BODY) + return -1; + if (c->in.len < c->header_len + c->content_length) + return 0; + c->req.body_len = c->content_length; + c->req.body = xmalloc(c->content_length + 1); + memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); + c->req.body[c->content_length] = '\0'; + return 1; +} + +static void send_bad_http_request(client *c) { + size_t len; + char *body = json_error_body("bad HTTP request", "invalid_request_error", &len); + c->close_after_write = 1; + append_http_response(c, 400, "application/json", body, len); + free(body); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + if (queue_write(c) != AE_OK) + close_client(c); +} + +static void reset_client_for_next_request(client *c) { + size_t consumed = c->header_len + c->content_length; + if (consumed > c->in.len) + consumed = c->in.len; + size_t remain = c->in.len - consumed; + if (remain) + memmove(c->in.ptr, c->in.ptr + consumed, remain); + c->in.len = remain; + if (c->in.ptr) + c->in.ptr[remain] = '\0'; + + http_request_free(&c->req); + sbuf_clear(&c->out); + c->sent = 0; + c->header_done = 0; + c->header_len = 0; + c->content_length = 0; + c->close_after_write = 0; +} + +static void arm_next_request(client *c) { + reset_client_for_next_request(c); + if (c->cancelled || c->fd < 0) + return; + + if (c->in.len) { + int done = complete_request(c); + if (done < 0) { + send_bad_http_request(c); + return; + } + if (done > 0) { + dispatch_request(c); + return; + } + } + + if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); +} + +static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; + (void)mask; + client *c = clientData; + while (c->sent < c->out.len) { + ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); + if (n > 0) { + c->sent += (size_t)n; + c->last_active_ms = mstime(); + continue; + } + if (n < 0 && errno == EINTR) + continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + return; + close_client(c); + return; + } + aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); + if (c->close_after_write) + close_client(c); + else + arm_next_request(c); +} + +int queue_write(client *c) { + if (!c || c->cancelled || c->fd < 0) + return -1; + return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); +} + +bool route_is_inference(const char *method, const char *path) { + return !strcmp(method, "POST") && + (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") || + !strcmp(path, "/v1/rerank")); +} + +void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; + (void)mask; + client *c = clientData; + char tmp[FFWD_HTTP_IO_BUF]; + for (;;) { + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + sbuf_append(&c->in, tmp, (size_t)n); + c->last_active_ms = mstime(); + int done = complete_request(c); + if (done < 0) { + send_bad_http_request(c); + return; + } + if (done > 0) { + dispatch_request(c); + return; + } + continue; + } + if (n == 0) { + close_client(c); + return; + } + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + close_client(c); + return; + } +} diff --git a/tools/server/json.c b/tools/server/json.c new file mode 100644 index 0000000..4812164 --- /dev/null +++ b/tools/server/json.c @@ -0,0 +1,217 @@ +#include "server_internal.h" +#include "util.h" + +#include <cjson/cJSON.h> +#include <ctype.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* ======================================================================== + * Request validation and error bodies + * + * Two JSON shapes are produced here: the OpenAI-style {"error":{message,type}} + * body (append_json_error / json_error_body / job_set_error) for transport and + * auth failures, and the Perplexity-style {"detail":[...]} 422 body built from + * ve_add entries for field validation. The parse_* / *_from_root helpers read + * and validate the request object; append_json_string escapes a value for + * inclusion in a response. */ + +static void append_json_error(sbuf *b, const char *message, const char *type) { + sbuf_puts(b, "{\"error\":{\"message\":"); + append_json_string(b, message ? message : ""); + sbuf_puts(b, ",\"type\":"); + append_json_string(b, type ? type : "server_error"); + sbuf_puts(b, "}}"); +} + +void append_json_string(sbuf *b, const char *s) { + sbuf_putc(b, '"'); + for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) { + switch (*p) { + case '"': + sbuf_puts(b, "\\\""); + break; + case '\\': + sbuf_puts(b, "\\\\"); + break; + case '\b': + sbuf_puts(b, "\\b"); + break; + case '\f': + sbuf_puts(b, "\\f"); + break; + case '\n': + sbuf_puts(b, "\\n"); + break; + case '\r': + sbuf_puts(b, "\\r"); + break; + case '\t': + sbuf_puts(b, "\\t"); + break; + default: + if (*p < 0x20) + sbuf_printf(b, "\\u%04x", (unsigned)*p); + else + sbuf_putc(b, (char)*p); + } + } + sbuf_putc(b, '"'); +} + +char *json_error_body(const char *message, const char *type, size_t *len) { + sbuf b = {0}; + append_json_error(&b, message, type); + *len = b.len; + return b.ptr; +} + +void job_set_error(job *j, int status, const char *message, const char *type) { + free(j->response); + j->status = status; + j->content_type = "application/json"; + j->response = json_error_body(message, type, &j->response_len); +} + +bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) { + cJSON *obj = cJSON_CreateObject(); + cJSON *loc = cJSON_Parse(loc_json); + if (!obj || !loc) { + cJSON_Delete(obj); + cJSON_Delete(loc); + return false; + } + cJSON_AddItemToObject(obj, "loc", loc); + cJSON_AddStringToObject(obj, "msg", msg ? msg : ""); + cJSON_AddStringToObject(obj, "type", type ? type : ""); + cJSON_AddItemToArray(detail, obj); + return true; +} + +void job_set_422(job *j, cJSON *detail) { + cJSON *root = cJSON_CreateObject(); + cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1)); + char *s = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + if (!s) { + job_set_error(j, 500, "failed to render validation error", "server_error"); + return; + } + free(j->response); + j->status = 422; + j->content_type = "application/json"; + j->response = xstrdup(s); + j->response_len = strlen(j->response); + cJSON_free(s); +} + +cJSON *parse_json_body(job *j, cJSON *detail) { + if (!j->body || j->body_len == 0) { + ve_add(detail, "[\"body\"]", "body required", "missing"); + return NULL; + } + const char *end = NULL; + cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false); + if (!root) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + return NULL; + } + const char *body_end = j->body + j->body_len; + while (end && end < body_end && isspace((unsigned char)*end)) + end++; + if (!end || end != body_end) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + cJSON_Delete(root); + return NULL; + } + if (!cJSON_IsObject(root)) { + ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object"); + cJSON_Delete(root); + return NULL; + } + return root; +} + +bool cjson_is_integer(cJSON *item) { + if (!cJSON_IsNumber(item)) + return false; + double d = item->valuedouble; + int i = item->valueint; + return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; +} + +const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) { + const char *dflt = api == FFWD_API_PERPLEXITY ? "base64_int8" : "float"; + cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); + if (!encoding) + return dflt; + if (!cJSON_IsString(encoding) || !encoding->valuestring) { + ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string", + "type_error.string"); + return dflt; + } + const char *v = encoding->valuestring; + int ok = api == FFWD_API_PERPLEXITY + ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || !strcmp(v, "float")) + : (!strcmp(v, "float") || !strcmp(v, "base64")); + if (!ok) { + ve_add(detail, "[\"body\",\"encoding_format\"]", + api == FFWD_API_PERPLEXITY + ? "value is not a valid enum member; permitted: " + "'base64_int8', 'base64_binary', 'float'" + : "value is not a valid enum member; permitted: 'float', 'base64'", + "enum"); + return dflt; + } + return v; +} + +/* Parse the `text_type` hint. Returns 1 when the model's query instruction + * should be prepended, 0 otherwise (document / absent). Models without a + * published query instruction reject the field. */ +int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) { + cJSON *tt = cJSON_GetObjectItemCaseSensitive(root, "text_type"); + if (!tt) + return 0; + if (!query_instruct) { + ve_add(detail, "[\"body\",\"text_type\"]", + "text_type is only supported for instruction embedding models", "extra_fields"); + return 0; + } + if (!cJSON_IsString(tt) || !tt->valuestring) { + ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string", "type_error.string"); + return 0; + } + const char *v = tt->valuestring; + if (!strcmp(v, "query")) + return 1; + if (!strcmp(v, "document")) + return 0; + ve_add(detail, "[\"body\",\"text_type\"]", + "value is not a valid enum member; permitted: 'query', 'document'", "enum"); + return 0; +} + +int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) { + int dims = max_dim; + cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); + if (dimensions) { + if (!cjson_is_integer(dimensions)) { + ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer", + "type_error.integer"); + } else { + dims = dimensions->valueint; + if (dims < min_dim || dims > max_dim) { + char msg[96]; + snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, max_dim); + ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); + } + } + } + if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) + ve_add(detail, "[\"body\",\"dimensions\"]", + "dimensions must be divisible by 8 for base64_binary", "value_error.divisible"); + return dims; +} diff --git a/tools/server/main.c b/tools/server/main.c new file mode 100644 index 0000000..1cbdc3b --- /dev/null +++ b/tools/server/main.c @@ -0,0 +1,342 @@ +/* main.c - ffwd-server entry point. + * + * The frontend for the ffwd-server binary: argument parsing, model-spec + * handling, and main(). The server logic lives in server.c; this file only + * builds the ffwd_server_config_t and calls ffwd_run_server. It is split + * from server.c so the test suite can include server.c (for its internal + * functions) without a conflicting main() and without a compile-out guard. */ +#include "build.h" +#include "server.h" +#include "server_internal.h" +#include "ffwd.h" +#include "internal.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static void print_usage(const char *prog) { + fprintf(stderr, + "Usage: %s --model LABEL=DIR [options]\n" + "\n" + "Options:\n" + " --model LABEL=DIR[:key=val]\n" + " Standard embedding model (repeatable)\n" + " --contextual-model LABEL=DIR[:key=val]\n" + " Contextual embedding model (repeatable)\n" + " --late-model LABEL=DIR[:key=val]\n" + " Late-interaction rerank model (repeatable)\n" + " keys: api=openai|perplexity, min_dim=N,\n" + " query=TEXT (last key; \\n escapes newline)\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 32)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N First-arrival micro-batch deadline in us\n" + " (default: %d)\n", + prog, ffwd_default_batch_wait_us()); + ffwd_server_help(stderr); + fprintf(stderr, + " -v, --verbose Verbose (-vv for debug)\n" + " -V, --version Print version and exit\n" + " --build-info Print build details and exit\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model docs=./model --port 8000\n" + " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n", + prog, prog); +} + +typedef struct { + ffwd_server_model_spec_t *v; + int n; + int cap; +} model_specs_t; + +static char *dup_range(const char *p, size_t n) { + char *s = (char *)malloc(n + 1); + if (!s) + return NULL; + memcpy(s, p, n); + s[n] = '\0'; + return s; +} + +static char *unescape_model_option(const char *s) { + size_t n = strlen(s); + char *out = (char *)malloc(n + 1); + if (!out) + return NULL; + size_t w = 0; + for (size_t r = 0; r < n; r++) { + if (s[r] == '\\' && r + 1 < n) { + r++; + if (s[r] == 'n') + out[w++] = '\n'; + else if (s[r] == 't') + out[w++] = '\t'; + else if (s[r] == 'r') + out[w++] = '\r'; + else + out[w++] = s[r]; + } else { + out[w++] = s[r]; + } + } + out[w] = '\0'; + return out; +} + +static int parse_model_option(ffwd_server_model_spec_t *spec, const char *opt, const char *flag) { + const char *eq = strchr(opt, '='); + if (!eq || eq == opt || !eq[1]) { + fprintf(stderr, "%s option expects key=value: %s\n", flag, opt); + return -1; + } + size_t key_len = (size_t)(eq - opt); + const char *val = eq + 1; + if (key_len == 3 && strncmp(opt, "api", key_len) == 0) { + if (!strcmp(val, "openai")) { + spec->api = FFWD_SERVER_API_OPENAI; + } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) { + spec->api = FFWD_SERVER_API_PERPLEXITY; + } else if (!strcmp(val, "default")) { + spec->api = FFWD_SERVER_API_DEFAULT; + } else { + fprintf(stderr, "%s api must be openai or perplexity\n", flag); + return -1; + } + return 0; + } + if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) { + char *end = NULL; + long v = strtol(val, &end, 10); + if (!end || *end || v <= 0 || v > INT_MAX) { + fprintf(stderr, "%s min_dim must be a positive integer\n", flag); + return -1; + } + spec->min_dim = (int)v; + return 0; + } + fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt); + return -1; +} + +static int append_model_spec(model_specs_t *specs, + const char *arg, + ffwd_server_model_kind_t kind, + const char *flag) { + const char *eq = strchr(arg, '='); + if (!eq || eq == arg || !eq[1]) { + fprintf(stderr, "%s expects LABEL=DIR\n", flag); + return -1; + } + if (specs->n == specs->cap) { + int new_cap = specs->cap ? specs->cap * 2 : 4; + void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); + if (!p) + return -1; + specs->v = p; + specs->cap = new_cap; + } + size_t id_len = (size_t)(eq - arg); + const char *path_start = eq + 1; + const char *opts = strchr(path_start, ':'); + size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start); + if (path_len == 0) { + fprintf(stderr, "%s expects a non-empty DIR\n", flag); + return -1; + } + ffwd_server_model_spec_t spec; + memset(&spec, 0, sizeof(spec)); + spec.id = dup_range(arg, id_len); + spec.path = dup_range(path_start, path_len); + spec.kind = kind; + if (!spec.id || !spec.path) { + free((char *)spec.id); + free((char *)spec.path); + return -1; + } + const char *p = opts ? opts + 1 : NULL; + while (p && *p) { + if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) { + const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15); + spec.query_instruct = unescape_model_option(val); + if (!spec.query_instruct) + goto fail; + break; + } + const char *next = strchr(p, ':'); + size_t opt_len = next ? (size_t)(next - p) : strlen(p); + char *opt = dup_range(p, opt_len); + if (!opt) + goto fail; + int rc = parse_model_option(&spec, opt, flag); + free(opt); + if (rc != 0) + goto fail; + p = next ? next + 1 : NULL; + } + specs->v[specs->n] = spec; + specs->n++; + return 0; + +fail: + free((char *)spec.id); + free((char *)spec.path); + free((char *)spec.query_instruct); + return -1; +} + +static void free_model_specs(model_specs_t *specs) { + if (!specs) + return; + for (int i = 0; i < specs->n; i++) { + free((char *)specs->v[i].id); + free((char *)specs->v[i].path); + free((char *)specs->v[i].query_instruct); + } + free(specs->v); +} + +int main(int argc, char *argv[]) { + int verbose = 0; + int batch_size = 0; + int max_batch_tokens = 0; + int batch_wait_us = -1; + /* Backend tuning flags parse into opts regardless of backend; the linked + * backend uses what it supports. */ + ffwd_options_t opts = {.gpu_quant_group_size = 64}; + const char *host = "127.0.0.1"; + const char *api_key = NULL; + int port = 8000; + model_specs_t model_specs = {0}; + + const char *prog = ffwd_prog_name(argv[0]); + int arg = 1; + while (arg < argc && argv[arg][0] == '-') { + const char *f = argv[arg]; + if (!strcmp(f, "-V") || !strcmp(f, "--version")) { + ffwd_print_version(prog); + free_model_specs(&model_specs); + return 0; + } else if (!strcmp(f, "--build-info")) { + ffwd_print_build_info(prog); + free_model_specs(&model_specs); + return 0; + } else if (!strcmp(f, "--model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_STANDARD, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--contextual-model")) { + if (arg + 1 >= argc || append_model_spec(&model_specs, argv[++arg], + FFWD_SERVER_MODEL_CONTEXTUAL, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--late-model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_LATE, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--gpu-quant-bits")) { + opts.gpu_quant_bits = atoi(argv[++arg]); + } else if (!strcmp(f, "--gpu-quant-group-size")) { + opts.gpu_quant_group_size = atoi(argv[++arg]); + } else if (!strcmp(f, "--memory-utilization")) { + opts.memory_utilization = atof(argv[++arg]); + if (opts.memory_utilization <= 0.0) { + fprintf(stderr, "--memory-utilization must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--gpu-gemm-mode")) { + opts.gpu_gemm_mode = argv[++arg]; + } else if (!strcmp(f, "--gpu-weight-dtype")) { + opts.gpu_weight_dtype = argv[++arg]; + } else if (!strcmp(f, "--host")) { + host = argv[++arg]; + } else if (!strcmp(f, "--port")) { + port = atoi(argv[++arg]); + } else if (!strcmp(f, "--api-key")) { + api_key = argv[++arg]; + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg]); + } else if (!strcmp(f, "--max-batch-tokens")) { + max_batch_tokens = atoi(argv[++arg]); + if (max_batch_tokens <= 0) { + fprintf(stderr, "--max-batch-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--batch-wait-us")) { + batch_wait_us = atoi(argv[++arg]); + if (batch_wait_us < 0) { + fprintf(stderr, "--batch-wait-us must be >= 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + opts.n_threads = atoi(argv[++arg]); + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(prog); + free_model_specs(&model_specs); + return 0; + } else { + fprintf(stderr, "unknown option: %s\n", f); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + arg++; + } + + ffwd_verbose = verbose; + + if (arg < argc) { + fprintf(stderr, "unexpected argument: %s\n", argv[arg]); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + if (model_specs.n == 0) { + fprintf(stderr, "at least one --model LABEL=DIR is required\n"); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + char err[256]; + if (ffwd_init(&opts, err, sizeof(err)) != 0) { + fprintf(stderr, "%s\n", err); + free_model_specs(&model_specs); + return 1; + } + + ffwd_server_config_t scfg = { + .models = model_specs.v, + .n_models = model_specs.n, + .host = host, + .port = port, + .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE, + .max_batch_tokens = max_batch_tokens, + .batch_wait_us = batch_wait_us, + .gpu_quantize_bits = opts.gpu_quant_bits, + .gpu_quantize_group_size = opts.gpu_quant_group_size, + .memory_utilization = opts.memory_utilization, + .api_key = api_key, + }; + int rc = ffwd_run_server(&scfg); + free_model_specs(&model_specs); + return rc; +} diff --git a/tools/server/models.c b/tools/server/models.c new file mode 100644 index 0000000..8fdc590 --- /dev/null +++ b/tools/server/models.c @@ -0,0 +1,342 @@ +#include "server_internal.h" +#include "util.h" + +#include <errno.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +/* ======================================================================== + * Model load/lifecycle, and tokenize/inference dispatch + * + * Each loaded model owns resolved serving metadata for the operator-chosen + * label. Transformer behavior comes from the model files through ffwd_config; + * server-only facts absent from those files, such as output API family and + * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_* + * helpers are the single dispatch point the request handlers call, hiding the + * compiled backend choice behind one signature. + * ======================================================================== */ + +loaded_model *loaded_model_for_label(http_server *s, const char *label) { + if (!label) + return NULL; + for (int i = 0; i < s->n_models; i++) { + if (s->models[i].info && !strcmp(label, s->models[i].info->id)) + return &s->models[i]; + } + return NULL; +} + +static model_kind model_kind_from_public(ffwd_server_model_kind_t kind) { + switch (kind) { + case FFWD_SERVER_MODEL_CONTEXTUAL: + return MODEL_KIND_CONTEXTUAL; + case FFWD_SERVER_MODEL_LATE: + return MODEL_KIND_LATE; + case FFWD_SERVER_MODEL_STANDARD: + default: + return MODEL_KIND_STANDARD; + } +} + +static embedding_api_t embedding_api_from_public(ffwd_server_embedding_api_t api) { + return api == FFWD_SERVER_API_PERPLEXITY ? FFWD_API_PERPLEXITY : FFWD_API_OPENAI; +} + +static int read_optional_text_file(const char *model_dir, const char *relative, char **out) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); + if (n < 0 || (size_t)n >= sizeof(path)) { + server_log("ffwd-server: model path too long"); + return -1; + } + FILE *f = fopen(path, "rb"); + if (!f && errno == ENOENT) + return 0; + if (!f) { + server_log("ffwd-server: failed to open %s", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + free(buf); + fclose(f); + return -1; + } + fclose(f); + buf[sz] = '\0'; + *out = buf; + return 1; +} + +static int read_query_instruct_from_files(const char *model_dir, char **out) { + char *buf = NULL; + int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf); + if (rc <= 0) + return rc; + cJSON *root = cJSON_Parse(buf); + free(buf); + if (!root) { + server_log("ffwd-server: invalid config_sentence_transformers.json: %s", model_dir); + return -1; + } + cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts"); + cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL; + if (query && !cJSON_IsString(query)) { + cJSON_Delete(root); + server_log("ffwd-server: prompts.query must be a string: %s", model_dir); + return -1; + } + if (query && query->valuestring && query->valuestring[0]) { + *out = xstrdup(query->valuestring); + cJSON_Delete(root); + return *out ? 1 : -1; + } + cJSON_Delete(root); + return 0; +} + +static int init_model_info(model_info **out, const ffwd_server_model_spec_t *spec) { + model_info *info = xcalloc(1, sizeof(*info)); + info->id = xstrdup(spec->id); + info->kind = model_kind_from_public(spec->kind); + info->api = embedding_api_from_public(spec->api); + info->min_dim = spec->min_dim; + if (spec->query_instruct) { + info->query_instruct = xstrdup(spec->query_instruct); + } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) { + free(info->id); + free(info); + return -1; + } + if (!info->id || (spec->query_instruct && !info->query_instruct)) { + free(info->query_instruct); + free(info->id); + free(info); + return -1; + } + *out = info; + return 0; +} + +static void free_model_info(model_info *info) { + if (!info) + return; + free(info->id); + free(info->query_instruct); + free(info); +} + +void free_token_bufs(token_buf *t, int n) { + if (!t) + return; + for (int i = 0; i < n; i++) + free(t[i].ids); +} + +void embedding_request_free(embedding_request *r) { + if (!r) + return; + free_token_bufs(r->tokens, r->n_inputs); + free(r->tokens); + free(r->inputs); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +void contextual_request_free(contextual_request *r) { + if (!r) + return; + for (int i = 0; i < r->n_docs; i++) { + free(r->docs[i].ids); + free(r->docs[i].spans); + } + free(r->docs); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +void late_tokens_free(late_tokens *t) { + if (!t) + return; + free(t->ids); + free(t->keep); + memset(t, 0, sizeof(*t)); +} + +void rerank_request_free(rerank_request *r) { + if (!r) + return; + late_tokens_free(&r->query); + for (int i = 0; i < r->n_documents; i++) + late_tokens_free(&r->documents[i]); + free(r->documents); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only + * add the server's per-stage timing onto the job; the request handlers call + * them unchanged. */ +int tokenize_input( + loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { + uint64_t t0 = nstime(); + out->ids = ffwd_tokenize(m->tok, text, query_instruct, &out->n_tokens); + if (j) + j->tokenize_ns += nstime() - t0; + if (!out->ids) { + memset(out, 0, sizeof(*out)); + return -1; + } + return 0; +} + +int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { + return tokenize_input(m, j, text, NULL, out); +} + +int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { + uint64_t t0 = nstime(); + int rc = ffwd_tokenize_late(m->tok, text, is_query, out); + if (j) + j->tokenize_ns += nstime() - t0; + return rc; +} + +int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_encode_batch(m->backend, inputs, batch, out); +} + +int model_ffwd_spans_batch(loaded_model *m, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_encode_spans_batch(m->backend, inputs, batch, out); +} + +static int model_uses_dense_batches(const loaded_model *m) { + return ffwd_uses_dense_batches(m->backend); +} + +/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX + * pads each dense row to the longest input in the chunk. */ +int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { + if (batch == 0) + return 1; + if (model_uses_dense_batches(m)) + return next_tokens <= max_batch_tokens / (batch + 1); + return next_tokens <= max_batch_tokens - packed_tokens; +} + +int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim) { + if (!m || !m->info || !config || config->hidden_size <= 0) { + if (m && m->info) + server_log("ffwd-server: model %s has invalid config", m->info->id); + return -1; + } + if (m->info->kind == MODEL_KIND_LATE) { + if (token_dim <= 0) { + server_log("ffwd-server: late model %s is missing token projection metadata", + m->info->id); + return -1; + } + } else if (token_dim != 0) { + server_log("ffwd-server: non-late model %s unexpectedly has token projection metadata", + m->info->id); + return -1; + } + + int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size; + if (min_dim <= 0 || min_dim > config->hidden_size) { + server_log("ffwd-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, min_dim, + config->hidden_size); + return -1; + } + m->info->dim = config->hidden_size; + m->info->min_dim = min_dim; + m->info->token_dim = token_dim; + m->info->attention_mode = config->attention_mode; + m->info->pooling_mode = config->pooling_mode; + m->info->normalize_embeddings = config->normalize_embeddings; + m->renormalize_truncated = + config->normalize_embeddings && config->pooling_mode == FFWD_POOL_LAST_TOKEN; + return 0; +} + +/* Apply the loaded model's config to its serving metadata and validate it. CPU + * runs this in load_one_model; a GPU build runs it on the inference thread once + * ffwd_activate has created the context. */ +int finalize_loaded_model(loaded_model *m) { + const ffwd_config_t *config = ffwd_config(m->backend); + int token_dim = ffwd_token_dim(m->backend); + ffwd_tok_set_append_terminal(m->tok, config->append_terminal_token); + return configure_loaded_model(m, config, token_dim); +} + +int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec) { + /* Grow the model list by one entry. */ + loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models)); + if (!nm) + return -1; + s->models = nm; + loaded_model *m = &s->models[s->n_models]; + memset(m, 0, sizeof(*m)); + s->n_models++; + + if (init_model_info(&m->info, spec) != 0) + return -1; + const char *path = spec->path; + m->path = xstrdup(path); + char err[256]; + /* The tokenizer (file probe, special-token resolution, [CLS]/[SEP] vs Qwen + * terminal-token layout) is owned by libffwd now. */ + m->tok = ffwd_tok_open(path, m->info->kind == MODEL_KIND_LATE, err, sizeof(err)); + if (!m->tok) { + server_log("ffwd-server: %s: %s", path, err); + return -1; + } + + m->backend = + ffwd_open(path, m->info->kind == MODEL_KIND_LATE, &s->backend_opts, err, sizeof(err)); + if (!m->backend) { + server_log("ffwd-server: %s: %s", path, err); + return -1; + } + /* CPU loads the model in open, so its config is ready to finalize now. A GPU + * build defers the load - and this finalize - to the inference thread. */ + if (ffwd_config(m->backend)) + return finalize_loaded_model(m); + return 0; +} + +void free_models(http_server *s) { + for (int i = 0; i < s->n_models; i++) { + loaded_model *m = &s->models[i]; + free_model_info(m->info); + free(m->path); + ffwd_tok_free(m->tok); + ffwd_free(m->backend); + } + free(s->models); + s->models = NULL; + s->n_models = 0; +} diff --git a/tools/server/sbuf.c b/tools/server/sbuf.c new file mode 100644 index 0000000..b341c31 --- /dev/null +++ b/tools/server/sbuf.c @@ -0,0 +1,74 @@ +#include "sbuf.h" +#include "util.h" + +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +void sbuf_reserve(sbuf *b, size_t add) { + if (add > SIZE_MAX - b->len - 1) + die_oom(); + size_t need = b->len + add + 1; + if (need <= b->cap) + return; + size_t cap = b->cap ? b->cap : 256; + while (cap < need) { + if (cap > SIZE_MAX / 2) { + cap = need; + break; + } + cap *= 2; + } + b->ptr = xrealloc(b->ptr, cap); + b->cap = cap; +} + +void sbuf_append(sbuf *b, const void *p, size_t n) { + sbuf_reserve(b, n); + memcpy(b->ptr + b->len, p, n); + b->len += n; + b->ptr[b->len] = '\0'; +} + +void sbuf_putc(sbuf *b, char c) { + sbuf_reserve(b, 1); + b->ptr[b->len++] = c; + b->ptr[b->len] = '\0'; +} + +void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } + +void sbuf_clear(sbuf *b) { + b->len = 0; + if (b->ptr) + b->ptr[0] = '\0'; +} + +void sbuf_printf(sbuf *b, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + for (;;) { + size_t avail = b->cap > b->len ? b->cap - b->len : 0; + va_list aq; + va_copy(aq, ap); + int n = vsnprintf(avail ? b->ptr + b->len : NULL, avail, fmt, aq); + va_end(aq); + if (n < 0) { + va_end(ap); + die_oom(); + } + if ((size_t)n < avail) { + b->len += (size_t)n; + va_end(ap); + return; + } + sbuf_reserve(b, (size_t)n); + } +} + +void sbuf_free(sbuf *b) { + free(b->ptr); + memset(b, 0, sizeof(*b)); +} diff --git a/tools/server/sbuf.h b/tools/server/sbuf.h new file mode 100644 index 0000000..3116090 --- /dev/null +++ b/tools/server/sbuf.h @@ -0,0 +1,24 @@ +#ifndef FFWD_SBUF_H +#define FFWD_SBUF_H + +#include <stddef.h> + +/* Growable, always-NUL-terminated byte buffer used to build HTTP responses and + * JSON bodies. Growth is die-on-OOM (see server_util.h), so the mutators never + * fail; callers do not check return values. */ + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} sbuf; + +void sbuf_reserve(sbuf *b, size_t add); +void sbuf_append(sbuf *b, const void *p, size_t n); +void sbuf_putc(sbuf *b, char c); +void sbuf_puts(sbuf *b, const char *s); +void sbuf_clear(sbuf *b); +void sbuf_printf(sbuf *b, const char *fmt, ...) __attribute__((format(printf, 2, 3))); +void sbuf_free(sbuf *b); + +#endif /* FFWD_SBUF_H */ diff --git a/tools/server/schedule.c b/tools/server/schedule.c new file mode 100644 index 0000000..c13e0cf --- /dev/null +++ b/tools/server/schedule.c @@ -0,0 +1,341 @@ +#include "server_internal.h" +#include "util.h" + +#include <errno.h> +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <unistd.h> + +/* ======================================================================== + * Job queue, micro-batching, and completion + * + * dispatch_request (http.c) enqueues one job per framed request; the + * inference worker thread (worker_main, server.c) drains them with + * collect_job_batch (a short first-arrival wait groups concurrent arrivals) and + * runs process_job_group, which routes each job to its endpoint handler and + * batches compatible ones. Finished jobs go on the done queue and wake the event + * loop through the completion pipe; completion_cb writes each response back on + * the loop thread, so sockets are only ever touched there. + * ======================================================================== */ + +static void enqueue_done(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->done_tail) + s->done_tail->next = j; + else + s->done_head = j; + s->done_tail = j; + pthread_mutex_unlock(&s->mu); + const unsigned char byte = 1; + if (write(s->completion_pipe[1], &byte, 1) < 0) { + /* best-effort wakeup; the event loop also drains the done queue */ + } +} + +static job *pop_job_locked(http_server *s) { + job *j = s->job_head; + if (j) { + s->job_head = j->next; + if (!s->job_head) + s->job_tail = NULL; + j->next = NULL; + } + return j; +} + +static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) { + uint64_t now_ns = nstime(); + if (now_ns >= deadline_ns) + return ETIMEDOUT; + + uint64_t remaining_ns = deadline_ns - now_ns; + struct timespec abs; + if (clock_gettime(CLOCK_REALTIME, &abs) != 0) { + struct timeval tv; + gettimeofday(&tv, NULL); + abs.tv_sec = tv.tv_sec; + abs.tv_nsec = (long)tv.tv_usec * 1000L; + } + abs.tv_sec += (time_t)(remaining_ns / 1000000000u); + abs.tv_nsec += (long)(remaining_ns % 1000000000u); + if (abs.tv_nsec >= 1000000000L) { + abs.tv_sec++; + abs.tv_nsec -= 1000000000L; + } + return pthread_cond_timedwait(cv, mu, &abs); +} + +int collect_job_batch(http_server *s, job **jobs, int max_jobs) { + if (max_jobs <= 0) + return 0; + + pthread_mutex_lock(&s->mu); + while (!s->job_head && !s->stopping) + pthread_cond_wait(&s->cv, &s->mu); + + job *first = pop_job_locked(s); + if (!first) { + pthread_mutex_unlock(&s->mu); + return 0; + } + + int n = 1; + jobs[0] = first; + int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") || + !strcmp(first->path, "/v1/contextualizedembeddings")); + if (!batchable) { + pthread_mutex_unlock(&s->mu); + return n; + } + + /* Exact item and token budgets are applied after worker-side tokenization; + * this queue stage only defines which arrival window may be grouped. */ + uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; + uint64_t deadline_ns = + first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns; + while (n < max_jobs) { + while (!s->job_head && !s->stopping && wait_ns > 0) { + int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); + if (rc == ETIMEDOUT) + break; + } + if (!s->job_head) + break; + if (wait_ns > 0 && s->job_head->created_ns > deadline_ns) + break; + jobs[n++] = pop_job_locked(s); + if (wait_ns == 0 && !s->job_head) + break; + } + pthread_mutex_unlock(&s->mu); + return n; +} + +void enqueue_job(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->job_tail) + s->job_tail->next = j; + else + s->job_head = j; + s->job_tail = j; + pthread_cond_signal(&s->cv); + pthread_mutex_unlock(&s->mu); +} + +static job *pop_done(http_server *s) { + pthread_mutex_lock(&s->mu); + job *j = s->done_head; + if (j) { + s->done_head = j->next; + if (!s->done_head) + s->done_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + +static void job_free(job *j) { + if (!j) + return; + free(j->body); + free(j->auth); + free(j->extra_headers); + free(j->response); + free(j); +} + +void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; + (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(fd, tmp, sizeof(tmp)) > 0) { + } + + job *j; + while ((j = pop_done(s)) != NULL) { + client *c = j->c; + int refs_to_drop = 1; /* job reference */ + if (!c->cancelled) { + append_http_response_ex(c, j->status, j->content_type, j->extra_headers, + j->response ? j->response : "", + j->response ? j->response_len : 0); + if (queue_write(c) != AE_OK) + refs_to_drop += close_client_unlink(c); + } + client_decref_n(c, refs_to_drop); + job_free(j); + } +} + +static void job_set_timing_header(job *j) { + if (!j || j->extra_headers || !j->created_ns) + return; + uint64_t done_ns = nstime(); + uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0; + uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0; + + char buf[384]; + int n = snprintf(buf, sizeof(buf), + "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " + "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " + "worker;dur=%.3f\r\n", + ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns), + ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); + if (n > 0 && (size_t)n < sizeof(buf)) + j->extra_headers = xstrdup(buf); +} + +static int parse_job_root(job *j, cJSON **out_root) { + cJSON *detail = cJSON_CreateArray(); + cJSON *root = parse_json_body(j, detail); + if (!root) { + job_set_422(j, detail); + cJSON_Delete(detail); + *out_root = NULL; + return -1; + } + cJSON_Delete(detail); + *out_root = root; + return 0; +} + +static void process_unknown_job(job *j) { + cJSON *root = NULL; + if (!j->started_ns) + j->started_ns = nstime(); + uint64_t t0 = nstime(); + int rc = parse_job_root(j, &root); + j->parse_ns += nstime() - t0; + if (rc != 0) + return; + job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); + cJSON_Delete(root); +} + +void process_job_group(http_server *s, job **jobs, int n_jobs) { + embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); + contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); + rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); + int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); + int *done = xcalloc((size_t)n_jobs, sizeof(*done)); + embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); + contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); + + for (int i = 0; i < n_jobs; i++) { + cJSON *root = NULL; + if (!jobs[i]->started_ns) + jobs[i]->started_ns = nstime(); + if (!strcmp(jobs[i]->path, "/v1/embeddings")) { + kind[i] = 1; + } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { + kind[i] = 2; + } else if (!strcmp(jobs[i]->path, "/v1/rerank")) { + kind[i] = 3; + } else { + continue; + } + uint64_t t0 = nstime(); + int parse_rc = parse_job_root(jobs[i], &root); + jobs[i]->parse_ns += nstime() - t0; + if (parse_rc != 0) + continue; + if (kind[i] == 1) + prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); + else if (kind[i] == 2) + prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); + else + prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]); + } + + for (int i = 0; i < n_jobs; i++) { + if (done[i]) + continue; + + if (!kind[i]) { + process_unknown_job(jobs[i]); + job_set_timing_header(jobs[i]); + enqueue_done(jobs[i]); + done[i] = 1; + continue; + } + + if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) || + (kind[i] == 3 && !rerank_reqs[i].ready)) { + job_set_timing_header(jobs[i]); + enqueue_done(jobs[i]); + done[i] = 1; + continue; + } + + if (kind[i] == 3) { + execute_rerank_request(&rerank_reqs[i]); + job_set_timing_header(jobs[i]); + enqueue_done(jobs[i]); + done[i] = 1; + } else if (kind[i] == 1) { + int group_n = 1; + int group_inputs = std_reqs[i].n_inputs; + int group_tokens = std_reqs[i].total_tokens; + std_group[0] = &std_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 1 && std_reqs[k].ready && + embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && + embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens, + s->batch_size, s->max_batch_tokens)) { + std_group[group_n++] = &std_reqs[k]; + group_inputs += std_reqs[k].n_inputs; + group_tokens += std_reqs[k].total_tokens; + } + } + execute_embedding_request_list(std_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(std_group[k] - std_reqs); + job_set_timing_header(jobs[idx]); + enqueue_done(jobs[idx]); + done[idx] = 1; + } + } else { + int group_n = 1; + int group_docs = ctx_reqs[i].n_docs; + int group_tokens = ctx_reqs[i].total_tokens; + ctx_group[0] = &ctx_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && + contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && + contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens, + s->batch_size, s->max_batch_tokens)) { + ctx_group[group_n++] = &ctx_reqs[k]; + group_docs += ctx_reqs[k].n_docs; + group_tokens += ctx_reqs[k].total_tokens; + } + } + execute_contextual_request_list(ctx_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(ctx_group[k] - ctx_reqs); + job_set_timing_header(jobs[idx]); + enqueue_done(jobs[idx]); + done[idx] = 1; + } + } + } + + for (int i = 0; i < n_jobs; i++) { + embedding_request_free(&std_reqs[i]); + contextual_request_free(&ctx_reqs[i]); + rerank_request_free(&rerank_reqs[i]); + } + free(ctx_group); + free(std_group); + free(done); + free(kind); + free(rerank_reqs); + free(ctx_reqs); + free(std_reqs); +} diff --git a/tools/server/server.c b/tools/server/server.c new file mode 100644 index 0000000..abff2c2 --- /dev/null +++ b/tools/server/server.c @@ -0,0 +1,414 @@ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#include "server.h" +#include "server_internal.h" +#include "util.h" +#include "sbuf.h" +#include "ffwd.h" +#include "ae.h" +#include "anet.h" + +#include <arpa/inet.h> +#include <ctype.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <netinet/in.h> +#include <pthread.h> +#include <signal.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +#ifdef __APPLE__ +# include <sys/sysctl.h> +#endif + +#include <cjson/cJSON.h> + +static volatile sig_atomic_t g_stop_requested = 0; +static volatile sig_atomic_t g_signal_wfd = -1; + +static void stop_signal_handler(int sig) { + (void)sig; + if (g_stop_requested) + _exit(130); + int saved = errno; + g_stop_requested = 1; + int fd = (int)g_signal_wfd; + if (fd >= 0) { + const unsigned char byte = 1; + if (write(fd, &byte, 1) < 0) { + /* best-effort stop wakeup; a failed self-pipe write is ignored */ + } + } + errno = saved; +} + +static bool auth_ok(http_server *s, const char *auth) { + if (!s->api_key || !s->api_key[0]) + return true; + if (!auth) + return false; + const char prefix[] = "Bearer "; + return !strncmp(auth, prefix, sizeof(prefix) - 1) && + !strcmp(auth + sizeof(prefix) - 1, s->api_key); +} + +void dispatch_request(client *c) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + + if (!strcmp(c->req.method, "OPTIONS")) { + append_http_response(c, 204, NULL, "", 0); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + if (!route_is_inference(c->req.method, c->req.path)) { + size_t len; + char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); + append_http_response(c, 404, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + if (!auth_ok(c->srv, c->req.auth)) { + size_t len; + char *body = json_error_body("missing or invalid bearer token", "authentication_error", &len); + append_http_response(c, 401, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + job *j = xcalloc(1, sizeof(*j)); + j->srv = c->srv; + j->c = c; + snprintf(j->method, sizeof(j->method), "%s", c->req.method); + snprintf(j->path, sizeof(j->path), "%s", c->req.path); + j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); + j->body_len = c->req.body_len; + j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; + j->created_ns = nstime(); + client_incref(c); + enqueue_job(j); +} + +static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; + (void)mask; + http_server *s = clientData; + char cip[64]; + int cport = 0; + for (;;) { + int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + if (errno == EINTR) + continue; + return; + } + set_nonblock(cfd); + client *c = xcalloc(1, sizeof(*c)); + c->srv = s; + c->fd = cfd; + c->refcount = 1; + c->last_active_ms = mstime(); + c->linked = 1; + c->next = s->clients; + if (s->clients) + s->clients->prev = c; + s->clients = c; + s->n_clients++; + if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); + } +} + +static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { + (void)loop; + (void)id; + http_server *s = clientData; + uint64_t now = mstime(); + client *c = s->clients; + while (c) { + client *next = c->next; + /* refcount > 1 means a request from this client is queued or being + * processed by the worker; long batch queues (e.g. 4B long-document + * batches) must not be reaped as idle. */ + if (!c->cancelled && c->refcount <= 1 && + now - c->last_active_ms > FFWD_HTTP_CLIENT_TIMEOUT_MS) + close_client(c); + c = next; + } + return FFWD_HTTP_SWEEP_MS; +} + +static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)fd; + (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { + } + aeStop(loop); +} + +static void *worker_main(void *arg) { + http_server *s = arg; + /* Create each model's context on this thread (a GPU build defers the load + * here so the context lives on the thread that uses it) and finalize the + * config a GPU build could not finalize at open time. CPU is loaded and + * finalized already, marked by info->dim being set. */ + int rc = 0; + char load_err[256]; + for (int i = 0; i < s->n_models; i++) { + loaded_model *m = &s->models[i]; + if (ffwd_activate(m->backend, load_err, sizeof(load_err)) != 0) { + server_log("ffwd-server: failed to load model on worker: %s: %s", m->path, load_err); + rc = 1; + break; + } + if (m->info->dim == 0 && finalize_loaded_model(m) != 0) { + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) + s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < s->n_models; i++) + ffwd_worker_free(s->models[i].backend); + return NULL; + } + + for (;;) { + job *jobs[FFWD_SERVER_MICROBATCH_MAX_JOBS]; + int n_jobs = collect_job_batch(s, jobs, FFWD_SERVER_MICROBATCH_MAX_JOBS); + if (n_jobs == 0) + break; + process_job_group(s, jobs, n_jobs); + } + for (int i = 0; i < s->n_models; i++) + ffwd_worker_free(s->models[i].backend); + return NULL; +} + +static int listen_on(const char *host, int port) { + char portbuf[32]; + snprintf(portbuf, sizeof(portbuf), "%d", port); + char err[ANET_ERR_LEN]; + int fd = anetTcpServer(err, port, (char *)host, FFWD_HTTP_BACKLOG); + (void)portbuf; + if (fd == ANET_ERR) { + server_log("ffwd-server: listen failed on %s:%d: %s", host, port, err); + return -1; + } + set_nonblock(fd); + return fd; +} + +static int validate_model_specs(const ffwd_server_config_t *cfg) { + for (int i = 0; i < cfg->n_models; i++) { + const ffwd_server_model_spec_t *spec = &cfg->models[i]; + if (!spec->id || !spec->id[0]) { + fprintf(stderr, "ffwd-server: model label must not be empty\n"); + return -1; + } + if (!spec->path || !spec->path[0]) { + fprintf(stderr, "ffwd-server: model %s has an empty path\n", spec->id); + return -1; + } + if (spec->kind < FFWD_SERVER_MODEL_STANDARD || spec->kind > FFWD_SERVER_MODEL_LATE) { + fprintf(stderr, "ffwd-server: model %s has an invalid kind\n", spec->id); + return -1; + } + if (spec->api < FFWD_SERVER_API_DEFAULT || spec->api > FFWD_SERVER_API_PERPLEXITY) { + fprintf(stderr, "ffwd-server: model %s has an invalid api\n", spec->id); + return -1; + } + if (spec->min_dim < 0) { + fprintf(stderr, "ffwd-server: model %s has an invalid min_dim\n", spec->id); + return -1; + } + for (int j = 0; j < i; j++) { + if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) { + fprintf(stderr, "ffwd-server: duplicate model label: %s\n", spec->id); + return -1; + } + } + } + return 0; +} + +/* Hold the model paths in one array so the backend preflight can size the set + * against host memory in a single pass. */ +static int run_memory_preflight(const ffwd_server_config_t *cfg, const ffwd_options_t *opts) { + const char **paths = (const char **)malloc((size_t)cfg->n_models * sizeof(*paths)); + if (!paths) + return -1; + for (int i = 0; i < cfg->n_models; i++) + paths[i] = cfg->models[i].path; + char err[256] = {0}; + int rc = ffwd_preflight(paths, cfg->n_models, opts, err, sizeof(err)); + free(paths); + if (rc != 0 && err[0]) + server_log("ffwd-server: %s", err); + return rc; +} + +int ffwd_run_server(const ffwd_server_config_t *cfg) { + if (!cfg || !cfg->models || cfg->n_models <= 0) { + fprintf(stderr, "ffwd-server: at least one model label and path is required\n"); + return 1; + } + ffwd_options_t backend_opts = { + .gpu_quant_bits = cfg->gpu_quantize_bits, + .gpu_quant_group_size = cfg->gpu_quantize_group_size > 0 ? cfg->gpu_quantize_group_size : 64, + .memory_utilization = cfg->memory_utilization, + }; + if (validate_model_specs(cfg) != 0 || run_memory_preflight(cfg, &backend_opts) != 0) + return 1; + + http_server s; + memset(&s, 0, sizeof(s)); + s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; + s.port = cfg->port > 0 ? cfg->port : 8000; + s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = + cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); + s.backend_opts = backend_opts; + if (cfg->api_key && cfg->api_key[0]) + s.api_key = xstrdup(cfg->api_key); + else { + const char *env_key = getenv("FFWD_API_KEY"); + if (env_key && env_key[0]) + s.api_key = xstrdup(env_key); + } + + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + for (int i = 0; i < cfg->n_models; i++) { + if (load_one_model(&s, &cfg->models[i]) != 0) { + free_models(&s); + return 1; + } + } + + if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { + perror("pipe"); + free_models(&s); + return 1; + } + set_nonblock(s.completion_pipe[0]); + set_nonblock(s.completion_pipe[1]); + set_nonblock(s.signal_pipe[0]); + set_nonblock(s.signal_pipe[1]); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = stop_signal_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + signal(SIGPIPE, SIG_IGN); + g_signal_wfd = s.signal_pipe[1]; + + s.loop = aeCreateEventLoop(FFWD_HTTP_SETSIZE); + if (!s.loop) { + fprintf(stderr, "ffwd-server: failed to create event loop\n"); + free_models(&s); + return 1; + } + + if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { + fprintf(stderr, "ffwd-server: failed to start worker\n"); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + pthread_mutex_lock(&s.mu); + while (!s.worker_ready) + pthread_cond_wait(&s.cv, &s.mu); + int worker_init_rc = s.worker_init_rc; + pthread_mutex_unlock(&s.mu); + if (worker_init_rc) { + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 1; + } + + s.listen_fd = listen_on(s.host, s.port); + if (s.listen_fd < 0) { + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + + aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); + aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); + aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); + aeCreateTimeEvent(s.loop, FFWD_HTTP_SWEEP_MS, timeout_cb, &s, NULL); + + server_log("ffwd-server: listening on http://%s:%d", s.host, s.port); + aeMain(s.loop); + + server_log("ffwd-server: shutdown requested"); + if (s.listen_fd >= 0) + close(s.listen_fd); + client *c = s.clients; + while (c) { + client *next = c->next; + close_client(c); + c = next; + } + + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + + aeDeleteEventLoop(s.loop); + close(s.completion_pipe[0]); + close(s.completion_pipe[1]); + close(s.signal_pipe[0]); + close(s.signal_pipe[1]); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 0; +} diff --git a/tools/server/server.h b/tools/server/server.h new file mode 100644 index 0000000..b8ec7e4 --- /dev/null +++ b/tools/server/server.h @@ -0,0 +1,57 @@ +#ifndef FFWD_SERVER_H +#define FFWD_SERVER_H + +/* Public-API export annotation. The libraries are built with + * -fvisibility=hidden, so only declarations carrying FFWD_API are exported + * from the shared library; everything else stays internal. */ +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif +#endif + +#include <stdbool.h> + +typedef enum { + FFWD_SERVER_MODEL_STANDARD = 0, + FFWD_SERVER_MODEL_CONTEXTUAL = 1, + FFWD_SERVER_MODEL_LATE = 2, +} ffwd_server_model_kind_t; + +typedef enum { + FFWD_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */ + FFWD_SERVER_API_OPENAI = 1, + FFWD_SERVER_API_PERPLEXITY = 2, +} ffwd_server_embedding_api_t; + +typedef struct { + const char *id; /* served label; requests match this string exactly */ + const char *path; + ffwd_server_model_kind_t kind; + ffwd_server_embedding_api_t api; + int min_dim; /* 0 means the loaded model's full hidden size */ + const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */ +} ffwd_server_model_spec_t; + +typedef struct { + const ffwd_server_model_spec_t *models; + int n_models; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + /* GPU build tuning; ignored by backends that do not use it. */ + int gpu_quantize_bits; + int gpu_quantize_group_size; + double memory_utilization; /* fraction of physical memory the GPU preflight + may plan for; 0 = default 0.90, above 1.0 + allows overcommit */ + const char *api_key; +} ffwd_server_config_t; + +FFWD_API int ffwd_run_server(const ffwd_server_config_t *cfg); + +#endif diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h new file mode 100644 index 0000000..91b51c5 --- /dev/null +++ b/tools/server/server_internal.h @@ -0,0 +1,314 @@ +#ifndef FFWD_SERVER_INTERNAL_H +#define FFWD_SERVER_INTERNAL_H + +/* Declarations shared across the ffwd-server translation units (HTTP framing, + * JSON validation, output encoding, scheduling, model lifecycle, endpoint + * handlers). The server is split by concern but compiles as several objects + * linked into one binary; this header is what they share. Not a public API - + * the CLI and library do not include it. */ + +#include "ffwd.h" +#include "server.h" +#include "sbuf.h" + +#include "ae.h" + +#include <cjson/cJSON.h> +#include <pthread.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define FFWD_HTTP_IO_BUF 8192 +#define FFWD_HTTP_MAX_HEADER (64u * 1024u) +#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define FFWD_HTTP_MAX_PATH 255u +#define FFWD_HTTP_BACKLOG 128 +#define FFWD_HTTP_SETSIZE 10240 +#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000 +#define FFWD_HTTP_SWEEP_MS 1000 + +#define FFWD_API_MAX_STANDARD_INPUTS 512 +#define FFWD_API_MAX_CONTEXT_DOCS 512 +#define FFWD_API_MAX_CONTEXT_CHUNKS 16000 +#define FFWD_API_MAX_RERANK_DOCUMENTS 1000 +#define FFWD_API_MAX_ITEM_TOKENS 32768 +#define FFWD_API_MAX_TOTAL_TOKENS 120000 +/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned + * by the tokenizer (ffwd_tok). */ +/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent + * short-request throughput over 8 with no long-document penalty, while 128 + * was no faster and inflated long-document tail latency. */ +#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32 +#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +/* Microseconds the worker waits for more requests before dispatching a batch. + * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU + * busy; MLX and CPU dispatch immediately, where waiting only adds latency. + * Override with --batch-wait-us. */ +#define FFWD_SERVER_BATCH_WAIT_US 0 +#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000 +#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128 +#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90 +#define FFWD_MLX_RESIDENT_MULTIPLIER 2 + +typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; + +typedef enum { + FFWD_API_OPENAI = 0, + FFWD_API_PERPLEXITY = 1, +} embedding_api_t; + +typedef struct { + char *id; + model_kind kind; + int dim; + int min_dim; + int token_dim; + ffwd_attention_mode_t attention_mode; + ffwd_pooling_mode_t pooling_mode; + int normalize_embeddings; + embedding_api_t api; + char *query_instruct; +} model_info; + +typedef struct { + model_info *info; + char *path; + ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ + ffwd_t *backend; /* compute handle */ + int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */ +} loaded_model; + +typedef struct client client; +typedef struct job job; + +typedef struct { + aeEventLoop *loop; + int listen_fd; + int completion_pipe[2]; + int signal_pipe[2]; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + char *api_key; + loaded_model *models; + int n_models; + ffwd_options_t backend_opts; + + pthread_t worker; + pthread_mutex_t mu; + pthread_cond_t cv; + int stopping; + int worker_ready; + int worker_init_rc; + job *job_head; + job *job_tail; + job *done_head; + job *done_tail; + + client *clients; + int n_clients; +} http_server; + +typedef struct { + char method[8]; + char path[FFWD_HTTP_MAX_PATH + 1]; + char version[16]; + char *body; + size_t body_len; + char *auth; +} http_request; + +struct client { + http_server *srv; + int fd; + int refcount; + int cancelled; + int linked; + uint64_t last_active_ms; + sbuf in; + sbuf out; + size_t sent; + int header_done; + size_t header_len; + size_t content_length; + int close_after_write; + http_request req; + client *prev; + client *next; +}; + +struct job { + http_server *srv; + client *c; + char method[8]; + char path[FFWD_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; + int status; + char *content_type; + char *extra_headers; + char *response; + size_t response_len; + uint64_t created_ns; + uint64_t started_ns; + uint64_t parse_ns; + uint64_t tokenize_ns; + uint64_t infer_ns; + uint64_t encode_ns; + job *next; +}; + +/* Parsed request state, shared by the prepare (handlers), execute (schedule), + * and render (encode) stages. Each carries its source cJSON root (freed when + * the request is freed), the resolved model, and the tokenized inputs. */ +typedef struct { + int *ids; + int n_tokens; +} token_buf; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + int n_inputs; + token_buf *tokens; + ffwd_input_t *inputs; + int total_tokens; + int ready; +} embedding_request; + +typedef struct { + int *ids; + int n_tokens; + ffwd_span_t *spans; + int n_spans; +} contextual_doc; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + contextual_doc *docs; + int n_docs; + int total_chunks; + int total_tokens; + int ready; +} contextual_request; + +/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */ +typedef ffwd_late_tokens_t late_tokens; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + late_tokens query; + late_tokens *documents; + int n_documents; + int top_n; + int return_documents; + int query_tokens; + int document_tokens; + int ready; +} rerank_request; + +/* ---- server_json.c: request validation and error bodies ---- */ +void append_json_string(sbuf *b, const char *s); +char *json_error_body(const char *message, const char *type, size_t *len); +void job_set_error(job *j, int status, const char *message, const char *type); +bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type); +void job_set_422(job *j, cJSON *detail); +cJSON *parse_json_body(job *j, cJSON *detail); +bool cjson_is_integer(cJSON *item); +const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api); +int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct); +int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); + +/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */ +int set_nonblock(int fd); +void client_incref(client *c); +void client_decref_n(client *c, int n); +int close_client_unlink(client *c); +void close_client(client *c); +void append_http_response_ex(client *c, + int status, + const char *ctype, + const char *extra_headers, + const char *body, + size_t body_len); +void append_http_response( + client *c, int status, const char *ctype, const char *body, size_t body_len); +int queue_write(client *c); +bool route_is_inference(const char *method, const char *path); +void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); +/* Defined in server.c (request routing): framed requests are handed here. */ +void dispatch_request(client *c); + +/* ---- server_encode.c: embedding output encoding ---- */ +signed char quantize_int8_tanh(float x); +char *encode_embedding(const float *emb, int dims, const char *encoding); +void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api); +void set_response_from_buf(job *j, sbuf *b); +int render_embedding_response(embedding_request *r, const float *embs); +void render_contextual_response(contextual_request *r, const float *embs); + +/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ +loaded_model *loaded_model_for_label(http_server *s, const char *label); +void free_token_bufs(token_buf *t, int n); +void embedding_request_free(embedding_request *r); +void contextual_request_free(contextual_request *r); +void late_tokens_free(late_tokens *t); +void rerank_request_free(rerank_request *r); +/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */ +int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); +int tokenize_input( + loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out); +int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out); +int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out); +int model_ffwd_spans_batch(loaded_model *m, + const ffwd_context_input_t *inputs, + int batch, + float *out); +int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); +int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim); +int finalize_loaded_model(loaded_model *m); +int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec); +void free_models(http_server *s); + +/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ +void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out); +int embedding_request_compatible(const embedding_request *a, const embedding_request *b); +int embedding_request_fits_group(const embedding_request *r, + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_embedding_request_list(embedding_request **reqs, int n_reqs); +void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out); +int contextual_request_compatible(const contextual_request *a, const contextual_request *b); +int contextual_request_fits_group(const contextual_request *r, + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_contextual_request_list(contextual_request **reqs, int n_reqs); +void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out); +void execute_rerank_request(rerank_request *r); + +/* ---- server_schedule.c: job queue, micro-batching, completion ---- */ +void enqueue_job(job *j); +int collect_job_batch(http_server *s, job **jobs, int max_jobs); +void process_job_group(http_server *s, job **jobs, int n_jobs); +void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask); + +#endif /* FFWD_SERVER_INTERNAL_H */ diff --git a/tools/server/test_base64.c b/tools/server/test_base64.c new file mode 100644 index 0000000..3519c0c --- /dev/null +++ b/tools/server/test_base64.c @@ -0,0 +1,51 @@ +/* test_base64.c - base64_encode against RFC 4648 vectors and binary inputs. */ + +#include "base64.h" +#include "test_util.h" + +#include <stdlib.h> +#include <string.h> + +static void test_base64_rfc4648(void) { + /* RFC 4648 test vectors. */ + static const struct { + const char *in, *out; + } cases[] = { + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in)); + TEST_ASSERT(strcmp(got, cases[i].out) == 0); + free(got); + } +} + +static void test_base64_binary_vectors(void) { + static const struct { + unsigned char bytes[5]; + size_t n; + const char *out; + } cases[] = { + {{0xff}, 1, "/w=="}, + {{0xff, 0xee}, 2, "/+4="}, + {{0xff, 0xee, 0xdd}, 3, "/+7d"}, + {{0x00, 0x01, 0x02, 0x03, 0x04}, 5, "AAECAwQ="}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char *got = base64_encode(cases[i].bytes, cases[i].n); + TEST_ASSERT(strcmp(got, cases[i].out) == 0); + free(got); + } +} + +int main(void) { + test_base64_rfc4648(); + test_base64_binary_vectors(); + return TEST_REPORT("base64"); +} diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c new file mode 100644 index 0000000..a7db9c4 --- /dev/null +++ b/tools/server/test_encode.c @@ -0,0 +1,90 @@ +/* test_encode.c - embedding quantization and the output encodings + * (base64_int8, base64_binary, base64 float32) plus float rendering. */ + +#include "base64.h" +#include "server_internal.h" +#include "test_util.h" + +#include <stdlib.h> +#include <string.h> + +static void test_quantize_int8_tanh(void) { + TEST_ASSERT(quantize_int8_tanh(0.0f) == 0); + /* tanh saturates to +-1, scaled by 127. */ + TEST_ASSERT(quantize_int8_tanh(100.0f) == 127); + TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127); + /* Symmetry. */ + TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f)); + /* round(tanh(0.5) * 127) = round(58.69) = 59. */ + TEST_ASSERT(quantize_int8_tanh(0.5f) == 59); +} + +static void test_encode_embedding_int8(void) { + float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; + signed char expect[4] = {0, 59, -59, 127}; + char *got = encode_embedding(emb, 4, "base64_int8"); + char *want = base64_encode((const unsigned char *)expect, 4); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_encode_embedding_binary(void) { + /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */ + float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f}; + unsigned char expect = 0x85; /* bits 0, 2, 7 set */ + char *got = encode_embedding(emb, 8, "base64_binary"); + char *want = base64_encode(&expect, 1); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); + + /* Zero counts as non-negative (sign bit set). */ + float zeros[8] = {0}; + unsigned char all = 0xff; + got = encode_embedding(zeros, 8, "base64_binary"); + want = base64_encode(&all, 1); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_encode_embedding_base64_float32(void) { + /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32 + * vector - lossless, unlike the int8 formats. */ + float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; + /* Reference bytes via memcpy: a float->byte reinterpret cast confuses the + * static analyzer's uninitialized-read model. */ + unsigned char bytes[sizeof emb]; + memcpy(bytes, emb, sizeof emb); + char *want = base64_encode(bytes, sizeof emb); + char *got = encode_embedding(emb, 4, "base64"); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_append_embedding_value_float(void) { + float emb[1] = {0.5f}; + + /* OpenAI-compatible models render the true float32 value. */ + sbuf b = {0}; + append_embedding_value(&b, 0, emb, 1, "float", FFWD_API_OPENAI); + TEST_ASSERT(strstr(b.ptr, "\"embedding\":[0.5]") != NULL); + sbuf_free(&b); + + /* Perplexity-compatible models render the int8-decoded view. */ + sbuf b2 = {0}; + append_embedding_value(&b2, 0, emb, 1, "float", FFWD_API_PERPLEXITY); + TEST_ASSERT(strstr(b2.ptr, "0.4609375") != NULL); + sbuf_free(&b2); +} + +int main(void) { + test_quantize_int8_tanh(); + test_encode_embedding_int8(); + test_encode_embedding_binary(); + test_encode_embedding_base64_float32(); + test_append_embedding_value_float(); + return TEST_REPORT("encode"); +} diff --git a/tools/server/test_json.c b/tools/server/test_json.c new file mode 100644 index 0000000..76a0f45 --- /dev/null +++ b/tools/server/test_json.c @@ -0,0 +1,143 @@ +/* test_json.c - request validation, JSON error bodies, and 422 detail + * handling (encoding_format, text_type, error escaping, job_set_422). */ + +#include "server_internal.h" +#include "test_util.h" +#include "util.h" /* xstrdup */ + +#include <stdlib.h> +#include <string.h> + +static const char *TEST_QWEN3_QUERY_INSTRUCT = + "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"; + +static void test_json_error_body_escaping(void) { + size_t len = 0; + char *body = json_error_body("line\nquote\" tab\t ctrl\001 slash\\", "bad\ntype", &len); + TEST_ASSERT(body != NULL); + if (!body) + return; + TEST_ASSERT(len == strlen(body)); + TEST_ASSERT(strchr(body, '\n') == NULL); + TEST_ASSERT(strchr(body, '\t') == NULL); + cJSON *root = cJSON_Parse(body); + TEST_ASSERT(root != NULL); + if (!root) { + free(body); + return; + } + cJSON *err = cJSON_GetObjectItemCaseSensitive(root, "error"); + cJSON *msg = err ? cJSON_GetObjectItemCaseSensitive(err, "message") : NULL; + cJSON *type = err ? cJSON_GetObjectItemCaseSensitive(err, "type") : NULL; + if (!msg || !type || !cJSON_IsString(msg) || !cJSON_IsString(type)) { + TEST_ASSERT(0 && "error body fields must be strings"); + cJSON_Delete(root); + free(body); + return; + } + TEST_ASSERT(strcmp(msg->valuestring, "line\nquote\" tab\t ctrl\001 slash\\") == 0); + TEST_ASSERT(strcmp(type->valuestring, "bad\ntype") == 0); + cJSON_Delete(root); + free(body); +} + +static void test_job_set_422_replaces_response(void) { + job j; + memset(&j, 0, sizeof(j)); + j.response = xstrdup("old response"); + j.response_len = strlen(j.response); + j.status = 500; + + cJSON *detail = cJSON_CreateArray(); + ve_add(detail, "[\"body\",\"field\"]", "bad value", "value_error"); + job_set_422(&j, detail); + TEST_ASSERT(j.status == 422); + TEST_ASSERT(j.response != NULL); + if (j.response) { + TEST_ASSERT(j.response_len == strlen(j.response)); + TEST_ASSERT(strstr(j.response, "bad value") != NULL); + } + + cJSON_Delete(detail); + free(j.response); +} + +static void test_encoding_from_root_family(void) { + cJSON *detail = cJSON_CreateArray(); + + /* Default encoding follows the explicit serving API. */ + cJSON *empty = cJSON_CreateObject(); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_OPENAI), "float") == 0); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_PERPLEXITY), "base64_int8") == + 0); + cJSON_Delete(empty); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* OpenAI rejects int8 formats; Perplexity rejects the OpenAI base64. */ + cJSON *o = cJSON_CreateObject(); + cJSON_AddStringToObject(o, "encoding_format", "base64_int8"); + encoding_from_root(o, detail, FFWD_API_OPENAI); + TEST_ASSERT(cJSON_GetArraySize(detail) == 1); + cJSON_Delete(o); + + cJSON *p = cJSON_CreateObject(); + cJSON_AddStringToObject(p, "encoding_format", "base64"); + encoding_from_root(p, detail, FFWD_API_PERPLEXITY); + TEST_ASSERT(cJSON_GetArraySize(detail) == 2); + cJSON_Delete(p); + + /* An unknown format is rejected. */ + cJSON *bad = cJSON_CreateObject(); + cJSON_AddStringToObject(bad, "encoding_format", "base64_fp16"); + encoding_from_root(bad, detail, FFWD_API_OPENAI); + TEST_ASSERT(cJSON_GetArraySize(detail) == 3); + cJSON_Delete(bad); + + cJSON_Delete(detail); +} + +static void test_text_type(void) { + cJSON *detail = cJSON_CreateArray(); + + /* Absent: no-op for both model types, no error. */ + cJSON *empty = cJSON_CreateObject(); + TEST_ASSERT(text_type_is_query(empty, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0); + cJSON_Delete(empty); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* Qwen3: query -> 1, document -> 0, both accepted. */ + cJSON *q = cJSON_CreateObject(); + cJSON_AddStringToObject(q, "text_type", "query"); + TEST_ASSERT(text_type_is_query(q, detail, TEST_QWEN3_QUERY_INSTRUCT) == 1); + cJSON_Delete(q); + cJSON *d = cJSON_CreateObject(); + cJSON_AddStringToObject(d, "text_type", "document"); + TEST_ASSERT(text_type_is_query(d, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + cJSON_Delete(d); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* Invalid enum value on Qwen3 is reported. */ + cJSON *bad = cJSON_CreateObject(); + cJSON_AddStringToObject(bad, "text_type", "passage"); + TEST_ASSERT(text_type_is_query(bad, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + TEST_ASSERT(cJSON_GetArraySize(detail) == 1); + cJSON_Delete(bad); + + /* text_type on a model with no query instruction is rejected. */ + cJSON *p = cJSON_CreateObject(); + cJSON_AddStringToObject(p, "text_type", "query"); + TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0); + TEST_ASSERT(cJSON_GetArraySize(detail) == 2); + cJSON_Delete(p); + + cJSON_Delete(detail); +} + +int main(void) { + test_json_error_body_escaping(); + test_job_set_422_replaces_response(); + test_encoding_from_root_family(); + test_text_type(); + return TEST_REPORT("json"); +} diff --git a/tools/server/test_models.c b/tools/server/test_models.c new file mode 100644 index 0000000..3dce62b --- /dev/null +++ b/tools/server/test_models.c @@ -0,0 +1,26 @@ +/* test_models.c - resolving a served model by its request label. */ + +#include "server_internal.h" +#include "test_util.h" + +#include <string.h> + +static void test_loaded_model_lookup(void) { + model_info info; + memset(&info, 0, sizeof(info)); + info.id = "served-label"; + loaded_model models[1]; + memset(models, 0, sizeof(models)); + models[0].info = &info; + http_server s; + memset(&s, 0, sizeof(s)); + s.models = models; + s.n_models = 1; + TEST_ASSERT(loaded_model_for_label(&s, "served-label") == &models[0]); + TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL); +} + +int main(void) { + test_loaded_model_lookup(); + return TEST_REPORT("models"); +} diff --git a/tools/server/test_sbuf.c b/tools/server/test_sbuf.c new file mode 100644 index 0000000..65d0eba --- /dev/null +++ b/tools/server/test_sbuf.c @@ -0,0 +1,35 @@ +/* test_sbuf.c - growable string buffer growth, formatting, and clearing. */ + +#include "sbuf.h" +#include "test_util.h" + +#include <string.h> + +static void test_sbuf_growth_and_formatting(void) { + sbuf b = {0}; + sbuf_puts(&b, "ab"); + sbuf_putc(&b, 'c'); + TEST_ASSERT(b.len == 3); + TEST_ASSERT(strcmp(b.ptr, "abc") == 0); + + size_t old_cap = b.cap; + sbuf_printf(&b, "-%04d-%s", 7, "tail"); + TEST_ASSERT(strcmp(b.ptr, "abc-0007-tail") == 0); + TEST_ASSERT(b.cap == old_cap); + + char big[600]; + memset(big, 'x', sizeof(big) - 1); + big[sizeof(big) - 1] = '\0'; + sbuf_puts(&b, big); + TEST_ASSERT(b.len == strlen("abc-0007-tail") + sizeof(big) - 1); + TEST_ASSERT(b.ptr[b.len] == '\0'); + sbuf_clear(&b); + TEST_ASSERT(b.len == 0); + TEST_ASSERT(b.ptr && b.ptr[0] == '\0'); + sbuf_free(&b); +} + +int main(void) { + test_sbuf_growth_and_formatting(); + return TEST_REPORT("sbuf"); +} diff --git a/tools/server/test_schedule.c b/tools/server/test_schedule.c new file mode 100644 index 0000000..1f799f1 --- /dev/null +++ b/tools/server/test_schedule.c @@ -0,0 +1,73 @@ +/* test_schedule.c - job queue micro-batching: the batch-wait deadline and the + * zero-wait fast path through enqueue_job / collect_job_batch. */ + +#include "server_internal.h" +#include "test_util.h" +#include "util.h" /* nstime */ + +#include <pthread.h> +#include <stdio.h> +#include <string.h> + +static void test_collect_job_batch_deadline(void) { + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + s.batch_wait_us = 1000; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + uint64_t first_ns = nstime() - 5000000u; + job first = {.srv = &s, .created_ns = first_ns}; + job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u}; + job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings"); + snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&before_deadline); + enqueue_job(&after_deadline); + + job *batch[3] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &before_deadline); + TEST_ASSERT(s.job_head == &after_deadline); + TEST_ASSERT(s.job_tail == &after_deadline); + + s.job_head = NULL; + s.job_tail = NULL; + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + +static void test_collect_job_batch_zero_wait(void) { + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + job first = {.srv = &s, .created_ns = nstime()}; + job second = {.srv = &s, .created_ns = nstime()}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&second); + + job *batch[2] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &second); + TEST_ASSERT(s.job_head == NULL); + TEST_ASSERT(s.job_tail == NULL); + + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + +int main(void) { + test_collect_job_batch_deadline(); + test_collect_job_batch_zero_wait(); + return TEST_REPORT("schedule"); +} diff --git a/tools/server/test_util.h b/tools/server/test_util.h new file mode 100644 index 0000000..c6b6a13 --- /dev/null +++ b/tools/server/test_util.h @@ -0,0 +1,26 @@ +#ifndef FFWD_SERVER_TEST_UTIL_H +#define FFWD_SERVER_TEST_UTIL_H + +/* Shared harness for the ffwd-server unit tests. TEST_ASSERT records failures + * (rather than aborting) so a single run reports every failure, not just the + * first. Each test file includes this, runs its checks, and ends main with + * `return TEST_REPORT("label");`, which prints a summary and yields the exit + * code (0 when clean, 1 otherwise). */ + +#include <stdio.h> + +static int test_failures = 0; + +#define TEST_ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + test_failures++; \ + } \ + } while (0) + +#define TEST_REPORT(label) \ + (test_failures ? (fprintf(stderr, "%s: %d failure(s)\n", (label), test_failures), 1) \ + : (printf("ok: %s\n", (label)), 0)) + +#endif /* FFWD_SERVER_TEST_UTIL_H */ diff --git a/tools/server/util.c b/tools/server/util.c new file mode 100644 index 0000000..1ec4a2a --- /dev/null +++ b/tools/server/util.c @@ -0,0 +1,82 @@ +#include "util.h" + +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <time.h> + +void die_oom(void) { + fprintf(stderr, "ffwd-server: out of memory\n"); + exit(1); +} + +void *xmalloc(size_t n) { + void *p = malloc(n ? n : 1); + if (!p) + die_oom(); + return p; +} + +void *xcalloc(size_t n, size_t sz) { + void *p = calloc(n ? n : 1, sz ? sz : 1); + if (!p) + die_oom(); + return p; +} + +void *xrealloc(void *p, size_t n) { + p = realloc(p, n ? n : 1); + if (!p) + die_oom(); + return p; +} + +char *xstrdup(const char *s) { + size_t n = strlen(s); + char *p = xmalloc(n + 1); + memcpy(p, s, n + 1); + return p; +} + +char *xstrndup(const char *s, size_t n) { + char *p = xmalloc(n + 1); + memcpy(p, s, n); + p[n] = '\0'; + return p; +} + +uint64_t mstime(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u; +} + +uint64_t nstime(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec; + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u; +} + +double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; } + +void server_log(const char *fmt, ...) { + char ts[32]; + time_t now = time(NULL); + struct tm tmv; + localtime_r(&now, &tmv); + strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv); + fprintf(stderr, "%s ", ts); + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} diff --git a/tools/server/util.h b/tools/server/util.h new file mode 100644 index 0000000..3175cd0 --- /dev/null +++ b/tools/server/util.h @@ -0,0 +1,36 @@ +#ifndef FFWD_SERVER_UTIL_H +#define FFWD_SERVER_UTIL_H + +#include <stdint.h> +#include <stddef.h> + +/* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by + * the ffwd-server translation units. The x* allocators abort the process on + * failure rather than returning NULL, so callers never null-check them; the + * attributes below carry that contract across translation-unit boundaries so + * the static analyzer does not flag the (correctly) unchecked uses. */ + +#if defined(__GNUC__) || defined(__clang__) +# define SERVER_NORETURN __attribute__((noreturn)) +# define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull)) +#else +# define SERVER_NORETURN +# define SERVER_RETURNS_NONNULL +#endif + +SERVER_NORETURN void die_oom(void); + +SERVER_RETURNS_NONNULL void *xmalloc(size_t n); +SERVER_RETURNS_NONNULL void *xcalloc(size_t n, size_t sz); +SERVER_RETURNS_NONNULL void *xrealloc(void *p, size_t n); +SERVER_RETURNS_NONNULL char *xstrdup(const char *s); +SERVER_RETURNS_NONNULL char *xstrndup(const char *s, size_t n); + +uint64_t mstime(void); +uint64_t nstime(void); +double ns_to_ms(uint64_t ns); + +/* printf-style line logger: timestamp prefix, newline appended, to stderr. */ +void server_log(const char *fmt, ...) __attribute__((format(printf, 1, 2))); + +#endif /* FFWD_SERVER_UTIL_H */ |
