aboutsummaryrefslogtreecommitdiff
path: root/tools/server/server_internal.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server/server_internal.h
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server/server_internal.h')
-rw-r--r--tools/server/server_internal.h314
1 files changed, 314 insertions, 0 deletions
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
new file mode 100644
index 0000000..91b51c5
--- /dev/null
+++ b/tools/server/server_internal.h
@@ -0,0 +1,314 @@
+#ifndef FFWD_SERVER_INTERNAL_H
+#define FFWD_SERVER_INTERNAL_H
+
+/* Declarations shared across the ffwd-server translation units (HTTP framing,
+ * JSON validation, output encoding, scheduling, model lifecycle, endpoint
+ * handlers). The server is split by concern but compiles as several objects
+ * linked into one binary; this header is what they share. Not a public API -
+ * the CLI and library do not include it. */
+
+#include "ffwd.h"
+#include "server.h"
+#include "sbuf.h"
+
+#include "ae.h"
+
+#include <cjson/cJSON.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define FFWD_HTTP_IO_BUF 8192
+#define FFWD_HTTP_MAX_HEADER (64u * 1024u)
+#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define FFWD_HTTP_MAX_PATH 255u
+#define FFWD_HTTP_BACKLOG 128
+#define FFWD_HTTP_SETSIZE 10240
+#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000
+#define FFWD_HTTP_SWEEP_MS 1000
+
+#define FFWD_API_MAX_STANDARD_INPUTS 512
+#define FFWD_API_MAX_CONTEXT_DOCS 512
+#define FFWD_API_MAX_CONTEXT_CHUNKS 16000
+#define FFWD_API_MAX_RERANK_DOCUMENTS 1000
+#define FFWD_API_MAX_ITEM_TOKENS 32768
+#define FFWD_API_MAX_TOTAL_TOKENS 120000
+/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned
+ * by the tokenizer (ffwd_tok). */
+/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
+ * short-request throughput over 8 with no long-document penalty, while 128
+ * was no faster and inflated long-document tail latency. */
+#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32
+#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
+/* Microseconds the worker waits for more requests before dispatching a batch.
+ * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
+ * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
+ * Override with --batch-wait-us. */
+#define FFWD_SERVER_BATCH_WAIT_US 0
+#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000
+#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128
+#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90
+#define FFWD_MLX_RESIDENT_MULTIPLIER 2
+
+typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
+
+typedef enum {
+ FFWD_API_OPENAI = 0,
+ FFWD_API_PERPLEXITY = 1,
+} embedding_api_t;
+
+typedef struct {
+ char *id;
+ model_kind kind;
+ int dim;
+ int min_dim;
+ int token_dim;
+ ffwd_attention_mode_t attention_mode;
+ ffwd_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
+ embedding_api_t api;
+ char *query_instruct;
+} model_info;
+
+typedef struct {
+ model_info *info;
+ char *path;
+ ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
+ ffwd_t *backend; /* compute handle */
+ int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */
+} loaded_model;
+
+typedef struct client client;
+typedef struct job job;
+
+typedef struct {
+ aeEventLoop *loop;
+ int listen_fd;
+ int completion_pipe[2];
+ int signal_pipe[2];
+ const char *host;
+ int port;
+ int batch_size;
+ int max_batch_tokens;
+ int batch_wait_us;
+ char *api_key;
+ loaded_model *models;
+ int n_models;
+ ffwd_options_t backend_opts;
+
+ pthread_t worker;
+ pthread_mutex_t mu;
+ pthread_cond_t cv;
+ int stopping;
+ int worker_ready;
+ int worker_init_rc;
+ job *job_head;
+ job *job_tail;
+ job *done_head;
+ job *done_tail;
+
+ client *clients;
+ int n_clients;
+} http_server;
+
+typedef struct {
+ char method[8];
+ char path[FFWD_HTTP_MAX_PATH + 1];
+ char version[16];
+ char *body;
+ size_t body_len;
+ char *auth;
+} http_request;
+
+struct client {
+ http_server *srv;
+ int fd;
+ int refcount;
+ int cancelled;
+ int linked;
+ uint64_t last_active_ms;
+ sbuf in;
+ sbuf out;
+ size_t sent;
+ int header_done;
+ size_t header_len;
+ size_t content_length;
+ int close_after_write;
+ http_request req;
+ client *prev;
+ client *next;
+};
+
+struct job {
+ http_server *srv;
+ client *c;
+ char method[8];
+ char path[FFWD_HTTP_MAX_PATH + 1];
+ char *body;
+ size_t body_len;
+ char *auth;
+ int status;
+ char *content_type;
+ char *extra_headers;
+ char *response;
+ size_t response_len;
+ uint64_t created_ns;
+ uint64_t started_ns;
+ uint64_t parse_ns;
+ uint64_t tokenize_ns;
+ uint64_t infer_ns;
+ uint64_t encode_ns;
+ job *next;
+};
+
+/* Parsed request state, shared by the prepare (handlers), execute (schedule),
+ * and render (encode) stages. Each carries its source cJSON root (freed when
+ * the request is freed), the resolved model, and the tokenized inputs. */
+typedef struct {
+ int *ids;
+ int n_tokens;
+} token_buf;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ int n_inputs;
+ token_buf *tokens;
+ ffwd_input_t *inputs;
+ int total_tokens;
+ int ready;
+} embedding_request;
+
+typedef struct {
+ int *ids;
+ int n_tokens;
+ ffwd_span_t *spans;
+ int n_spans;
+} contextual_doc;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ contextual_doc *docs;
+ int n_docs;
+ int total_chunks;
+ int total_tokens;
+ int ready;
+} contextual_request;
+
+/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */
+typedef ffwd_late_tokens_t late_tokens;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ late_tokens query;
+ late_tokens *documents;
+ int n_documents;
+ int top_n;
+ int return_documents;
+ int query_tokens;
+ int document_tokens;
+ int ready;
+} rerank_request;
+
+/* ---- server_json.c: request validation and error bodies ---- */
+void append_json_string(sbuf *b, const char *s);
+char *json_error_body(const char *message, const char *type, size_t *len);
+void job_set_error(job *j, int status, const char *message, const char *type);
+bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type);
+void job_set_422(job *j, cJSON *detail);
+cJSON *parse_json_body(job *j, cJSON *detail);
+bool cjson_is_integer(cJSON *item);
+const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api);
+int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct);
+int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding);
+
+/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */
+int set_nonblock(int fd);
+void client_incref(client *c);
+void client_decref_n(client *c, int n);
+int close_client_unlink(client *c);
+void close_client(client *c);
+void append_http_response_ex(client *c,
+ int status,
+ const char *ctype,
+ const char *extra_headers,
+ const char *body,
+ size_t body_len);
+void append_http_response(
+ client *c, int status, const char *ctype, const char *body, size_t body_len);
+int queue_write(client *c);
+bool route_is_inference(const char *method, const char *path);
+void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
+/* Defined in server.c (request routing): framed requests are handed here. */
+void dispatch_request(client *c);
+
+/* ---- server_encode.c: embedding output encoding ---- */
+signed char quantize_int8_tanh(float x);
+char *encode_embedding(const float *emb, int dims, const char *encoding);
+void append_embedding_value(
+ sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api);
+void set_response_from_buf(job *j, sbuf *b);
+int render_embedding_response(embedding_request *r, const float *embs);
+void render_contextual_response(contextual_request *r, const float *embs);
+
+/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */
+loaded_model *loaded_model_for_label(http_server *s, const char *label);
+void free_token_bufs(token_buf *t, int n);
+void embedding_request_free(embedding_request *r);
+void contextual_request_free(contextual_request *r);
+void late_tokens_free(late_tokens *t);
+void rerank_request_free(rerank_request *r);
+/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out);
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out);
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out);
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim);
+int finalize_loaded_model(loaded_model *m);
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec);
+void free_models(http_server *s);
+
+/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
+void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out);
+int embedding_request_compatible(const embedding_request *a, const embedding_request *b);
+int embedding_request_fits_group(const embedding_request *r,
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens);
+void execute_embedding_request_list(embedding_request **reqs, int n_reqs);
+void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out);
+int contextual_request_compatible(const contextual_request *a, const contextual_request *b);
+int contextual_request_fits_group(const contextual_request *r,
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens);
+void execute_contextual_request_list(contextual_request **reqs, int n_reqs);
+void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out);
+void execute_rerank_request(rerank_request *r);
+
+/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
+void enqueue_job(job *j);
+int collect_job_batch(http_server *s, job **jobs, int max_jobs);
+void process_job_group(http_server *s, job **jobs, int n_jobs);
+void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
+
+#endif /* FFWD_SERVER_INTERNAL_H */