diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server/server_internal.h | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server/server_internal.h')
| -rw-r--r-- | tools/server/server_internal.h | 314 |
1 files changed, 314 insertions, 0 deletions
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h new file mode 100644 index 0000000..91b51c5 --- /dev/null +++ b/tools/server/server_internal.h @@ -0,0 +1,314 @@ +#ifndef FFWD_SERVER_INTERNAL_H +#define FFWD_SERVER_INTERNAL_H + +/* Declarations shared across the ffwd-server translation units (HTTP framing, + * JSON validation, output encoding, scheduling, model lifecycle, endpoint + * handlers). The server is split by concern but compiles as several objects + * linked into one binary; this header is what they share. Not a public API - + * the CLI and library do not include it. */ + +#include "ffwd.h" +#include "server.h" +#include "sbuf.h" + +#include "ae.h" + +#include <cjson/cJSON.h> +#include <pthread.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define FFWD_HTTP_IO_BUF 8192 +#define FFWD_HTTP_MAX_HEADER (64u * 1024u) +#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define FFWD_HTTP_MAX_PATH 255u +#define FFWD_HTTP_BACKLOG 128 +#define FFWD_HTTP_SETSIZE 10240 +#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000 +#define FFWD_HTTP_SWEEP_MS 1000 + +#define FFWD_API_MAX_STANDARD_INPUTS 512 +#define FFWD_API_MAX_CONTEXT_DOCS 512 +#define FFWD_API_MAX_CONTEXT_CHUNKS 16000 +#define FFWD_API_MAX_RERANK_DOCUMENTS 1000 +#define FFWD_API_MAX_ITEM_TOKENS 32768 +#define FFWD_API_MAX_TOTAL_TOKENS 120000 +/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned + * by the tokenizer (ffwd_tok). */ +/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent + * short-request throughput over 8 with no long-document penalty, while 128 + * was no faster and inflated long-document tail latency. */ +#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32 +#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +/* Microseconds the worker waits for more requests before dispatching a batch. + * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU + * busy; MLX and CPU dispatch immediately, where waiting only adds latency. + * Override with --batch-wait-us. */ +#define FFWD_SERVER_BATCH_WAIT_US 0 +#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000 +#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128 +#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90 +#define FFWD_MLX_RESIDENT_MULTIPLIER 2 + +typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; + +typedef enum { + FFWD_API_OPENAI = 0, + FFWD_API_PERPLEXITY = 1, +} embedding_api_t; + +typedef struct { + char *id; + model_kind kind; + int dim; + int min_dim; + int token_dim; + ffwd_attention_mode_t attention_mode; + ffwd_pooling_mode_t pooling_mode; + int normalize_embeddings; + embedding_api_t api; + char *query_instruct; +} model_info; + +typedef struct { + model_info *info; + char *path; + ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ + ffwd_t *backend; /* compute handle */ + int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */ +} loaded_model; + +typedef struct client client; +typedef struct job job; + +typedef struct { + aeEventLoop *loop; + int listen_fd; + int completion_pipe[2]; + int signal_pipe[2]; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + char *api_key; + loaded_model *models; + int n_models; + ffwd_options_t backend_opts; + + pthread_t worker; + pthread_mutex_t mu; + pthread_cond_t cv; + int stopping; + int worker_ready; + int worker_init_rc; + job *job_head; + job *job_tail; + job *done_head; + job *done_tail; + + client *clients; + int n_clients; +} http_server; + +typedef struct { + char method[8]; + char path[FFWD_HTTP_MAX_PATH + 1]; + char version[16]; + char *body; + size_t body_len; + char *auth; +} http_request; + +struct client { + http_server *srv; + int fd; + int refcount; + int cancelled; + int linked; + uint64_t last_active_ms; + sbuf in; + sbuf out; + size_t sent; + int header_done; + size_t header_len; + size_t content_length; + int close_after_write; + http_request req; + client *prev; + client *next; +}; + +struct job { + http_server *srv; + client *c; + char method[8]; + char path[FFWD_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; + int status; + char *content_type; + char *extra_headers; + char *response; + size_t response_len; + uint64_t created_ns; + uint64_t started_ns; + uint64_t parse_ns; + uint64_t tokenize_ns; + uint64_t infer_ns; + uint64_t encode_ns; + job *next; +}; + +/* Parsed request state, shared by the prepare (handlers), execute (schedule), + * and render (encode) stages. Each carries its source cJSON root (freed when + * the request is freed), the resolved model, and the tokenized inputs. */ +typedef struct { + int *ids; + int n_tokens; +} token_buf; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + int n_inputs; + token_buf *tokens; + ffwd_input_t *inputs; + int total_tokens; + int ready; +} embedding_request; + +typedef struct { + int *ids; + int n_tokens; + ffwd_span_t *spans; + int n_spans; +} contextual_doc; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + contextual_doc *docs; + int n_docs; + int total_chunks; + int total_tokens; + int ready; +} contextual_request; + +/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */ +typedef ffwd_late_tokens_t late_tokens; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + late_tokens query; + late_tokens *documents; + int n_documents; + int top_n; + int return_documents; + int query_tokens; + int document_tokens; + int ready; +} rerank_request; + +/* ---- server_json.c: request validation and error bodies ---- */ +void append_json_string(sbuf *b, const char *s); +char *json_error_body(const char *message, const char *type, size_t *len); +void job_set_error(job *j, int status, const char *message, const char *type); +bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type); +void job_set_422(job *j, cJSON *detail); +cJSON *parse_json_body(job *j, cJSON *detail); +bool cjson_is_integer(cJSON *item); +const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api); +int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct); +int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); + +/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */ +int set_nonblock(int fd); +void client_incref(client *c); +void client_decref_n(client *c, int n); +int close_client_unlink(client *c); +void close_client(client *c); +void append_http_response_ex(client *c, + int status, + const char *ctype, + const char *extra_headers, + const char *body, + size_t body_len); +void append_http_response( + client *c, int status, const char *ctype, const char *body, size_t body_len); +int queue_write(client *c); +bool route_is_inference(const char *method, const char *path); +void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); +/* Defined in server.c (request routing): framed requests are handed here. */ +void dispatch_request(client *c); + +/* ---- server_encode.c: embedding output encoding ---- */ +signed char quantize_int8_tanh(float x); +char *encode_embedding(const float *emb, int dims, const char *encoding); +void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api); +void set_response_from_buf(job *j, sbuf *b); +int render_embedding_response(embedding_request *r, const float *embs); +void render_contextual_response(contextual_request *r, const float *embs); + +/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ +loaded_model *loaded_model_for_label(http_server *s, const char *label); +void free_token_bufs(token_buf *t, int n); +void embedding_request_free(embedding_request *r); +void contextual_request_free(contextual_request *r); +void late_tokens_free(late_tokens *t); +void rerank_request_free(rerank_request *r); +/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */ +int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); +int tokenize_input( + loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out); +int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out); +int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out); +int model_ffwd_spans_batch(loaded_model *m, + const ffwd_context_input_t *inputs, + int batch, + float *out); +int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); +int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim); +int finalize_loaded_model(loaded_model *m); +int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec); +void free_models(http_server *s); + +/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ +void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out); +int embedding_request_compatible(const embedding_request *a, const embedding_request *b); +int embedding_request_fits_group(const embedding_request *r, + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_embedding_request_list(embedding_request **reqs, int n_reqs); +void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out); +int contextual_request_compatible(const contextual_request *a, const contextual_request *b); +int contextual_request_fits_group(const contextual_request *r, + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_contextual_request_list(contextual_request **reqs, int n_reqs); +void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out); +void execute_rerank_request(rerank_request *r); + +/* ---- server_schedule.c: job queue, micro-batching, completion ---- */ +void enqueue_job(job *j); +int collect_job_batch(http_server *s, job **jobs, int max_jobs); +void process_job_group(http_server *s, job **jobs, int n_jobs); +void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask); + +#endif /* FFWD_SERVER_INTERNAL_H */ |
