diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /src/server_internal.h | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'src/server_internal.h')
| -rw-r--r-- | src/server_internal.h | 361 |
1 files changed, 0 insertions, 361 deletions
diff --git a/src/server_internal.h b/src/server_internal.h deleted file mode 100644 index 1f106ca..0000000 --- a/src/server_internal.h +++ /dev/null @@ -1,361 +0,0 @@ -#ifndef EMBED_SERVER_INTERNAL_H -#define EMBED_SERVER_INTERNAL_H - -/* Declarations shared across the embed-server translation units (HTTP framing, - * JSON validation, output encoding, scheduling, model lifecycle, endpoint - * handlers). The server is split by concern but compiles as several objects - * linked into one binary; this header is what they share. Not a public API - - * the CLI and library do not include it. */ - -#include "embed.h" -#include "server.h" -#include "sbuf.h" -#include "tokenizer_bpe.h" -#include "tokenizer_sentencepiece.h" -#include "tokenizer_wordpiece.h" - -#ifdef USE_MLX -#include "mlx.h" -#endif -#ifdef USE_CUDA -#include "cuda.h" -#endif - -#include "deps/ae/ae.h" - -#include <cjson/cJSON.h> -#include <pthread.h> -#include <stdbool.h> -#include <stddef.h> -#include <stdint.h> - -#define EMBED_HTTP_IO_BUF 8192 -#define EMBED_HTTP_MAX_HEADER (64u * 1024u) -#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define EMBED_HTTP_MAX_PATH 255u -#define EMBED_HTTP_BACKLOG 128 -#define EMBED_HTTP_SETSIZE 10240 -#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 -#define EMBED_HTTP_SWEEP_MS 1000 - -#define EMBED_API_MAX_STANDARD_INPUTS 512 -#define EMBED_API_MAX_CONTEXT_DOCS 512 -#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 -#define EMBED_API_MAX_RERANK_DOCUMENTS 1000 -#define EMBED_API_MAX_ITEM_TOKENS 32768 -#define EMBED_API_MAX_TOTAL_TOKENS 120000 -#define EMBED_LATE_QUERY_TOKENS 32 -#define EMBED_LATE_MASK_TOKEN_ID 151642 -#define EMBED_LATE_QUERY_PREFIX_ID 151669 -#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 -/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent - * short-request throughput over 8 with no long-document penalty, while 128 - * was no faster and inflated long-document tail latency. */ -#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 -#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 -/* Microseconds the worker waits for more requests before dispatching a batch. - * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU - * busy; MLX and CPU dispatch immediately, where waiting only adds latency. - * Override with --batch-wait-us. */ -#define EMBED_SERVER_BATCH_WAIT_US 0 -#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 -#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 -#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 -#define EMBED_MLX_RESIDENT_MULTIPLIER 2 - -typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; - -typedef enum { - EMBED_API_OPENAI = 0, - EMBED_API_PERPLEXITY = 1, -} embedding_api_t; - -typedef struct { - char *id; - model_kind kind; - int dim; - int min_dim; - int token_dim; - embed_attention_mode_t attention_mode; - embed_pooling_mode_t pooling_mode; - int normalize_embeddings; - embedding_api_t api; - char *query_instruct; -} model_info; - -typedef struct { - model_info *info; - char *path; - embed_tokenizer_t *tok; - embed_tokenizer_workspace_t *tok_ws; - /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model. - * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the - * Qwen byte-level BPE fields above stay NULL. */ - wordpiece_tokenizer_t *wp_tok; - wordpiece_workspace_t *wp_tok_ws; - sentencepiece_tokenizer_t *sp_tok; - sentencepiece_workspace_t *sp_tok_ws; - int cls_id; - int sep_id; - int context_separator_id; - int late_mask_id; - int late_query_prefix_id; - int late_document_prefix_id; - int late_skip_ids[64]; - int n_late_skip_ids; - int append_terminal_token; - int terminal_token_id; - int renormalize_truncated; - embed_model_t *cpu_model; - embed_workspace_t *cpu_ws; - embed_late_model_t *cpu_late_model; - embed_late_workspace_t *cpu_late_ws; -#ifdef USE_MLX - embed_mlx_ctx_t *mlx_ctx; - embed_mlx_late_ctx_t *mlx_late_ctx; -#endif -#ifdef USE_CUDA - embed_cuda_ctx_t *cuda_ctx; - embed_cuda_late_ctx_t *cuda_late_ctx; -#endif -} loaded_model; - -typedef struct client client; -typedef struct job job; - -typedef struct { - aeEventLoop *loop; - int listen_fd; - int completion_pipe[2]; - int signal_pipe[2]; - const char *host; - int port; - int batch_size; - int max_batch_tokens; - int batch_wait_us; - char *api_key; - loaded_model *models; - int n_models; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; - - pthread_t worker; - pthread_mutex_t mu; - pthread_cond_t cv; - int stopping; - int worker_ready; - int worker_init_rc; - job *job_head; - job *job_tail; - job *done_head; - job *done_tail; - - client *clients; - int n_clients; -} http_server; - -typedef struct { - char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; - char version[16]; - char *body; - size_t body_len; - char *auth; -} http_request; - -struct client { - http_server *srv; - int fd; - int refcount; - int cancelled; - int linked; - uint64_t last_active_ms; - sbuf in; - sbuf out; - size_t sent; - int header_done; - size_t header_len; - size_t content_length; - int close_after_write; - http_request req; - client *prev; - client *next; -}; - -struct job { - http_server *srv; - client *c; - char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; - char *body; - size_t body_len; - char *auth; - int status; - char *content_type; - char *extra_headers; - char *response; - size_t response_len; - uint64_t created_ns; - uint64_t started_ns; - uint64_t parse_ns; - uint64_t tokenize_ns; - uint64_t infer_ns; - uint64_t encode_ns; - job *next; -}; - -/* Parsed request state, shared by the prepare (handlers), execute (schedule), - * and render (encode) stages. Each carries its source cJSON root (freed when - * the request is freed), the resolved model, and the tokenized inputs. */ -typedef struct { - int *ids; - int n_tokens; -} token_buf; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - int n_inputs; - token_buf *tokens; - embed_input_t *inputs; - int total_tokens; - int ready; -} embedding_request; - -typedef struct { - int *ids; - int n_tokens; - embed_span_t *spans; - int n_spans; -} contextual_doc; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - contextual_doc *docs; - int n_docs; - int total_chunks; - int total_tokens; - int ready; -} contextual_request; - -typedef struct { - int *ids; - int n_tokens; - int *keep; - int n_keep; -} late_tokens; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - late_tokens query; - late_tokens *documents; - int n_documents; - int top_n; - int return_documents; - int query_tokens; - int document_tokens; - int ready; -} rerank_request; - -/* ---- server_json.c: request validation and error bodies ---- */ -void append_json_string(sbuf *b, const char *s); -char *json_error_body(const char *message, const char *type, size_t *len); -void job_set_error(job *j, int status, const char *message, const char *type); -bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type); -void job_set_422(job *j, cJSON *detail); -cJSON *parse_json_body(job *j, cJSON *detail); -bool cjson_is_integer(cJSON *item); -const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api); -int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct); -int dimensions_from_root( - cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); - -/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */ -int set_nonblock(int fd); -void client_incref(client *c); -void client_decref_n(client *c, int n); -int close_client_unlink(client *c); -void close_client(client *c); -void append_http_response_ex(client *c, - int status, - const char *ctype, - const char *extra_headers, - const char *body, - size_t body_len); -void append_http_response( - client *c, int status, const char *ctype, const char *body, size_t body_len); -int queue_write(client *c); -bool route_is_inference(const char *method, const char *path); -void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); -/* Defined in server.c (request routing): framed requests are handed here. */ -void dispatch_request(client *c); - -/* ---- server_encode.c: embedding output encoding ---- */ -signed char quantize_int8_tanh(float x); -char *encode_embedding(const float *emb, int dims, const char *encoding); -void append_embedding_value( - sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api); -void set_response_from_buf(job *j, sbuf *b); -int render_embedding_response(embedding_request *r, const float *embs); -void render_contextual_response(contextual_request *r, const float *embs); - -/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ -loaded_model *loaded_model_for_label(http_server *s, const char *label); -void free_token_bufs(token_buf *t, int n); -void embedding_request_free(embedding_request *r); -void contextual_request_free(contextual_request *r); -void late_tokens_free(late_tokens *t); -void rerank_request_free(rerank_request *r); -int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); -int tokenize_input( - loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out); -int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out); -int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out); -int model_embed_spans_batch(loaded_model *m, - const embed_context_input_t *inputs, - int batch, - float *out); -int inference_batch_accepts_input( - const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); -int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim); -int load_one_model(http_server *s, const embed_server_model_spec_t *spec); -void free_models(http_server *s); - -/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ -void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out); -int embedding_request_compatible(const embedding_request *a, const embedding_request *b); -int embedding_request_fits_group(const embedding_request *r, - int group_inputs, - int group_tokens, - int max_batch, - int max_batch_tokens); -void execute_embedding_request_list(embedding_request **reqs, int n_reqs); -void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out); -int contextual_request_compatible(const contextual_request *a, const contextual_request *b); -int contextual_request_fits_group(const contextual_request *r, - int group_docs, - int group_tokens, - int max_batch, - int max_batch_tokens); -void execute_contextual_request_list(contextual_request **reqs, int n_reqs); -void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out); -void execute_rerank_request(rerank_request *r); - -/* ---- server_schedule.c: job queue, micro-batching, completion ---- */ -void enqueue_job(job *j); -int collect_job_batch(http_server *s, job **jobs, int max_jobs); -void process_job_group(http_server *s, job **jobs, int n_jobs); -void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask); - -#endif /* EMBED_SERVER_INTERNAL_H */ |
