diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:56:50 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:56:50 +0200 |
| commit | 230dc1748aa68cccde6de9aa3cdade7d64aede56 (patch) | |
| tree | 48174fe9d12890b0ecf9bf818c3ad80262ceb9b8 /src/server_internal.h | |
| parent | e54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c (diff) | |
| download | ffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.tar.gz ffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.zip | |
server: extract shared structs and macros into server_internal.h
Diffstat (limited to 'src/server_internal.h')
| -rw-r--r-- | src/server_internal.h | 241 |
1 files changed, 241 insertions, 0 deletions
diff --git a/src/server_internal.h b/src/server_internal.h new file mode 100644 index 0000000..c72831e --- /dev/null +++ b/src/server_internal.h @@ -0,0 +1,241 @@ +#ifndef EMBED_SERVER_INTERNAL_H +#define EMBED_SERVER_INTERNAL_H + +/* Declarations shared across the embed-server translation units (HTTP framing, + * JSON validation, output encoding, scheduling, model lifecycle, endpoint + * handlers). The server is split by concern but compiles as several objects + * linked into one binary; this header is what they share. Not a public API - + * the CLI and library do not include it. */ + +#include "embed.h" +#include "tokenizer_bpe.h" +#include "tokenizer_wordpiece.h" + +#ifdef USE_MLX +#include "mlx.h" +#endif +#ifdef USE_CUDA +#include "cuda.h" +#endif + +#include "deps/ae/ae.h" + +#include <pthread.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define EMBED_HTTP_IO_BUF 8192 +#define EMBED_HTTP_MAX_HEADER (64u * 1024u) +#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define EMBED_HTTP_MAX_PATH 255u +#define EMBED_HTTP_BACKLOG 128 +#define EMBED_HTTP_SETSIZE 10240 +#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 +#define EMBED_HTTP_SWEEP_MS 1000 + +#define EMBED_API_MAX_STANDARD_INPUTS 512 +#define EMBED_API_MAX_CONTEXT_DOCS 512 +#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 +#define EMBED_API_MAX_RERANK_DOCUMENTS 1000 +#define EMBED_API_MAX_ITEM_TOKENS 32768 +#define EMBED_API_MAX_TOTAL_TOKENS 120000 +#define EMBED_LATE_QUERY_TOKENS 32 +#define EMBED_LATE_MASK_TOKEN_ID 151642 +#define EMBED_LATE_QUERY_PREFIX_ID 151669 +#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 +/* `text_type: query` prepends the model's published retrieval instruction. + * Documents and the default pass through unchanged. Callers needing a custom + * instruction prepend it themselves and leave text_type at "document". */ +#define EMBED_QWEN3_QUERY_INSTRUCT \ + "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:" +#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ + "Instruct: Given a web search query, retrieve relevant passages that answer the " \ + "query\nQuery: " +/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent + * short-request throughput over 8 with no long-document penalty, while 128 + * was no faster and inflated long-document tail latency. */ +#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 +#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +/* Microseconds the worker waits for more requests before dispatching a batch. + * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU + * busy; MLX and CPU dispatch immediately, where waiting only adds latency. + * Override with --batch-wait-us. */ +#define EMBED_SERVER_BATCH_WAIT_US 0 +#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 +#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 +#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 +#define EMBED_MLX_RESIDENT_MULTIPLIER 2 + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} sbuf; + +typedef enum { + MODEL_STD_06, + MODEL_STD_4, + MODEL_QWEN3_06, + MODEL_QWEN3_4, + MODEL_QWEN3_8, + MODEL_GTE_QWEN2_15, + MODEL_MINILM_L6, + MODEL_BGE_SMALL, + MODEL_BGE_BASE, + MODEL_BGE_LARGE, + MODEL_CTX_06, + MODEL_CTX_4, + MODEL_LATE_06, + MODEL_COUNT, + MODEL_UNKNOWN +} model_slot; + +typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; + +/* Which embedding API a model speaks. pplx-embed models follow the Perplexity + * API: the canonical output is the tanh int8 quantization, so the default + * encoding is base64_int8 and "float" is the int8-decoded view (int8/128). + * Instruction embedding models follow the OpenAI-compatible API: "float" is + * the true float32 vector and "base64" is base64 of little-endian float32. + * int8 quantization underuses the range of L2-normalized vectors, so it is not + * used for these models. */ +typedef enum { + EMBED_API_PERPLEXITY = 0, + EMBED_API_OPENAI = 1, +} embedding_api_t; + +typedef struct { + const char *id; + model_kind kind; + int dim; + int min_dim; + int token_dim; + embed_attention_mode_t attention_mode; + embed_pooling_mode_t pooling_mode; + int normalize_embeddings; + embedding_api_t api; + const char *query_instruct; +} model_info; + +typedef struct { + const model_info *info; + char *path; + embed_tokenizer_t *tok; + embed_tokenizer_workspace_t *tok_ws; + /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model. + * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the + * Qwen byte-level BPE fields above stay NULL. */ + wordpiece_tokenizer_t *wp_tok; + wordpiece_workspace_t *wp_tok_ws; + int cls_id; + int sep_id; + int context_separator_id; + int late_mask_id; + int late_query_prefix_id; + int late_document_prefix_id; + int late_skip_ids[64]; + int n_late_skip_ids; + int append_terminal_token; + int terminal_token_id; + int renormalize_truncated; + embed_model_t *cpu_model; + embed_workspace_t *cpu_ws; + embed_late_model_t *cpu_late_model; + embed_late_workspace_t *cpu_late_ws; +#ifdef USE_MLX + embed_mlx_ctx_t *mlx_ctx; + embed_mlx_late_ctx_t *mlx_late_ctx; +#endif +#ifdef USE_CUDA + embed_cuda_ctx_t *cuda_ctx; + embed_cuda_late_ctx_t *cuda_late_ctx; +#endif +} loaded_model; + +typedef struct client client; +typedef struct job job; + +typedef struct { + aeEventLoop *loop; + int listen_fd; + int completion_pipe[2]; + int signal_pipe[2]; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + char *api_key; + loaded_model models[MODEL_COUNT]; + int use_mlx; + int use_cuda; + int mlx_quantize_bits; + int mlx_quantize_group_size; + + pthread_t worker; + pthread_mutex_t mu; + pthread_cond_t cv; + int stopping; + int worker_ready; + int worker_init_rc; + job *job_head; + job *job_tail; + job *done_head; + job *done_tail; + + client *clients; + int n_clients; +} http_server; + +typedef struct { + char method[8]; + char path[EMBED_HTTP_MAX_PATH + 1]; + char version[16]; + char *body; + size_t body_len; + char *auth; +} http_request; + +struct client { + http_server *srv; + int fd; + int refcount; + int cancelled; + int linked; + uint64_t last_active_ms; + sbuf in; + sbuf out; + size_t sent; + int header_done; + size_t header_len; + size_t content_length; + int close_after_write; + http_request req; + client *prev; + client *next; +}; + +struct job { + http_server *srv; + client *c; + char method[8]; + char path[EMBED_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; + int status; + char *content_type; + char *extra_headers; + char *response; + size_t response_len; + uint64_t created_ns; + uint64_t started_ns; + uint64_t parse_ns; + uint64_t tokenize_ns; + uint64_t infer_ns; + uint64_t encode_ns; + job *next; +}; + +#endif /* EMBED_SERVER_INTERNAL_H */ |
