diff options
Diffstat (limited to 'src/server.c')
| -rw-r--r-- | src/server.c | 212 |
1 files changed, 1 insertions, 211 deletions
diff --git a/src/server.c b/src/server.c index c3abbac..0305689 100644 --- a/src/server.c +++ b/src/server.c @@ -4,6 +4,7 @@ #include "build.h" #include "server.h" +#include "server_internal.h" #include "embed.h" #include "safetensors.h" #include "kernels.h" @@ -48,54 +49,6 @@ #include <cjson/cJSON.h> -#define EMBED_HTTP_IO_BUF 8192 -#define EMBED_HTTP_MAX_HEADER (64u * 1024u) -#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define EMBED_HTTP_MAX_PATH 255u -#define EMBED_HTTP_BACKLOG 128 -#define EMBED_HTTP_SETSIZE 10240 -#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 -#define EMBED_HTTP_SWEEP_MS 1000 - -#define EMBED_API_MAX_STANDARD_INPUTS 512 -#define EMBED_API_MAX_CONTEXT_DOCS 512 -#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 -#define EMBED_API_MAX_RERANK_DOCUMENTS 1000 -#define EMBED_API_MAX_ITEM_TOKENS 32768 -#define EMBED_API_MAX_TOTAL_TOKENS 120000 -#define EMBED_LATE_QUERY_TOKENS 32 -#define EMBED_LATE_MASK_TOKEN_ID 151642 -#define EMBED_LATE_QUERY_PREFIX_ID 151669 -#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 -/* `text_type: query` prepends the model's published retrieval instruction. - * Documents and the default pass through unchanged. Callers needing a custom - * instruction prepend it themselves and leave text_type at "document". */ -#define EMBED_QWEN3_QUERY_INSTRUCT \ - "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:" -#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ - "Instruct: Given a web search query, retrieve relevant passages that answer the " \ - "query\nQuery: " -/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent - * short-request throughput over 8 with no long-document penalty, while 128 - * was no faster and inflated long-document tail latency. */ -#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 -#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 -/* Microseconds the worker waits for more requests before dispatching a batch. - * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU - * busy; MLX and CPU dispatch immediately, where waiting only adds latency. - * Override with --batch-wait-us. */ -#define EMBED_SERVER_BATCH_WAIT_US 0 -#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 -#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 -#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 -#define EMBED_MLX_RESIDENT_MULTIPLIER 2 - -typedef struct { - char *ptr; - size_t len; - size_t cap; -} sbuf; - static void die_oom(void) { fprintf(stderr, "embed-server: out of memory\n"); exit(1); @@ -223,50 +176,6 @@ static void sbuf_free(sbuf *b) { memset(b, 0, sizeof(*b)); } -typedef enum { - MODEL_STD_06, - MODEL_STD_4, - MODEL_QWEN3_06, - MODEL_QWEN3_4, - MODEL_QWEN3_8, - MODEL_GTE_QWEN2_15, - MODEL_MINILM_L6, - MODEL_BGE_SMALL, - MODEL_BGE_BASE, - MODEL_BGE_LARGE, - MODEL_CTX_06, - MODEL_CTX_4, - MODEL_LATE_06, - MODEL_COUNT, - MODEL_UNKNOWN -} model_slot; - -typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; - -/* Which embedding API a model speaks. pplx-embed models follow the Perplexity - * API: the canonical output is the tanh int8 quantization, so the default - * encoding is base64_int8 and "float" is the int8-decoded view (int8/128). - * Instruction embedding models follow the OpenAI-compatible API: "float" is - * the true float32 vector and "base64" is base64 of little-endian float32. - * int8 quantization underuses the range of L2-normalized vectors, so it is not - * used for these models. */ -typedef enum { - EMBED_API_PERPLEXITY = 0, - EMBED_API_OPENAI = 1, -} embedding_api_t; - -typedef struct { - const char *id; - model_kind kind; - int dim; - int min_dim; - int token_dim; - embed_attention_mode_t attention_mode; - embed_pooling_mode_t pooling_mode; - int normalize_embeddings; - embedding_api_t api; - const char *query_instruct; -} model_info; static const model_info k_models[] = { {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, @@ -310,125 +219,6 @@ static model_slot model_slot_for_id(const char *id) { return MODEL_UNKNOWN; } -typedef struct { - const model_info *info; - char *path; - embed_tokenizer_t *tok; - embed_tokenizer_workspace_t *tok_ws; - /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model. - * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the - * Qwen byte-level BPE fields above stay NULL. */ - wordpiece_tokenizer_t *wp_tok; - wordpiece_workspace_t *wp_tok_ws; - int cls_id; - int sep_id; - int context_separator_id; - int late_mask_id; - int late_query_prefix_id; - int late_document_prefix_id; - int late_skip_ids[64]; - int n_late_skip_ids; - int append_terminal_token; - int terminal_token_id; - int renormalize_truncated; - embed_model_t *cpu_model; - embed_workspace_t *cpu_ws; - embed_late_model_t *cpu_late_model; - embed_late_workspace_t *cpu_late_ws; -#ifdef USE_MLX - embed_mlx_ctx_t *mlx_ctx; - embed_mlx_late_ctx_t *mlx_late_ctx; -#endif -#ifdef USE_CUDA - embed_cuda_ctx_t *cuda_ctx; - embed_cuda_late_ctx_t *cuda_late_ctx; -#endif -} loaded_model; - -typedef struct client client; -typedef struct job job; - -typedef struct { - aeEventLoop *loop; - int listen_fd; - int completion_pipe[2]; - int signal_pipe[2]; - const char *host; - int port; - int batch_size; - int max_batch_tokens; - int batch_wait_us; - char *api_key; - loaded_model models[MODEL_COUNT]; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; - - pthread_t worker; - pthread_mutex_t mu; - pthread_cond_t cv; - int stopping; - int worker_ready; - int worker_init_rc; - job *job_head; - job *job_tail; - job *done_head; - job *done_tail; - - client *clients; - int n_clients; -} http_server; - -typedef struct { - char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; - char version[16]; - char *body; - size_t body_len; - char *auth; -} http_request; - -struct client { - http_server *srv; - int fd; - int refcount; - int cancelled; - int linked; - uint64_t last_active_ms; - sbuf in; - sbuf out; - size_t sent; - int header_done; - size_t header_len; - size_t content_length; - int close_after_write; - http_request req; - client *prev; - client *next; -}; - -struct job { - http_server *srv; - client *c; - char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; - char *body; - size_t body_len; - char *auth; - int status; - char *content_type; - char *extra_headers; - char *response; - size_t response_len; - uint64_t created_ns; - uint64_t started_ns; - uint64_t parse_ns; - uint64_t tokenize_ns; - uint64_t infer_ns; - uint64_t encode_ns; - job *next; -}; static int queue_write(client *c); static void dispatch_request(client *c); |
