aboutsummaryrefslogtreecommitdiff
path: root/src/server.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:56:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:56:50 +0200
commit230dc1748aa68cccde6de9aa3cdade7d64aede56 (patch)
tree48174fe9d12890b0ecf9bf818c3ad80262ceb9b8 /src/server.c
parente54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c (diff)
downloadffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.tar.gz
ffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.zip
server: extract shared structs and macros into server_internal.h
Diffstat (limited to 'src/server.c')
-rw-r--r--src/server.c212
1 files changed, 1 insertions, 211 deletions
diff --git a/src/server.c b/src/server.c
index c3abbac..0305689 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4,6 +4,7 @@
#include "build.h"
#include "server.h"
+#include "server_internal.h"
#include "embed.h"
#include "safetensors.h"
#include "kernels.h"
@@ -48,54 +49,6 @@
#include <cjson/cJSON.h>
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
-#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
-
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
-#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
-/* `text_type: query` prepends the model's published retrieval instruction.
- * Documents and the default pass through unchanged. Callers needing a custom
- * instruction prepend it themselves and leave text_type at "document". */
-#define EMBED_QWEN3_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
-#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the " \
- "query\nQuery: "
-/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
- * short-request throughput over 8 with no long-document penalty, while 128
- * was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
-#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
-/* Microseconds the worker waits for more requests before dispatching a batch.
- * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
- * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
- * Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
-#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
-
-typedef struct {
- char *ptr;
- size_t len;
- size_t cap;
-} sbuf;
-
static void die_oom(void) {
fprintf(stderr, "embed-server: out of memory\n");
exit(1);
@@ -223,50 +176,6 @@ static void sbuf_free(sbuf *b) {
memset(b, 0, sizeof(*b));
}
-typedef enum {
- MODEL_STD_06,
- MODEL_STD_4,
- MODEL_QWEN3_06,
- MODEL_QWEN3_4,
- MODEL_QWEN3_8,
- MODEL_GTE_QWEN2_15,
- MODEL_MINILM_L6,
- MODEL_BGE_SMALL,
- MODEL_BGE_BASE,
- MODEL_BGE_LARGE,
- MODEL_CTX_06,
- MODEL_CTX_4,
- MODEL_LATE_06,
- MODEL_COUNT,
- MODEL_UNKNOWN
-} model_slot;
-
-typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
-
-/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
- * API: the canonical output is the tanh int8 quantization, so the default
- * encoding is base64_int8 and "float" is the int8-decoded view (int8/128).
- * Instruction embedding models follow the OpenAI-compatible API: "float" is
- * the true float32 vector and "base64" is base64 of little-endian float32.
- * int8 quantization underuses the range of L2-normalized vectors, so it is not
- * used for these models. */
-typedef enum {
- EMBED_API_PERPLEXITY = 0,
- EMBED_API_OPENAI = 1,
-} embedding_api_t;
-
-typedef struct {
- const char *id;
- model_kind kind;
- int dim;
- int min_dim;
- int token_dim;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- embedding_api_t api;
- const char *query_instruct;
-} model_info;
static const model_info k_models[] = {
{"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
@@ -310,125 +219,6 @@ static model_slot model_slot_for_id(const char *id) {
return MODEL_UNKNOWN;
}
-typedef struct {
- const model_info *info;
- char *path;
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model.
- * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- int cls_id;
- int sep_id;
- int context_separator_id;
- int late_mask_id;
- int late_query_prefix_id;
- int late_document_prefix_id;
- int late_skip_ids[64];
- int n_late_skip_ids;
- int append_terminal_token;
- int terminal_token_id;
- int renormalize_truncated;
- embed_model_t *cpu_model;
- embed_workspace_t *cpu_ws;
- embed_late_model_t *cpu_late_model;
- embed_late_workspace_t *cpu_late_ws;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
- embed_mlx_late_ctx_t *mlx_late_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
- embed_cuda_late_ctx_t *cuda_late_ctx;
-#endif
-} loaded_model;
-
-typedef struct client client;
-typedef struct job job;
-
-typedef struct {
- aeEventLoop *loop;
- int listen_fd;
- int completion_pipe[2];
- int signal_pipe[2];
- const char *host;
- int port;
- int batch_size;
- int max_batch_tokens;
- int batch_wait_us;
- char *api_key;
- loaded_model models[MODEL_COUNT];
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
-
- pthread_t worker;
- pthread_mutex_t mu;
- pthread_cond_t cv;
- int stopping;
- int worker_ready;
- int worker_init_rc;
- job *job_head;
- job *job_tail;
- job *done_head;
- job *done_tail;
-
- client *clients;
- int n_clients;
-} http_server;
-
-typedef struct {
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char version[16];
- char *body;
- size_t body_len;
- char *auth;
-} http_request;
-
-struct client {
- http_server *srv;
- int fd;
- int refcount;
- int cancelled;
- int linked;
- uint64_t last_active_ms;
- sbuf in;
- sbuf out;
- size_t sent;
- int header_done;
- size_t header_len;
- size_t content_length;
- int close_after_write;
- http_request req;
- client *prev;
- client *next;
-};
-
-struct job {
- http_server *srv;
- client *c;
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char *body;
- size_t body_len;
- char *auth;
- int status;
- char *content_type;
- char *extra_headers;
- char *response;
- size_t response_len;
- uint64_t created_ns;
- uint64_t started_ns;
- uint64_t parse_ns;
- uint64_t tokenize_ns;
- uint64_t infer_ns;
- uint64_t encode_ns;
- job *next;
-};
static int queue_write(client *c);
static void dispatch_request(client *c);