aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:56:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 17:56:50 +0200
commit230dc1748aa68cccde6de9aa3cdade7d64aede56 (patch)
tree48174fe9d12890b0ecf9bf818c3ad80262ceb9b8
parente54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c (diff)
downloadffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.tar.gz
ffwd-230dc1748aa68cccde6de9aa3cdade7d64aede56.zip
server: extract shared structs and macros into server_internal.h
-rw-r--r--src/server.c212
-rw-r--r--src/server_internal.h241
2 files changed, 242 insertions, 211 deletions
diff --git a/src/server.c b/src/server.c
index c3abbac..0305689 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4,6 +4,7 @@
#include "build.h"
#include "server.h"
+#include "server_internal.h"
#include "embed.h"
#include "safetensors.h"
#include "kernels.h"
@@ -48,54 +49,6 @@
#include <cjson/cJSON.h>
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
-#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
-
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
-#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
-/* `text_type: query` prepends the model's published retrieval instruction.
- * Documents and the default pass through unchanged. Callers needing a custom
- * instruction prepend it themselves and leave text_type at "document". */
-#define EMBED_QWEN3_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
-#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the " \
- "query\nQuery: "
-/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
- * short-request throughput over 8 with no long-document penalty, while 128
- * was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
-#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
-/* Microseconds the worker waits for more requests before dispatching a batch.
- * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
- * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
- * Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
-#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
-
-typedef struct {
- char *ptr;
- size_t len;
- size_t cap;
-} sbuf;
-
static void die_oom(void) {
fprintf(stderr, "embed-server: out of memory\n");
exit(1);
@@ -223,50 +176,6 @@ static void sbuf_free(sbuf *b) {
memset(b, 0, sizeof(*b));
}
-typedef enum {
- MODEL_STD_06,
- MODEL_STD_4,
- MODEL_QWEN3_06,
- MODEL_QWEN3_4,
- MODEL_QWEN3_8,
- MODEL_GTE_QWEN2_15,
- MODEL_MINILM_L6,
- MODEL_BGE_SMALL,
- MODEL_BGE_BASE,
- MODEL_BGE_LARGE,
- MODEL_CTX_06,
- MODEL_CTX_4,
- MODEL_LATE_06,
- MODEL_COUNT,
- MODEL_UNKNOWN
-} model_slot;
-
-typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
-
-/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
- * API: the canonical output is the tanh int8 quantization, so the default
- * encoding is base64_int8 and "float" is the int8-decoded view (int8/128).
- * Instruction embedding models follow the OpenAI-compatible API: "float" is
- * the true float32 vector and "base64" is base64 of little-endian float32.
- * int8 quantization underuses the range of L2-normalized vectors, so it is not
- * used for these models. */
-typedef enum {
- EMBED_API_PERPLEXITY = 0,
- EMBED_API_OPENAI = 1,
-} embedding_api_t;
-
-typedef struct {
- const char *id;
- model_kind kind;
- int dim;
- int min_dim;
- int token_dim;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- embedding_api_t api;
- const char *query_instruct;
-} model_info;
static const model_info k_models[] = {
{"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
@@ -310,125 +219,6 @@ static model_slot model_slot_for_id(const char *id) {
return MODEL_UNKNOWN;
}
-typedef struct {
- const model_info *info;
- char *path;
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model.
- * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- int cls_id;
- int sep_id;
- int context_separator_id;
- int late_mask_id;
- int late_query_prefix_id;
- int late_document_prefix_id;
- int late_skip_ids[64];
- int n_late_skip_ids;
- int append_terminal_token;
- int terminal_token_id;
- int renormalize_truncated;
- embed_model_t *cpu_model;
- embed_workspace_t *cpu_ws;
- embed_late_model_t *cpu_late_model;
- embed_late_workspace_t *cpu_late_ws;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
- embed_mlx_late_ctx_t *mlx_late_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
- embed_cuda_late_ctx_t *cuda_late_ctx;
-#endif
-} loaded_model;
-
-typedef struct client client;
-typedef struct job job;
-
-typedef struct {
- aeEventLoop *loop;
- int listen_fd;
- int completion_pipe[2];
- int signal_pipe[2];
- const char *host;
- int port;
- int batch_size;
- int max_batch_tokens;
- int batch_wait_us;
- char *api_key;
- loaded_model models[MODEL_COUNT];
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
-
- pthread_t worker;
- pthread_mutex_t mu;
- pthread_cond_t cv;
- int stopping;
- int worker_ready;
- int worker_init_rc;
- job *job_head;
- job *job_tail;
- job *done_head;
- job *done_tail;
-
- client *clients;
- int n_clients;
-} http_server;
-
-typedef struct {
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char version[16];
- char *body;
- size_t body_len;
- char *auth;
-} http_request;
-
-struct client {
- http_server *srv;
- int fd;
- int refcount;
- int cancelled;
- int linked;
- uint64_t last_active_ms;
- sbuf in;
- sbuf out;
- size_t sent;
- int header_done;
- size_t header_len;
- size_t content_length;
- int close_after_write;
- http_request req;
- client *prev;
- client *next;
-};
-
-struct job {
- http_server *srv;
- client *c;
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char *body;
- size_t body_len;
- char *auth;
- int status;
- char *content_type;
- char *extra_headers;
- char *response;
- size_t response_len;
- uint64_t created_ns;
- uint64_t started_ns;
- uint64_t parse_ns;
- uint64_t tokenize_ns;
- uint64_t infer_ns;
- uint64_t encode_ns;
- job *next;
-};
static int queue_write(client *c);
static void dispatch_request(client *c);
diff --git a/src/server_internal.h b/src/server_internal.h
new file mode 100644
index 0000000..c72831e
--- /dev/null
+++ b/src/server_internal.h
@@ -0,0 +1,241 @@
+#ifndef EMBED_SERVER_INTERNAL_H
+#define EMBED_SERVER_INTERNAL_H
+
+/* Declarations shared across the embed-server translation units (HTTP framing,
+ * JSON validation, output encoding, scheduling, model lifecycle, endpoint
+ * handlers). The server is split by concern but compiles as several objects
+ * linked into one binary; this header is what they share. Not a public API -
+ * the CLI and library do not include it. */
+
+#include "embed.h"
+#include "tokenizer_bpe.h"
+#include "tokenizer_wordpiece.h"
+
+#ifdef USE_MLX
+#include "mlx.h"
+#endif
+#ifdef USE_CUDA
+#include "cuda.h"
+#endif
+
+#include "deps/ae/ae.h"
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define EMBED_HTTP_IO_BUF 8192
+#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
+#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define EMBED_HTTP_MAX_PATH 255u
+#define EMBED_HTTP_BACKLOG 128
+#define EMBED_HTTP_SETSIZE 10240
+#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
+#define EMBED_HTTP_SWEEP_MS 1000
+
+#define EMBED_API_MAX_STANDARD_INPUTS 512
+#define EMBED_API_MAX_CONTEXT_DOCS 512
+#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
+#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
+#define EMBED_API_MAX_ITEM_TOKENS 32768
+#define EMBED_API_MAX_TOTAL_TOKENS 120000
+#define EMBED_LATE_QUERY_TOKENS 32
+#define EMBED_LATE_MASK_TOKEN_ID 151642
+#define EMBED_LATE_QUERY_PREFIX_ID 151669
+#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
+/* `text_type: query` prepends the model's published retrieval instruction.
+ * Documents and the default pass through unchanged. Callers needing a custom
+ * instruction prepend it themselves and leave text_type at "document". */
+#define EMBED_QWEN3_QUERY_INSTRUCT \
+ "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
+#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
+ "Instruct: Given a web search query, retrieve relevant passages that answer the " \
+ "query\nQuery: "
+/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
+ * short-request throughput over 8 with no long-document penalty, while 128
+ * was no faster and inflated long-document tail latency. */
+#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
+#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
+/* Microseconds the worker waits for more requests before dispatching a batch.
+ * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
+ * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
+ * Override with --batch-wait-us. */
+#define EMBED_SERVER_BATCH_WAIT_US 0
+#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
+#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
+#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
+#define EMBED_MLX_RESIDENT_MULTIPLIER 2
+
+typedef struct {
+ char *ptr;
+ size_t len;
+ size_t cap;
+} sbuf;
+
+typedef enum {
+ MODEL_STD_06,
+ MODEL_STD_4,
+ MODEL_QWEN3_06,
+ MODEL_QWEN3_4,
+ MODEL_QWEN3_8,
+ MODEL_GTE_QWEN2_15,
+ MODEL_MINILM_L6,
+ MODEL_BGE_SMALL,
+ MODEL_BGE_BASE,
+ MODEL_BGE_LARGE,
+ MODEL_CTX_06,
+ MODEL_CTX_4,
+ MODEL_LATE_06,
+ MODEL_COUNT,
+ MODEL_UNKNOWN
+} model_slot;
+
+typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
+
+/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
+ * API: the canonical output is the tanh int8 quantization, so the default
+ * encoding is base64_int8 and "float" is the int8-decoded view (int8/128).
+ * Instruction embedding models follow the OpenAI-compatible API: "float" is
+ * the true float32 vector and "base64" is base64 of little-endian float32.
+ * int8 quantization underuses the range of L2-normalized vectors, so it is not
+ * used for these models. */
+typedef enum {
+ EMBED_API_PERPLEXITY = 0,
+ EMBED_API_OPENAI = 1,
+} embedding_api_t;
+
+typedef struct {
+ const char *id;
+ model_kind kind;
+ int dim;
+ int min_dim;
+ int token_dim;
+ embed_attention_mode_t attention_mode;
+ embed_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
+ embedding_api_t api;
+ const char *query_instruct;
+} model_info;
+
+typedef struct {
+ const model_info *info;
+ char *path;
+ embed_tokenizer_t *tok;
+ embed_tokenizer_workspace_t *tok_ws;
+ /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model.
+ * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the
+ * Qwen byte-level BPE fields above stay NULL. */
+ wordpiece_tokenizer_t *wp_tok;
+ wordpiece_workspace_t *wp_tok_ws;
+ int cls_id;
+ int sep_id;
+ int context_separator_id;
+ int late_mask_id;
+ int late_query_prefix_id;
+ int late_document_prefix_id;
+ int late_skip_ids[64];
+ int n_late_skip_ids;
+ int append_terminal_token;
+ int terminal_token_id;
+ int renormalize_truncated;
+ embed_model_t *cpu_model;
+ embed_workspace_t *cpu_ws;
+ embed_late_model_t *cpu_late_model;
+ embed_late_workspace_t *cpu_late_ws;
+#ifdef USE_MLX
+ embed_mlx_ctx_t *mlx_ctx;
+ embed_mlx_late_ctx_t *mlx_late_ctx;
+#endif
+#ifdef USE_CUDA
+ embed_cuda_ctx_t *cuda_ctx;
+ embed_cuda_late_ctx_t *cuda_late_ctx;
+#endif
+} loaded_model;
+
+typedef struct client client;
+typedef struct job job;
+
+typedef struct {
+ aeEventLoop *loop;
+ int listen_fd;
+ int completion_pipe[2];
+ int signal_pipe[2];
+ const char *host;
+ int port;
+ int batch_size;
+ int max_batch_tokens;
+ int batch_wait_us;
+ char *api_key;
+ loaded_model models[MODEL_COUNT];
+ int use_mlx;
+ int use_cuda;
+ int mlx_quantize_bits;
+ int mlx_quantize_group_size;
+
+ pthread_t worker;
+ pthread_mutex_t mu;
+ pthread_cond_t cv;
+ int stopping;
+ int worker_ready;
+ int worker_init_rc;
+ job *job_head;
+ job *job_tail;
+ job *done_head;
+ job *done_tail;
+
+ client *clients;
+ int n_clients;
+} http_server;
+
+typedef struct {
+ char method[8];
+ char path[EMBED_HTTP_MAX_PATH + 1];
+ char version[16];
+ char *body;
+ size_t body_len;
+ char *auth;
+} http_request;
+
+struct client {
+ http_server *srv;
+ int fd;
+ int refcount;
+ int cancelled;
+ int linked;
+ uint64_t last_active_ms;
+ sbuf in;
+ sbuf out;
+ size_t sent;
+ int header_done;
+ size_t header_len;
+ size_t content_length;
+ int close_after_write;
+ http_request req;
+ client *prev;
+ client *next;
+};
+
+struct job {
+ http_server *srv;
+ client *c;
+ char method[8];
+ char path[EMBED_HTTP_MAX_PATH + 1];
+ char *body;
+ size_t body_len;
+ char *auth;
+ int status;
+ char *content_type;
+ char *extra_headers;
+ char *response;
+ size_t response_len;
+ uint64_t created_ns;
+ uint64_t started_ns;
+ uint64_t parse_ns;
+ uint64_t tokenize_ns;
+ uint64_t infer_ns;
+ uint64_t encode_ns;
+ job *next;
+};
+
+#endif /* EMBED_SERVER_INTERNAL_H */