aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile2
-rw-r--r--src/server.c438
-rw-r--r--src/server_internal.h23
-rw-r--r--src/server_models.c457
4 files changed, 481 insertions, 439 deletions
diff --git a/Makefile b/Makefile
index 8ea9393..1a4ff2f 100644
--- a/Makefile
+++ b/Makefile
@@ -83,7 +83,7 @@ CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \
# Server-only objects: the embed-server binary and the server test link these
# alongside server.o. Not in libembed.a - the CLI and library do not use them.
-SERVER_SRCS = src/server_util.c src/sbuf.c src/base64.c src/server_json.c src/server_encode.c src/server_http.c
+SERVER_SRCS = src/server_util.c src/sbuf.c src/base64.c src/server_json.c src/server_encode.c src/server_http.c src/server_models.c
SERVER_OBJS = $(SERVER_SRCS:.c=.o)
SRCS = $(CORE_SRCS) $(TOKENIZER_SRCS) \
diff --git a/src/server.c b/src/server.c
index 411ad3d..f52d8d1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -52,48 +52,6 @@
#include <cjson/cJSON.h>
-static const model_info k_models[] = {
- {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT},
- /* BERT encoders: WordPiece tokenizer, learned positions, no instruction
- * prefix. all-MiniLM-L6-v2 mean-pools; bge-small-en-v1.5 pools the [CLS]
- * token. Both emit L2-normalized float32, so they speak the OpenAI API. */
- {"all-MiniLM-L6-v2", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, NULL},
- {"bge-small-en-v1.5", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"bge-base-en-v1.5", MODEL_KIND_STANDARD, 768, 768, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
-};
-
-static model_slot model_slot_for_id(const char *id) {
- if (!id)
- return MODEL_UNKNOWN;
- for (int i = 0; i < MODEL_COUNT; i++) {
- if (!strcmp(id, k_models[i].id))
- return (model_slot)i;
- }
- return MODEL_UNKNOWN;
-}
-
static volatile sig_atomic_t g_stop_requested = 0;
static volatile sig_atomic_t g_signal_wfd = -1;
@@ -400,222 +358,6 @@ static void job_set_timing_header(job *j) {
j->extra_headers = xstrdup(buf);
}
-static void free_token_bufs(token_buf *t, int n) {
- if (!t)
- return;
- for (int i = 0; i < n; i++)
- free(t[i].ids);
-}
-
-static void embedding_request_free(embedding_request *r) {
- if (!r)
- return;
- free_token_bufs(r->tokens, r->n_inputs);
- free(r->tokens);
- free(r->inputs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-static void contextual_request_free(contextual_request *r) {
- if (!r)
- return;
- for (int i = 0; i < r->n_docs; i++) {
- free(r->docs[i].ids);
- free(r->docs[i].spans);
- }
- free(r->docs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-static void late_tokens_free(late_tokens *t) {
- if (!t)
- return;
- free(t->ids);
- free(t->keep);
- memset(t, 0, sizeof(*t));
-}
-
-static void rerank_request_free(rerank_request *r) {
- if (!r)
- return;
- late_tokens_free(&r->query);
- for (int i = 0; i < r->n_documents; i++)
- late_tokens_free(&r->documents[i]);
- free(r->documents);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
- memset(out, 0, sizeof(*out));
- uint64_t t0 = nstime();
- if (m->wp_tok) {
- int n = 0;
- int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n);
- if (j)
- j->tokenize_ns += nstime() - t0;
- /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE
- * branch below appends its terminal token. A whitespace-only input
- * cleans to zero core tokens (core == NULL, n == 0); the bare
- * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the
- * returned buffer in place so only one allocation is touched. */
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = m->cls_id;
- ids[n + 1] = m->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- return 0;
- }
- out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
- if (j)
- j->tokenize_ns += nstime() - t0;
- if (!out->ids || out->n_tokens <= 0) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- if (m->append_terminal_token) {
- if (out->n_tokens == INT_MAX) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
- if (!ids) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- out->ids = ids;
- out->ids[out->n_tokens++] = m->terminal_token_id;
- }
- return 0;
-}
-
-/* Tokenize one embedding input, optionally prepending the model's query
- * instruction. The instruction and input are tokenized in one pass. */
-static int tokenize_input(
- loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
- if (!query_instruct)
- return tokenize_one(m, j, text, out);
- size_t plen = strlen(query_instruct);
- size_t tlen = strlen(text);
- char *buf = (char *)malloc(plen + tlen + 1);
- if (!buf) {
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memcpy(buf, query_instruct, plen);
- memcpy(buf + plen, text, tlen + 1);
- int rc = tokenize_one(m, j, buf, out);
- free(buf);
- return rc;
-}
-
-static int late_id_is_skipped(const loaded_model *m, int id) {
- for (int i = 0; i < m->n_late_skip_ids; i++) {
- if (m->late_skip_ids[i] == id)
- return 1;
- }
- return 0;
-}
-
-static int
-tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
- token_buf raw = {0};
- if (tokenize_one(m, j, text, &raw) != 0)
- return -1;
-
- int raw_tokens = raw.n_tokens;
- int target = raw_tokens + 1;
- if (is_query && target < EMBED_LATE_QUERY_TOKENS)
- target = EMBED_LATE_QUERY_TOKENS;
-
- out->ids = xmalloc((size_t)target * sizeof(*out->ids));
- out->ids[0] = raw.ids[0];
- out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
- if (raw_tokens > 1) {
- memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
- }
- out->n_tokens = raw_tokens + 1;
- if (is_query) {
- while (out->n_tokens < EMBED_LATE_QUERY_TOKENS)
- out->ids[out->n_tokens++] = m->late_mask_id;
- out->n_keep = out->n_tokens;
- } else {
- out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep));
- for (int i = 0; i < out->n_tokens; i++) {
- if (!late_id_is_skipped(m, out->ids[i]))
- out->keep[out->n_keep++] = i;
- }
- }
- free(raw.ids);
- return out->n_keep > 0 ? 0 : -1;
-}
-
-static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-static int model_embed_spans_batch(loaded_model *m,
- const embed_context_input_t *inputs,
- int batch,
- float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-static int model_uses_dense_batches(const loaded_model *m) {
-#ifdef USE_MLX
- return m->mlx_ctx != NULL;
-#else
- (void)m;
- return 0;
-#endif
-}
-
-/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
- * pads each dense row to the longest input in the chunk. */
-static int inference_batch_accepts_input(
- const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
- if (batch == 0)
- return 1;
- if (model_uses_dense_batches(m))
- return next_tokens <= max_batch_tokens / (batch + 1);
- return next_tokens <= max_batch_tokens - packed_tokens;
-}
-
static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
!strcmp(a->encoding, b->encoding);
@@ -1675,22 +1417,6 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
free(std_reqs);
}
-static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
- if (!m || !m->info || !config || config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode ||
- config->pooling_mode != m->info->pooling_mode ||
- config->normalize_embeddings != m->info->normalize_embeddings) {
- if (m && m->info && config)
- server_log("embed-server: model %s has incompatible config", m->info->id);
- return -1;
- }
- m->append_terminal_token = config->append_terminal_token;
- /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
- m->renormalize_truncated =
- config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
- return 0;
-}
-
static void *worker_main(void *arg) {
http_server *s = arg;
#ifdef USE_MLX
@@ -1866,170 +1592,6 @@ static int listen_on(const char *host, int port) {
return fd;
}
-static int load_one_model(http_server *s, model_slot slot, const char *path) {
- loaded_model *m = &s->models[slot];
- m->info = &k_models[slot];
- m->path = xstrdup(path);
- m->terminal_token_id = -1;
- m->cls_id = -1;
- m->sep_id = -1;
- /* Pick the tokenizer by the files present: a WordPiece vocab.txt marks a
- * BERT encoder; otherwise the Qwen byte-level BPE vocab.json. The two never
- * coexist in a model directory, so the probe is unambiguous. */
- char wp_path[1024];
- snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path);
- if (access(wp_path, R_OK) == 0) {
- m->wp_tok = wordpiece_tokenizer_load(path);
- if (!m->wp_tok) {
- server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path);
- return -1;
- }
- m->wp_tok_ws = wordpiece_workspace_new();
- if (!m->wp_tok_ws) {
- server_log("embed-server: failed to allocate WordPiece workspace: %s", path);
- return -1;
- }
- /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */
- m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]");
- m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]");
- if (m->cls_id < 0 || m->sep_id < 0) {
- server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path);
- return -1;
- }
- } else {
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
- m->tok = embed_tokenizer_load(vocab_path);
- if (!m->tok) {
- server_log("embed-server: failed to load tokenizer: %s", vocab_path);
- return -1;
- }
- m->tok_ws = embed_tokenizer_workspace_new();
- if (!m->tok_ws) {
- server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
- return -1;
- }
- /* The contextual chunk separator is the tokenizer's <|endoftext|>. The
- * released models keep added special tokens out of vocab.json, so the
- * family constant is the normal case; a vocab that does define the
- * token (e.g. small test models) takes precedence so the id stays
- * inside that model's embedding table. */
- int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>");
- m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
- * suffix - the same token as the separator, not the model's chat
- * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
- m->terminal_token_id = m->context_separator_id;
- if (m->info->kind == MODEL_KIND_LATE) {
- int id = embed_tokenizer_token_id(m->tok, "[MASK]");
- m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
- id = embed_tokenizer_token_id(m->tok, "[Q]");
- m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
- id = embed_tokenizer_token_id(m->tok, "[D]");
- m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
-
- const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p &&
- m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0]));
- p++) {
- char text[2] = {*p, '\0'};
- int n_ids = 0;
- int *ids = embed_tokenizer_encode(m->tok, text, &n_ids);
- if (ids && n_ids == 1)
- m->late_skip_ids[m->n_late_skip_ids++] = ids[0];
- free(ids);
- }
- }
- }
-#ifdef USE_MLX
- if (s->use_mlx) {
- /* MLX streams are thread-local. The inference worker loads the MLX
- * model so all MLX arrays/streams are created, used, and freed by the
- * same thread. */
- return 0;
- }
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- return 0;
- }
-#endif
- const embed_config_t *config = NULL;
- int token_dim = 0;
- if (m->info->kind == MODEL_KIND_LATE) {
- m->cpu_late_model = embed_late_model_load(path);
- if (!m->cpu_late_model) {
- server_log("embed-server: failed to load late model: %s", path);
- return -1;
- }
- m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
- if (!m->cpu_late_ws) {
- server_log("embed-server: failed to allocate late workspace: %s", path);
- return -1;
- }
- config = embed_late_model_config(m->cpu_late_model);
- token_dim = embed_late_model_token_dim(m->cpu_late_model);
- } else {
- m->cpu_model = embed_model_load(path);
- if (!m->cpu_model) {
- server_log("embed-server: failed to load model: %s", path);
- return -1;
- }
- m->cpu_ws = embed_workspace_new(m->cpu_model);
- if (!m->cpu_ws) {
- server_log("embed-server: failed to allocate workspace: %s", path);
- return -1;
- }
- config = embed_model_config(m->cpu_model);
- }
- if (configure_loaded_model(m, config, token_dim) != 0) {
- return -1;
- }
- if (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size)) {
- server_log("embed-server: late special-token ids exceed model vocab");
- return -1;
- }
- return 0;
-}
-
-static void free_models(http_server *s) {
- for (int i = 0; i < MODEL_COUNT; i++) {
- loaded_model *m = &s->models[i];
- free(m->path);
- if (m->tok_ws)
- embed_tokenizer_workspace_free(m->tok_ws);
- if (m->tok)
- embed_tokenizer_free(m->tok);
- if (m->wp_tok_ws)
- wordpiece_workspace_free(m->wp_tok_ws);
- if (m->wp_tok)
- wordpiece_tokenizer_free(m->wp_tok);
- if (m->cpu_ws)
- embed_workspace_free(m->cpu_ws);
- if (m->cpu_model)
- embed_model_free(m->cpu_model);
- if (m->cpu_late_ws)
- embed_late_workspace_free(m->cpu_late_ws);
- if (m->cpu_late_model)
- embed_late_model_free(m->cpu_late_model);
-#ifdef USE_MLX
- if (m->mlx_ctx)
- embed_mlx_free(m->mlx_ctx);
- if (m->mlx_late_ctx)
- embed_mlx_late_free(m->mlx_late_ctx);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- embed_cuda_free(m->cuda_ctx);
- if (m->cuda_late_ctx)
- embed_cuda_late_free(m->cuda_late_ctx);
-#endif
- }
-}
-
static uint64_t physical_memory_nbytes(void) {
#ifdef __APPLE__
uint64_t bytes = 0;
diff --git a/src/server_internal.h b/src/server_internal.h
index 9f7ba81..88bd5d4 100644
--- a/src/server_internal.h
+++ b/src/server_internal.h
@@ -338,4 +338,27 @@ void set_response_from_buf(job *j, sbuf *b);
int render_embedding_response(embedding_request *r, const float *embs);
void render_contextual_response(contextual_request *r, const float *embs);
+/* ---- server_models.c: registry, load/lifecycle, tokenize/inference dispatch ---- */
+extern const model_info k_models[];
+model_slot model_slot_for_id(const char *id);
+void free_token_bufs(token_buf *t, int n);
+void embedding_request_free(embedding_request *r);
+void contextual_request_free(contextual_request *r);
+void late_tokens_free(late_tokens *t);
+void rerank_request_free(rerank_request *r);
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out);
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
+int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out);
+int model_embed_spans_batch(loaded_model *m,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out);
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
+int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim);
+int load_one_model(http_server *s, model_slot slot, const char *path);
+void free_models(http_server *s);
+
#endif /* EMBED_SERVER_INTERNAL_H */
diff --git a/src/server_models.c b/src/server_models.c
new file mode 100644
index 0000000..ebe5811
--- /dev/null
+++ b/src/server_models.c
@@ -0,0 +1,457 @@
+#include "server_internal.h"
+#include "server_util.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * Model registry, load/lifecycle, and tokenize/inference dispatch
+ *
+ * k_models is the static table of every model the server can serve (name,
+ * dimensions, attention/pooling mode, API family). load_one_model resolves a
+ * served id to a slot, picks the tokenizer by the files present (WordPiece
+ * vocab.txt -> BERT, else byte-level BPE vocab.json), and loads the CPU model
+ * (GPU backends load on the worker thread). The tokenize_* and model_embed_*
+ * helpers are the single dispatch point the request handlers call, hiding the
+ * CPU/MLX/CUDA backend choice behind one signature.
+ * ======================================================================== */
+
+const model_info k_models[] = {
+ {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
+ {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
+ {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
+ {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
+ {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
+ {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT},
+ /* BERT encoders: WordPiece tokenizer, learned positions, no instruction
+ * prefix. all-MiniLM-L6-v2 mean-pools; bge-small-en-v1.5 pools the [CLS]
+ * token. Both emit L2-normalized float32, so they speak the OpenAI API. */
+ {"all-MiniLM-L6-v2", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, NULL},
+ {"bge-small-en-v1.5", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
+ {"bge-base-en-v1.5", MODEL_KIND_STANDARD, 768, 768, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
+ {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
+ {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
+ {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
+ {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
+};
+
+model_slot model_slot_for_id(const char *id) {
+ if (!id)
+ return MODEL_UNKNOWN;
+ for (int i = 0; i < MODEL_COUNT; i++) {
+ if (!strcmp(id, k_models[i].id))
+ return (model_slot)i;
+ }
+ return MODEL_UNKNOWN;
+}
+
+void free_token_bufs(token_buf *t, int n) {
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
+}
+
+void embedding_request_free(embedding_request *r) {
+ if (!r)
+ return;
+ free_token_bufs(r->tokens, r->n_inputs);
+ free(r->tokens);
+ free(r->inputs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void contextual_request_free(contextual_request *r) {
+ if (!r)
+ return;
+ for (int i = 0; i < r->n_docs; i++) {
+ free(r->docs[i].ids);
+ free(r->docs[i].spans);
+ }
+ free(r->docs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void late_tokens_free(late_tokens *t) {
+ if (!t)
+ return;
+ free(t->ids);
+ free(t->keep);
+ memset(t, 0, sizeof(*t));
+}
+
+void rerank_request_free(rerank_request *r) {
+ if (!r)
+ return;
+ late_tokens_free(&r->query);
+ for (int i = 0; i < r->n_documents; i++)
+ late_tokens_free(&r->documents[i]);
+ free(r->documents);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
+ memset(out, 0, sizeof(*out));
+ uint64_t t0 = nstime();
+ if (m->wp_tok) {
+ int n = 0;
+ int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE
+ * branch below appends its terminal token. A whitespace-only input
+ * cleans to zero core tokens (core == NULL, n == 0); the bare
+ * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the
+ * returned buffer in place so only one allocation is touched. */
+ if (n < 0 || n > INT_MAX - 2) {
+ free(core);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = m->cls_id;
+ ids[n + 1] = m->sep_id;
+ out->ids = ids;
+ out->n_tokens = n + 2;
+ return 0;
+ }
+ out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ if (!out->ids || out->n_tokens <= 0) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ if (m->append_terminal_token) {
+ if (out->n_tokens == INT_MAX) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ if (!ids) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ out->ids = ids;
+ out->ids[out->n_tokens++] = m->terminal_token_id;
+ }
+ return 0;
+}
+
+/* Tokenize one embedding input, optionally prepending the model's query
+ * instruction. The instruction and input are tokenized in one pass. */
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
+ if (!query_instruct)
+ return tokenize_one(m, j, text, out);
+ size_t plen = strlen(query_instruct);
+ size_t tlen = strlen(text);
+ char *buf = (char *)malloc(plen + tlen + 1);
+ if (!buf) {
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ memcpy(buf, query_instruct, plen);
+ memcpy(buf + plen, text, tlen + 1);
+ int rc = tokenize_one(m, j, buf, out);
+ free(buf);
+ return rc;
+}
+
+static int late_id_is_skipped(const loaded_model *m, int id) {
+ for (int i = 0; i < m->n_late_skip_ids; i++) {
+ if (m->late_skip_ids[i] == id)
+ return 1;
+ }
+ return 0;
+}
+
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
+ token_buf raw = {0};
+ if (tokenize_one(m, j, text, &raw) != 0)
+ return -1;
+
+ int raw_tokens = raw.n_tokens;
+ int target = raw_tokens + 1;
+ if (is_query && target < EMBED_LATE_QUERY_TOKENS)
+ target = EMBED_LATE_QUERY_TOKENS;
+
+ out->ids = xmalloc((size_t)target * sizeof(*out->ids));
+ out->ids[0] = raw.ids[0];
+ out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
+ if (raw_tokens > 1) {
+ memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
+ }
+ out->n_tokens = raw_tokens + 1;
+ if (is_query) {
+ while (out->n_tokens < EMBED_LATE_QUERY_TOKENS)
+ out->ids[out->n_tokens++] = m->late_mask_id;
+ out->n_keep = out->n_tokens;
+ } else {
+ out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep));
+ for (int i = 0; i < out->n_tokens; i++) {
+ if (!late_id_is_skipped(m, out->ids[i]))
+ out->keep[out->n_keep++] = i;
+ }
+ }
+ free(raw.ids);
+ return out->n_keep > 0 ? 0 : -1;
+}
+
+int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
+#ifdef USE_MLX
+ if (m->mlx_ctx)
+ return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
+#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx)
+ return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out);
+#endif
+ return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
+}
+
+int model_embed_spans_batch(loaded_model *m,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out) {
+#ifdef USE_MLX
+ if (m->mlx_ctx)
+ return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
+#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx)
+ return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
+#endif
+ return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
+}
+
+static int model_uses_dense_batches(const loaded_model *m) {
+#ifdef USE_MLX
+ return m->mlx_ctx != NULL;
+#else
+ (void)m;
+ return 0;
+#endif
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
+int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size != m->info->dim ||
+ token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode ||
+ config->pooling_mode != m->info->pooling_mode ||
+ config->normalize_embeddings != m->info->normalize_embeddings) {
+ if (m && m->info && config)
+ server_log("embed-server: model %s has incompatible config", m->info->id);
+ return -1;
+ }
+ m->append_terminal_token = config->append_terminal_token;
+ /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
+ m->renormalize_truncated =
+ config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ return 0;
+}
+
+int load_one_model(http_server *s, model_slot slot, const char *path) {
+ loaded_model *m = &s->models[slot];
+ m->info = &k_models[slot];
+ m->path = xstrdup(path);
+ m->terminal_token_id = -1;
+ m->cls_id = -1;
+ m->sep_id = -1;
+ /* Pick the tokenizer by the files present: a WordPiece vocab.txt marks a
+ * BERT encoder; otherwise the Qwen byte-level BPE vocab.json. The two never
+ * coexist in a model directory, so the probe is unambiguous. */
+ char wp_path[1024];
+ snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path);
+ if (access(wp_path, R_OK) == 0) {
+ m->wp_tok = wordpiece_tokenizer_load(path);
+ if (!m->wp_tok) {
+ server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path);
+ return -1;
+ }
+ m->wp_tok_ws = wordpiece_workspace_new();
+ if (!m->wp_tok_ws) {
+ server_log("embed-server: failed to allocate WordPiece workspace: %s", path);
+ return -1;
+ }
+ /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */
+ m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]");
+ m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]");
+ if (m->cls_id < 0 || m->sep_id < 0) {
+ server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path);
+ return -1;
+ }
+ } else {
+ char vocab_path[1024];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
+ m->tok = embed_tokenizer_load(vocab_path);
+ if (!m->tok) {
+ server_log("embed-server: failed to load tokenizer: %s", vocab_path);
+ return -1;
+ }
+ m->tok_ws = embed_tokenizer_workspace_new();
+ if (!m->tok_ws) {
+ server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
+ return -1;
+ }
+ /* The contextual chunk separator is the tokenizer's <|endoftext|>. The
+ * released models keep added special tokens out of vocab.json, so the
+ * family constant is the normal case; a vocab that does define the
+ * token (e.g. small test models) takes precedence so the id stays
+ * inside that model's embedding table. */
+ int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>");
+ m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
+ * suffix - the same token as the separator, not the model's chat
+ * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
+ m->terminal_token_id = m->context_separator_id;
+ if (m->info->kind == MODEL_KIND_LATE) {
+ int id = embed_tokenizer_token_id(m->tok, "[MASK]");
+ m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
+ id = embed_tokenizer_token_id(m->tok, "[Q]");
+ m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
+ id = embed_tokenizer_token_id(m->tok, "[D]");
+ m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
+
+ const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
+ for (const char *p = punct;
+ *p &&
+ m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0]));
+ p++) {
+ char text[2] = {*p, '\0'};
+ int n_ids = 0;
+ int *ids = embed_tokenizer_encode(m->tok, text, &n_ids);
+ if (ids && n_ids == 1)
+ m->late_skip_ids[m->n_late_skip_ids++] = ids[0];
+ free(ids);
+ }
+ }
+ }
+#ifdef USE_MLX
+ if (s->use_mlx) {
+ /* MLX streams are thread-local. The inference worker loads the MLX
+ * model so all MLX arrays/streams are created, used, and freed by the
+ * same thread. */
+ return 0;
+ }
+#endif
+#ifdef USE_CUDA
+ if (s->use_cuda) {
+ return 0;
+ }
+#endif
+ const embed_config_t *config = NULL;
+ int token_dim = 0;
+ if (m->info->kind == MODEL_KIND_LATE) {
+ m->cpu_late_model = embed_late_model_load(path);
+ if (!m->cpu_late_model) {
+ server_log("embed-server: failed to load late model: %s", path);
+ return -1;
+ }
+ m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
+ if (!m->cpu_late_ws) {
+ server_log("embed-server: failed to allocate late workspace: %s", path);
+ return -1;
+ }
+ config = embed_late_model_config(m->cpu_late_model);
+ token_dim = embed_late_model_token_dim(m->cpu_late_model);
+ } else {
+ m->cpu_model = embed_model_load(path);
+ if (!m->cpu_model) {
+ server_log("embed-server: failed to load model: %s", path);
+ return -1;
+ }
+ m->cpu_ws = embed_workspace_new(m->cpu_model);
+ if (!m->cpu_ws) {
+ server_log("embed-server: failed to allocate workspace: %s", path);
+ return -1;
+ }
+ config = embed_model_config(m->cpu_model);
+ }
+ if (configure_loaded_model(m, config, token_dim) != 0) {
+ return -1;
+ }
+ if (m->info->kind == MODEL_KIND_LATE &&
+ (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
+ m->late_document_prefix_id >= config->vocab_size)) {
+ server_log("embed-server: late special-token ids exceed model vocab");
+ return -1;
+ }
+ return 0;
+}
+
+void free_models(http_server *s) {
+ for (int i = 0; i < MODEL_COUNT; i++) {
+ loaded_model *m = &s->models[i];
+ free(m->path);
+ if (m->tok_ws)
+ embed_tokenizer_workspace_free(m->tok_ws);
+ if (m->tok)
+ embed_tokenizer_free(m->tok);
+ if (m->wp_tok_ws)
+ wordpiece_workspace_free(m->wp_tok_ws);
+ if (m->wp_tok)
+ wordpiece_tokenizer_free(m->wp_tok);
+ if (m->cpu_ws)
+ embed_workspace_free(m->cpu_ws);
+ if (m->cpu_model)
+ embed_model_free(m->cpu_model);
+ if (m->cpu_late_ws)
+ embed_late_workspace_free(m->cpu_late_ws);
+ if (m->cpu_late_model)
+ embed_late_model_free(m->cpu_late_model);
+#ifdef USE_MLX
+ if (m->mlx_ctx)
+ embed_mlx_free(m->mlx_ctx);
+ if (m->mlx_late_ctx)
+ embed_mlx_late_free(m->mlx_late_ctx);
+#endif
+#ifdef USE_CUDA
+ if (m->cuda_ctx)
+ embed_cuda_free(m->cuda_ctx);
+ if (m->cuda_late_ctx)
+ embed_cuda_late_free(m->cuda_late_ctx);
+#endif
+ }
+}