aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-18 06:17:41 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-18 06:40:54 +0200
commitd4bdb8dae753462f621507d5fc3582e0e53da35c (patch)
tree2e67f32e02fd6ba739e1601493246086d7e0b81b /src
parent262804db7a2e9ee2e56a7d7faed646152b1e486b (diff)
downloadffwd-d4bdb8dae753462f621507d5fc3582e0e53da35c.tar.gz
ffwd-d4bdb8dae753462f621507d5fc3582e0e53da35c.zip
server: serve models by explicit labels
Diffstat (limited to 'src')
-rw-r--r--src/mlx.c3
-rw-r--r--src/server.c218
-rw-r--r--src/server.h18
-rw-r--r--src/server_encode.c16
-rw-r--r--src/server_handlers.c81
-rw-r--r--src/server_internal.h60
-rw-r--r--src/server_json.c20
-rw-r--r--src/server_models.c230
8 files changed, 416 insertions, 230 deletions
diff --git a/src/mlx.c b/src/mlx.c
index 4d5abe6..7d05e91 100644
--- a/src/mlx.c
+++ b/src/mlx.c
@@ -520,6 +520,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
}
ctx->layer_start = layer_start;
ctx->layer_end = layer_end;
+ int h = c->hidden_size;
ctx->ms = multi_safetensors_open(model_dir);
if (!ctx->ms) {
@@ -541,7 +542,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
goto fail;
}
- int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
+ int qd = c->q_dim, kvd = c->kv_dim;
int inter = c->intermediate_size, hd = c->head_dim;
char name[256];
diff --git a/src/server.c b/src/server.c
index cb5a857..808c6f8 100644
--- a/src/server.c
+++ b/src/server.c
@@ -79,7 +79,6 @@ static bool auth_ok(http_server *s, const char *auth) {
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
}
-
void dispatch_request(client *c) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -190,10 +189,8 @@ static void *worker_main(void *arg) {
#ifdef USE_MLX
if (s->use_mlx) {
int rc = 0;
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
- if (!m->info)
- continue;
embed_mlx_options_t mlx_opts = {
.quantize_bits = s->mlx_quantize_bits,
.quantize_group_size = s->mlx_quantize_group_size,
@@ -230,7 +227,7 @@ static void *worker_main(void *arg) {
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
if (m->mlx_ctx) {
embed_mlx_free(m->mlx_ctx);
@@ -248,10 +245,8 @@ static void *worker_main(void *arg) {
#ifdef USE_CUDA
if (s->use_cuda) {
int rc = 0;
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
- if (!m->info)
- continue;
if (m->info->kind == MODEL_KIND_LATE)
m->cuda_late_ctx = embed_cuda_late_load(m->path);
else
@@ -284,7 +279,7 @@ static void *worker_main(void *arg) {
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
if (m->cuda_ctx) {
embed_cuda_free(m->cuda_ctx);
@@ -315,7 +310,7 @@ static void *worker_main(void *arg) {
}
#ifdef USE_MLX
if (s->use_mlx) {
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
if (m->mlx_ctx) {
embed_mlx_free(m->mlx_ctx);
@@ -330,7 +325,7 @@ static void *worker_main(void *arg) {
#endif
#ifdef USE_CUDA
if (s->use_cuda) {
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
if (m->cuda_ctx) {
embed_cuda_free(m->cuda_ctx);
@@ -375,23 +370,34 @@ static uint64_t physical_memory_nbytes(void) {
}
static int validate_model_specs(const embed_server_config_t *cfg) {
- int seen[MODEL_COUNT] = {0};
for (int i = 0; i < cfg->n_models; i++) {
const embed_server_model_spec_t *spec = &cfg->models[i];
- model_slot slot = model_slot_for_id(spec->id);
- if (slot == MODEL_UNKNOWN) {
- fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : "<null>");
+ if (!spec->id || !spec->id[0]) {
+ fprintf(stderr, "embed-server: model label must not be empty\n");
return -1;
}
if (!spec->path || !spec->path[0]) {
fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id);
return -1;
}
- if (seen[slot]) {
- fprintf(stderr, "embed-server: duplicate model id: %s\n", spec->id);
+ if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) {
+ fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id);
+ return -1;
+ }
+ if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) {
+ fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id);
+ return -1;
+ }
+ if (spec->min_dim < 0) {
+ fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id);
return -1;
}
- seen[slot] = 1;
+ for (int j = 0; j < i; j++) {
+ if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) {
+ fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id);
+ return -1;
+ }
+ }
}
return 0;
}
@@ -452,7 +458,7 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) {
int embed_run_server(const embed_server_config_t *cfg) {
if (!cfg || !cfg->models || cfg->n_models <= 0) {
- fprintf(stderr, "embed-server: at least one --model MODEL_ID=PATH is required\n");
+ fprintf(stderr, "embed-server: at least one model label and path is required\n");
return 1;
}
if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0)
@@ -485,13 +491,7 @@ int embed_run_server(const embed_server_config_t *cfg) {
pthread_cond_init(&s.cv, NULL);
for (int i = 0; i < cfg->n_models; i++) {
- model_slot slot = model_slot_for_id(cfg->models[i].id);
- if (slot == MODEL_UNKNOWN) {
- fprintf(stderr, "embed-server: unknown model id: %s\n", cfg->models[i].id);
- free_models(&s);
- return 1;
- }
- if (load_one_model(&s, slot, cfg->models[i].path) != 0) {
+ if (load_one_model(&s, &cfg->models[i]) != 0) {
free_models(&s);
return 1;
}
@@ -599,10 +599,17 @@ int embed_run_server(const embed_server_config_t *cfg) {
static void print_usage(const char *prog) {
fprintf(stderr,
- "Usage: %s --model ID=DIR [options]\n"
+ "Usage: %s --model LABEL=DIR [options]\n"
"\n"
"Options:\n"
- " --model ID=DIR Model to serve (repeatable)\n"
+ " --model LABEL=DIR[:key=val]\n"
+ " Standard embedding model (repeatable)\n"
+ " --contextual-model LABEL=DIR[:key=val]\n"
+ " Contextual embedding model (repeatable)\n"
+ " --late-model LABEL=DIR[:key=val]\n"
+ " Late-interaction rerank model (repeatable)\n"
+ " keys: api=openai|perplexity, min_dim=N,\n"
+ " query=TEXT (last key; \\n escapes newline)\n"
#ifdef USE_MLX
" --mlx Use Apple MLX GPU backend\n"
#endif
@@ -641,14 +648,15 @@ static void print_usage(const char *prog) {
" -h, --help Show this help\n"
"\n"
"Examples:\n"
- " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " %s --model docs=./model --port 8000\n"
+ " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n"
#ifdef USE_MLX
" %s --mlx --mlx-quant-bits 8 \\\n"
- " --model pplx-embed-v1-4b=./model-4b-bf16\n",
- prog, prog, prog);
+ " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n",
+ prog, prog, prog, prog);
#else
,
- prog, prog);
+ prog, prog, prog);
#endif
}
@@ -658,10 +666,82 @@ typedef struct {
int cap;
} model_specs_t;
-static int append_model_spec(model_specs_t *specs, const char *arg) {
+static char *dup_range(const char *p, size_t n) {
+ char *s = (char *)malloc(n + 1);
+ if (!s)
+ return NULL;
+ memcpy(s, p, n);
+ s[n] = '\0';
+ return s;
+}
+
+static char *unescape_model_option(const char *s) {
+ size_t n = strlen(s);
+ char *out = (char *)malloc(n + 1);
+ if (!out)
+ return NULL;
+ size_t w = 0;
+ for (size_t r = 0; r < n; r++) {
+ if (s[r] == '\\' && r + 1 < n) {
+ r++;
+ if (s[r] == 'n')
+ out[w++] = '\n';
+ else if (s[r] == 't')
+ out[w++] = '\t';
+ else if (s[r] == 'r')
+ out[w++] = '\r';
+ else
+ out[w++] = s[r];
+ } else {
+ out[w++] = s[r];
+ }
+ }
+ out[w] = '\0';
+ return out;
+}
+
+static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) {
+ const char *eq = strchr(opt, '=');
+ if (!eq || eq == opt || !eq[1]) {
+ fprintf(stderr, "%s option expects key=value: %s\n", flag, opt);
+ return -1;
+ }
+ size_t key_len = (size_t)(eq - opt);
+ const char *val = eq + 1;
+ if (key_len == 3 && strncmp(opt, "api", key_len) == 0) {
+ if (!strcmp(val, "openai")) {
+ spec->api = EMBED_SERVER_API_OPENAI;
+ } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) {
+ spec->api = EMBED_SERVER_API_PERPLEXITY;
+ } else if (!strcmp(val, "default")) {
+ spec->api = EMBED_SERVER_API_DEFAULT;
+ } else {
+ fprintf(stderr, "%s api must be openai or perplexity\n", flag);
+ return -1;
+ }
+ return 0;
+ }
+ if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) {
+ char *end = NULL;
+ long v = strtol(val, &end, 10);
+ if (!end || *end || v <= 0 || v > INT_MAX) {
+ fprintf(stderr, "%s min_dim must be a positive integer\n", flag);
+ return -1;
+ }
+ spec->min_dim = (int)v;
+ return 0;
+ }
+ fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt);
+ return -1;
+}
+
+static int append_model_spec(model_specs_t *specs,
+ const char *arg,
+ embed_server_model_kind_t kind,
+ const char *flag) {
const char *eq = strchr(arg, '=');
if (!eq || eq == arg || !eq[1]) {
- fprintf(stderr, "--model expects MODEL_ID=DIR\n");
+ fprintf(stderr, "%s expects LABEL=DIR\n", flag);
return -1;
}
if (specs->n == specs->cap) {
@@ -673,18 +753,52 @@ static int append_model_spec(model_specs_t *specs, const char *arg) {
specs->cap = new_cap;
}
size_t id_len = (size_t)(eq - arg);
- char *id = (char *)malloc(id_len + 1);
- char *path = strdup(eq + 1);
- if (id) {
- memcpy(id, arg, id_len);
- id[id_len] = '\0';
+ const char *path_start = eq + 1;
+ const char *opts = strchr(path_start, ':');
+ size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start);
+ if (path_len == 0) {
+ fprintf(stderr, "%s expects a non-empty DIR\n", flag);
+ return -1;
}
- specs->v[specs->n].id = id;
- specs->v[specs->n].path = path;
- if (!specs->v[specs->n].id || !specs->v[specs->n].path)
+ embed_server_model_spec_t spec;
+ memset(&spec, 0, sizeof(spec));
+ spec.id = dup_range(arg, id_len);
+ spec.path = dup_range(path_start, path_len);
+ spec.kind = kind;
+ if (!spec.id || !spec.path) {
+ free((char *)spec.id);
+ free((char *)spec.path);
return -1;
+ }
+ const char *p = opts ? opts + 1 : NULL;
+ while (p && *p) {
+ if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) {
+ const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15);
+ spec.query_instruct = unescape_model_option(val);
+ if (!spec.query_instruct)
+ goto fail;
+ break;
+ }
+ const char *next = strchr(p, ':');
+ size_t opt_len = next ? (size_t)(next - p) : strlen(p);
+ char *opt = dup_range(p, opt_len);
+ if (!opt)
+ goto fail;
+ int rc = parse_model_option(&spec, opt, flag);
+ free(opt);
+ if (rc != 0)
+ goto fail;
+ p = next ? next + 1 : NULL;
+ }
+ specs->v[specs->n] = spec;
specs->n++;
return 0;
+
+fail:
+ free((char *)spec.id);
+ free((char *)spec.path);
+ free((char *)spec.query_instruct);
+ return -1;
}
static void free_model_specs(model_specs_t *specs) {
@@ -693,6 +807,7 @@ static void free_model_specs(model_specs_t *specs) {
for (int i = 0; i < specs->n; i++) {
free((char *)specs->v[i].id);
free((char *)specs->v[i].path);
+ free((char *)specs->v[i].query_instruct);
}
free(specs->v);
}
@@ -732,7 +847,22 @@ int main(int argc, char *argv[]) {
free_model_specs(&model_specs);
return 0;
} else if (!strcmp(f, "--model")) {
- if (append_model_spec(&model_specs, argv[++arg]) != 0) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) !=
+ 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--contextual-model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) !=
+ 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--late-model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) {
free_model_specs(&model_specs);
return 1;
}
@@ -822,7 +952,7 @@ int main(int argc, char *argv[]) {
return 1;
}
if (model_specs.n == 0) {
- fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n");
+ fprintf(stderr, "at least one --model LABEL=DIR is required\n");
print_usage(prog);
free_model_specs(&model_specs);
return 1;
diff --git a/src/server.h b/src/server.h
index 2a39b94..5ad2b12 100644
--- a/src/server.h
+++ b/src/server.h
@@ -14,9 +14,25 @@
#include <stdbool.h>
+typedef enum {
+ EMBED_SERVER_MODEL_STANDARD = 0,
+ EMBED_SERVER_MODEL_CONTEXTUAL = 1,
+ EMBED_SERVER_MODEL_LATE = 2,
+} embed_server_model_kind_t;
+
+typedef enum {
+ EMBED_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
+ EMBED_SERVER_API_OPENAI = 1,
+ EMBED_SERVER_API_PERPLEXITY = 2,
+} embed_server_embedding_api_t;
+
typedef struct {
- const char *id;
+ const char *id; /* served label; requests match this string exactly */
const char *path;
+ embed_server_model_kind_t kind;
+ embed_server_embedding_api_t api;
+ int min_dim; /* 0 means the loaded model's full hidden size */
+ const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
} embed_server_model_spec_t;
typedef struct {
diff --git a/src/server_encode.c b/src/server_encode.c
index 7e86a74..ef4e3ec 100644
--- a/src/server_encode.c
+++ b/src/server_encode.c
@@ -11,11 +11,10 @@
/* ========================================================================
* Embedding output encoding
*
- * Turns a pooled float vector into the wire form the model's API family wants:
- * base64 of int8 (tanh-quantized, Perplexity default), base64 of a binary sign
- * bitmap, base64 of little-endian float32, or a plain JSON float array. The
- * render_* functions assemble the full OpenAI-style response object from a
- * prepared request and its computed embeddings.
+ * Turns a pooled float vector into the requested wire form. OpenAI-compatible
+ * models emit true float32 arrays by default. Perplexity-compatible models
+ * emit tanh-quantized int8 by default, and their "float" view is the decoded
+ * int8 value for SDK compatibility.
* ======================================================================== */
signed char quantize_int8_tanh(float x) {
@@ -66,10 +65,6 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) {
sbuf_puts(b, "\"}");
}
-/* Emit one embedding into the response, following the model's API family.
- * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope
- * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every
- * other encoding renders a base64 string via encode_embedding(). */
void append_embedding_value(
sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) {
if (!strcmp(encoding, "float")) {
@@ -77,7 +72,8 @@ void append_embedding_value(
for (int i = 0; i < dims; i++) {
if (i)
sbuf_putc(b, ',');
- float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f;
+ float v =
+ api == EMBED_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i];
sbuf_printf(b, "%.9g", (double)v);
}
sbuf_puts(b, "]}");
diff --git a/src/server_handlers.c b/src/server_handlers.c
index 95df1de..b3476d9 100644
--- a/src/server_handlers.c
+++ b/src/server_handlers.c
@@ -271,37 +271,31 @@ static int validate_common(cJSON *root,
http_server *s) {
cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
const char *model_id = NULL;
+ loaded_model *m = NULL;
if (!model_item) {
ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
} else if (!cJSON_IsString(model_item) || !model_item->valuestring) {
ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
} else {
model_id = model_item->valuestring;
- model_slot slot = model_slot_for_id(model_id);
- if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) {
+ m = loaded_model_for_label(s, model_id);
+ if (!m) {
ve_add(detail, "[\"body\",\"model\"]",
"value is not a valid enum member for this endpoint", "enum");
- } else {
- loaded_model *m = &s->models[slot];
- if (!m->info) {
- *out_model = NULL;
- } else {
- *out_model = m;
- }
+ } else if (m->info->kind != expected_kind) {
+ ve_add(detail, "[\"body\",\"model\"]",
+ "model is not available on this endpoint", "enum");
+ m = NULL;
}
}
- model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN;
- embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY;
- const char *enc = encoding_from_root(root, detail, api);
- int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128;
- int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560;
+ *out_model = m;
+ const char *enc = encoding_from_root(root, detail, m ? m->info->api : EMBED_API_OPENAI);
+ int min_dim = m ? m->info->min_dim : 128;
+ int max_dim = m ? m->info->dim : 2560;
int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
*out_encoding = enc;
*out_dims = dims;
- return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN &&
- !s->models[model_slot_for_id(model_id)].info
- ? 503
- : 0;
+ return 0;
}
void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
@@ -313,22 +307,15 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
+ validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
- /* Resolve request behavior from the model registry independent of load
- * state. The query instruction, when requested, applies to every input. */
- cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
- model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring)
- ? model_slot_for_id(model_item->valuestring)
- : MODEL_UNKNOWN;
- const char *query_instruct =
- api_slot != MODEL_UNKNOWN &&
- text_type_is_query(root, detail, k_models[api_slot].query_instruct)
- ? k_models[api_slot].query_instruct
- : NULL;
+ /* The query instruction, when requested, applies to every input. */
+ const char *query_instruct = m && text_type_is_query(root, detail, m->info->query_instruct)
+ ? m->info->query_instruct
+ : NULL;
cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
int n_inputs = 0;
@@ -363,12 +350,6 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
"type_error");
}
- if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
- cJSON_Delete(detail);
- return;
- }
-
if (cJSON_GetArraySize(detail) == 0 && m && input) {
out->n_inputs = n_inputs;
out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens));
@@ -436,7 +417,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
+ validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
@@ -497,12 +478,6 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
"value_error.list.max_items");
}
- if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
- cJSON_Delete(detail);
- return;
- }
-
if (cJSON_GetArraySize(detail) == 0 && m) {
out->n_docs = n_docs;
out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
@@ -594,14 +569,18 @@ validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model *
ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
return 0;
}
- model_slot slot = model_slot_for_id(item->valuestring);
- if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) {
+ loaded_model *m = loaded_model_for_label(s, item->valuestring);
+ if (!m) {
ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
"enum");
return 0;
}
- *out_model = s->models[slot].info ? &s->models[slot] : NULL;
- return *out_model ? 0 : 503;
+ if (m->info->kind != MODEL_KIND_LATE) {
+ ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum");
+ return 0;
+ }
+ *out_model = m;
+ return 0;
}
void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
@@ -610,7 +589,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
out->root = root;
cJSON *detail = cJSON_CreateArray();
- int unloaded = validate_rerank_model(root, detail, s, &out->model);
+ validate_rerank_model(root, detail, s, &out->model);
cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query");
cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents");
/* query_str is set only once query passes every check; the tokenize step
@@ -682,12 +661,6 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
out->return_documents = cJSON_IsTrue(return_documents);
}
- if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
- cJSON_Delete(detail);
- return;
- }
-
if (cJSON_GetArraySize(detail) == 0 && out->model) {
out->n_documents = n_documents;
out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
diff --git a/src/server_internal.h b/src/server_internal.h
index 56bf25b..1f106ca 100644
--- a/src/server_internal.h
+++ b/src/server_internal.h
@@ -8,6 +8,7 @@
* the CLI and library do not include it. */
#include "embed.h"
+#include "server.h"
#include "sbuf.h"
#include "tokenizer_bpe.h"
#include "tokenizer_sentencepiece.h"
@@ -47,18 +48,6 @@
#define EMBED_LATE_MASK_TOKEN_ID 151642
#define EMBED_LATE_QUERY_PREFIX_ID 151669
#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
-/* `text_type: query` prepends the model's published retrieval instruction.
- * Documents and the default pass through unchanged. Callers needing a custom
- * instruction prepend it themselves and leave text_type at "document". */
-#define EMBED_QWEN3_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
-#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the " \
- "query\nQuery: "
-#define EMBED_E5_QUERY_INSTRUCT \
- "Instruct: Given a web search query, retrieve relevant passages that answer the " \
- "query\nQuery: "
-#define EMBED_SNOWFLAKE_QUERY_INSTRUCT "query: "
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
@@ -74,42 +63,15 @@
#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
#define EMBED_MLX_RESIDENT_MULTIPLIER 2
-typedef enum {
- MODEL_STD_06,
- MODEL_STD_4,
- MODEL_QWEN3_06,
- MODEL_QWEN3_4,
- MODEL_QWEN3_8,
- MODEL_GTE_QWEN2_15,
- MODEL_MINILM_L6,
- MODEL_BGE_SMALL,
- MODEL_BGE_BASE,
- MODEL_BGE_LARGE,
- MODEL_E5_MULTI_LARGE_INSTRUCT,
- MODEL_SNOWFLAKE_ARCTIC_L_V2,
- MODEL_CTX_06,
- MODEL_CTX_4,
- MODEL_LATE_06,
- MODEL_COUNT,
- MODEL_UNKNOWN
-} model_slot;
-
typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
-/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
- * API: the canonical output is the tanh int8 quantization, so the default
- * encoding is base64_int8 and "float" is the int8-decoded view (int8/128).
- * Instruction embedding models follow the OpenAI-compatible API: "float" is
- * the true float32 vector and "base64" is base64 of little-endian float32.
- * int8 quantization underuses the range of L2-normalized vectors, so it is not
- * used for these models. */
typedef enum {
- EMBED_API_PERPLEXITY = 0,
- EMBED_API_OPENAI = 1,
+ EMBED_API_OPENAI = 0,
+ EMBED_API_PERPLEXITY = 1,
} embedding_api_t;
typedef struct {
- const char *id;
+ char *id;
model_kind kind;
int dim;
int min_dim;
@@ -118,11 +80,11 @@ typedef struct {
embed_pooling_mode_t pooling_mode;
int normalize_embeddings;
embedding_api_t api;
- const char *query_instruct;
+ char *query_instruct;
} model_info;
typedef struct {
- const model_info *info;
+ model_info *info;
char *path;
embed_tokenizer_t *tok;
embed_tokenizer_workspace_t *tok_ws;
@@ -172,7 +134,8 @@ typedef struct {
int max_batch_tokens;
int batch_wait_us;
char *api_key;
- loaded_model models[MODEL_COUNT];
+ loaded_model *models;
+ int n_models;
int use_mlx;
int use_cuda;
int mlx_quantize_bits;
@@ -347,9 +310,8 @@ void set_response_from_buf(job *j, sbuf *b);
int render_embedding_response(embedding_request *r, const float *embs);
void render_contextual_response(contextual_request *r, const float *embs);
-/* ---- server_models.c: registry, load/lifecycle, tokenize/inference dispatch ---- */
-extern const model_info k_models[];
-model_slot model_slot_for_id(const char *id);
+/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */
+loaded_model *loaded_model_for_label(http_server *s, const char *label);
void free_token_bufs(token_buf *t, int n);
void embedding_request_free(embedding_request *r);
void contextual_request_free(contextual_request *r);
@@ -367,7 +329,7 @@ int model_embed_spans_batch(loaded_model *m,
int inference_batch_accepts_input(
const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim);
-int load_one_model(http_server *s, model_slot slot, const char *path);
+int load_one_model(http_server *s, const embed_server_model_spec_t *spec);
void free_models(http_server *s);
/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
diff --git a/src/server_json.c b/src/server_json.c
index e45c2c7..cca8fd3 100644
--- a/src/server_json.c
+++ b/src/server_json.c
@@ -143,10 +143,7 @@ bool cjson_is_integer(cJSON *item) {
}
const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
- /* Default and accepted encodings follow the model's API family: Perplexity
- * defaults to base64_int8 and accepts {base64_int8, base64_binary, float};
- * OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */
- const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8";
+ const char *dflt = api == EMBED_API_PERPLEXITY ? "base64_int8" : "float";
cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
if (!encoding)
return dflt;
@@ -156,15 +153,16 @@ const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api)
return dflt;
}
const char *v = encoding->valuestring;
- int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64"))
- : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
- !strcmp(v, "float"));
+ int ok = api == EMBED_API_PERPLEXITY
+ ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
+ !strcmp(v, "float"))
+ : (!strcmp(v, "float") || !strcmp(v, "base64"));
if (!ok) {
ve_add(detail, "[\"body\",\"encoding_format\"]",
- api == EMBED_API_OPENAI
- ? "value is not a valid enum member; permitted: 'float', 'base64'"
- : "value is not a valid enum member; permitted: "
- "'base64_int8', 'base64_binary', 'float'",
+ api == EMBED_API_PERPLEXITY
+ ? "value is not a valid enum member; permitted: "
+ "'base64_int8', 'base64_binary', 'float'"
+ : "value is not a valid enum member; permitted: 'float', 'base64'",
"enum");
return dflt;
}
diff --git a/src/server_models.c b/src/server_models.c
index 32199d9..320840b 100644
--- a/src/server_models.c
+++ b/src/server_models.c
@@ -1,6 +1,7 @@
#include "server_internal.h"
#include "server_util.h"
+#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
@@ -8,63 +9,137 @@
#include <unistd.h>
/* ========================================================================
- * Model registry, load/lifecycle, and tokenize/inference dispatch
+ * Model load/lifecycle, and tokenize/inference dispatch
*
- * k_models is the static table of every model the server can serve (name,
- * dimensions, attention/pooling mode, API family). load_one_model resolves a
- * served id to a slot, picks the tokenizer by the files present (WordPiece
- * vocab.txt, SentencePiece .model, else byte-level BPE vocab.json), and loads the CPU model
- * (GPU backends load on the worker thread). The tokenize_* and model_embed_*
+ * Each loaded model owns resolved serving metadata for the operator-chosen
+ * label. Transformer behavior comes from the model files through embed_config;
+ * server-only facts absent from those files, such as output API family and
+ * Matryoshka floor, come from the load spec. The tokenize_* and model_embed_*
* helpers are the single dispatch point the request handlers call, hiding the
* CPU/MLX/CUDA backend choice behind one signature.
* ======================================================================== */
-const model_info k_models[] = {
- {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
- {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT},
- /* BERT encoders: WordPiece tokenizer, learned positions, no instruction
- * prefix. all-MiniLM-L6-v2 mean-pools; bge-small-en-v1.5 pools the [CLS]
- * token. Both emit L2-normalized float32, so they speak the OpenAI API. */
- {"all-MiniLM-L6-v2", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, NULL},
- {"bge-small-en-v1.5", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"bge-base-en-v1.5", MODEL_KIND_STANDARD, 768, 768, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
- {"multilingual-e5-large-instruct", MODEL_KIND_STANDARD, 1024, 1024, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 1, EMBED_API_OPENAI,
- EMBED_E5_QUERY_INSTRUCT},
- {"snowflake-arctic-embed-l-v2.0", MODEL_KIND_STANDARD, 1024, 256, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI,
- EMBED_SNOWFLAKE_QUERY_INSTRUCT},
- {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
- {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
-};
+loaded_model *loaded_model_for_label(http_server *s, const char *label) {
+ if (!label)
+ return NULL;
+ for (int i = 0; i < s->n_models; i++) {
+ if (s->models[i].info && !strcmp(label, s->models[i].info->id))
+ return &s->models[i];
+ }
+ return NULL;
+}
+
+static model_kind model_kind_from_public(embed_server_model_kind_t kind) {
+ switch (kind) {
+ case EMBED_SERVER_MODEL_CONTEXTUAL:
+ return MODEL_KIND_CONTEXTUAL;
+ case EMBED_SERVER_MODEL_LATE:
+ return MODEL_KIND_LATE;
+ case EMBED_SERVER_MODEL_STANDARD:
+ default:
+ return MODEL_KIND_STANDARD;
+ }
+}
+
+static embedding_api_t embedding_api_from_public(embed_server_embedding_api_t api) {
+ return api == EMBED_SERVER_API_PERPLEXITY ? EMBED_API_PERPLEXITY : EMBED_API_OPENAI;
+}
+
+static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ server_log("embed-server: model path too long");
+ return -1;
+ }
+ FILE *f = fopen(path, "rb");
+ if (!f && errno == ENOENT)
+ return 0;
+ if (!f) {
+ server_log("embed-server: failed to open %s", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ free(buf);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+ *out = buf;
+ return 1;
+}
+
+static int read_query_instruct_from_files(const char *model_dir, char **out) {
+ char *buf = NULL;
+ int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
+ if (rc <= 0)
+ return rc;
+ cJSON *root = cJSON_Parse(buf);
+ free(buf);
+ if (!root) {
+ server_log("embed-server: invalid config_sentence_transformers.json: %s", model_dir);
+ return -1;
+ }
+ cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
+ cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
+ if (query && !cJSON_IsString(query)) {
+ cJSON_Delete(root);
+ server_log("embed-server: prompts.query must be a string: %s", model_dir);
+ return -1;
+ }
+ if (query && query->valuestring && query->valuestring[0]) {
+ *out = xstrdup(query->valuestring);
+ cJSON_Delete(root);
+ return *out ? 1 : -1;
+ }
+ cJSON_Delete(root);
+ return 0;
+}
-model_slot model_slot_for_id(const char *id) {
- if (!id)
- return MODEL_UNKNOWN;
- for (int i = 0; i < MODEL_COUNT; i++) {
- if (!strcmp(id, k_models[i].id))
- return (model_slot)i;
+static int init_model_info(model_info **out, const embed_server_model_spec_t *spec) {
+ model_info *info = xcalloc(1, sizeof(*info));
+ info->id = xstrdup(spec->id);
+ info->kind = model_kind_from_public(spec->kind);
+ info->api = embedding_api_from_public(spec->api);
+ info->min_dim = spec->min_dim;
+ if (spec->query_instruct) {
+ info->query_instruct = xstrdup(spec->query_instruct);
+ } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ if (!info->id || (spec->query_instruct && !info->query_instruct)) {
+ free(info->query_instruct);
+ free(info->id);
+ free(info);
+ return -1;
}
- return MODEL_UNKNOWN;
+ *out = info;
+ return 0;
+}
+
+static void free_model_info(model_info *info) {
+ if (!info)
+ return;
+ free(info->id);
+ free(info->query_instruct);
+ free(info);
}
void free_token_bufs(token_buf *t, int n) {
@@ -307,14 +382,35 @@ int inference_batch_accepts_input(
}
int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
- if (!m || !m->info || !config || config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode ||
- config->pooling_mode != m->info->pooling_mode ||
- config->normalize_embeddings != m->info->normalize_embeddings) {
- if (m && m->info && config)
- server_log("embed-server: model %s has incompatible config", m->info->id);
+ if (!m || !m->info || !config || config->hidden_size <= 0) {
+ if (m && m->info)
+ server_log("embed-server: model %s has invalid config", m->info->id);
+ return -1;
+ }
+ if (m->info->kind == MODEL_KIND_LATE) {
+ if (token_dim <= 0) {
+ server_log("embed-server: late model %s is missing token projection metadata",
+ m->info->id);
+ return -1;
+ }
+ } else if (token_dim != 0) {
+ server_log("embed-server: non-late model %s unexpectedly has token projection metadata",
+ m->info->id);
return -1;
}
+
+ int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
+ if (min_dim <= 0 || min_dim > config->hidden_size) {
+ server_log("embed-server: model %s has invalid min_dim=%d for dim=%d", m->info->id,
+ min_dim, config->hidden_size);
+ return -1;
+ }
+ m->info->dim = config->hidden_size;
+ m->info->min_dim = min_dim;
+ m->info->token_dim = token_dim;
+ m->info->attention_mode = config->attention_mode;
+ m->info->pooling_mode = config->pooling_mode;
+ m->info->normalize_embeddings = config->normalize_embeddings;
m->append_terminal_token = config->append_terminal_token;
/* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
m->renormalize_truncated =
@@ -322,9 +418,19 @@ int configure_loaded_model(loaded_model *m, const embed_config_t *config, int to
return 0;
}
-int load_one_model(http_server *s, model_slot slot, const char *path) {
- loaded_model *m = &s->models[slot];
- m->info = &k_models[slot];
+int load_one_model(http_server *s, const embed_server_model_spec_t *spec) {
+ /* Grow the model list by one entry. */
+ loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
+ if (!nm)
+ return -1;
+ s->models = nm;
+ loaded_model *m = &s->models[s->n_models];
+ memset(m, 0, sizeof(*m));
+ s->n_models++;
+
+ if (init_model_info(&m->info, spec) != 0)
+ return -1;
+ const char *path = spec->path;
m->path = xstrdup(path);
m->terminal_token_id = -1;
m->cls_id = -1;
@@ -478,8 +584,9 @@ int load_one_model(http_server *s, model_slot slot, const char *path) {
}
void free_models(http_server *s) {
- for (int i = 0; i < MODEL_COUNT; i++) {
+ for (int i = 0; i < s->n_models; i++) {
loaded_model *m = &s->models[i];
+ free_model_info(m->info);
free(m->path);
if (m->tok_ws)
embed_tokenizer_workspace_free(m->tok_ws);
@@ -514,4 +621,7 @@ void free_models(http_server *s) {
embed_cuda_late_free(m->cuda_late_ctx);
#endif
}
+ free(s->models);
+ s->models = NULL;
+ s->n_models = 0;
}