diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/mlx.c | 3 | ||||
| -rw-r--r-- | src/server.c | 218 | ||||
| -rw-r--r-- | src/server.h | 18 | ||||
| -rw-r--r-- | src/server_encode.c | 16 | ||||
| -rw-r--r-- | src/server_handlers.c | 81 | ||||
| -rw-r--r-- | src/server_internal.h | 60 | ||||
| -rw-r--r-- | src/server_json.c | 20 | ||||
| -rw-r--r-- | src/server_models.c | 230 |
8 files changed, 416 insertions, 230 deletions
@@ -520,6 +520,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, } ctx->layer_start = layer_start; ctx->layer_end = layer_end; + int h = c->hidden_size; ctx->ms = multi_safetensors_open(model_dir); if (!ctx->ms) { @@ -541,7 +542,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, goto fail; } - int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; + int qd = c->q_dim, kvd = c->kv_dim; int inter = c->intermediate_size, hd = c->head_dim; char name[256]; diff --git a/src/server.c b/src/server.c index cb5a857..808c6f8 100644 --- a/src/server.c +++ b/src/server.c @@ -79,7 +79,6 @@ static bool auth_ok(http_server *s, const char *auth) { !strcmp(auth + sizeof(prefix) - 1, s->api_key); } - void dispatch_request(client *c) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); @@ -190,10 +189,8 @@ static void *worker_main(void *arg) { #ifdef USE_MLX if (s->use_mlx) { int rc = 0; - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; - if (!m->info) - continue; embed_mlx_options_t mlx_opts = { .quantize_bits = s->mlx_quantize_bits, .quantize_group_size = s->mlx_quantize_group_size, @@ -230,7 +227,7 @@ static void *worker_main(void *arg) { pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; if (m->mlx_ctx) { embed_mlx_free(m->mlx_ctx); @@ -248,10 +245,8 @@ static void *worker_main(void *arg) { #ifdef USE_CUDA if (s->use_cuda) { int rc = 0; - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; - if (!m->info) - continue; if (m->info->kind == MODEL_KIND_LATE) m->cuda_late_ctx = embed_cuda_late_load(m->path); else @@ -284,7 +279,7 @@ static void *worker_main(void *arg) { pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; if (m->cuda_ctx) { embed_cuda_free(m->cuda_ctx); @@ -315,7 +310,7 @@ static void *worker_main(void *arg) { } #ifdef USE_MLX if (s->use_mlx) { - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; if (m->mlx_ctx) { embed_mlx_free(m->mlx_ctx); @@ -330,7 +325,7 @@ static void *worker_main(void *arg) { #endif #ifdef USE_CUDA if (s->use_cuda) { - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; if (m->cuda_ctx) { embed_cuda_free(m->cuda_ctx); @@ -375,23 +370,34 @@ static uint64_t physical_memory_nbytes(void) { } static int validate_model_specs(const embed_server_config_t *cfg) { - int seen[MODEL_COUNT] = {0}; for (int i = 0; i < cfg->n_models; i++) { const embed_server_model_spec_t *spec = &cfg->models[i]; - model_slot slot = model_slot_for_id(spec->id); - if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : "<null>"); + if (!spec->id || !spec->id[0]) { + fprintf(stderr, "embed-server: model label must not be empty\n"); return -1; } if (!spec->path || !spec->path[0]) { fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); return -1; } - if (seen[slot]) { - fprintf(stderr, "embed-server: duplicate model id: %s\n", spec->id); + if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) { + fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id); + return -1; + } + if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) { + fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id); + return -1; + } + if (spec->min_dim < 0) { + fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id); return -1; } - seen[slot] = 1; + for (int j = 0; j < i; j++) { + if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) { + fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id); + return -1; + } + } } return 0; } @@ -452,7 +458,7 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) { int embed_run_server(const embed_server_config_t *cfg) { if (!cfg || !cfg->models || cfg->n_models <= 0) { - fprintf(stderr, "embed-server: at least one --model MODEL_ID=PATH is required\n"); + fprintf(stderr, "embed-server: at least one model label and path is required\n"); return 1; } if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) @@ -485,13 +491,7 @@ int embed_run_server(const embed_server_config_t *cfg) { pthread_cond_init(&s.cv, NULL); for (int i = 0; i < cfg->n_models; i++) { - model_slot slot = model_slot_for_id(cfg->models[i].id); - if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "embed-server: unknown model id: %s\n", cfg->models[i].id); - free_models(&s); - return 1; - } - if (load_one_model(&s, slot, cfg->models[i].path) != 0) { + if (load_one_model(&s, &cfg->models[i]) != 0) { free_models(&s); return 1; } @@ -599,10 +599,17 @@ int embed_run_server(const embed_server_config_t *cfg) { static void print_usage(const char *prog) { fprintf(stderr, - "Usage: %s --model ID=DIR [options]\n" + "Usage: %s --model LABEL=DIR [options]\n" "\n" "Options:\n" - " --model ID=DIR Model to serve (repeatable)\n" + " --model LABEL=DIR[:key=val]\n" + " Standard embedding model (repeatable)\n" + " --contextual-model LABEL=DIR[:key=val]\n" + " Contextual embedding model (repeatable)\n" + " --late-model LABEL=DIR[:key=val]\n" + " Late-interaction rerank model (repeatable)\n" + " keys: api=openai|perplexity, min_dim=N,\n" + " query=TEXT (last key; \\n escapes newline)\n" #ifdef USE_MLX " --mlx Use Apple MLX GPU backend\n" #endif @@ -641,14 +648,15 @@ static void print_usage(const char *prog) { " -h, --help Show this help\n" "\n" "Examples:\n" - " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" + " %s --model docs=./model --port 8000\n" + " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n" #ifdef USE_MLX " %s --mlx --mlx-quant-bits 8 \\\n" - " --model pplx-embed-v1-4b=./model-4b-bf16\n", - prog, prog, prog); + " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n", + prog, prog, prog, prog); #else , - prog, prog); + prog, prog, prog); #endif } @@ -658,10 +666,82 @@ typedef struct { int cap; } model_specs_t; -static int append_model_spec(model_specs_t *specs, const char *arg) { +static char *dup_range(const char *p, size_t n) { + char *s = (char *)malloc(n + 1); + if (!s) + return NULL; + memcpy(s, p, n); + s[n] = '\0'; + return s; +} + +static char *unescape_model_option(const char *s) { + size_t n = strlen(s); + char *out = (char *)malloc(n + 1); + if (!out) + return NULL; + size_t w = 0; + for (size_t r = 0; r < n; r++) { + if (s[r] == '\\' && r + 1 < n) { + r++; + if (s[r] == 'n') + out[w++] = '\n'; + else if (s[r] == 't') + out[w++] = '\t'; + else if (s[r] == 'r') + out[w++] = '\r'; + else + out[w++] = s[r]; + } else { + out[w++] = s[r]; + } + } + out[w] = '\0'; + return out; +} + +static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) { + const char *eq = strchr(opt, '='); + if (!eq || eq == opt || !eq[1]) { + fprintf(stderr, "%s option expects key=value: %s\n", flag, opt); + return -1; + } + size_t key_len = (size_t)(eq - opt); + const char *val = eq + 1; + if (key_len == 3 && strncmp(opt, "api", key_len) == 0) { + if (!strcmp(val, "openai")) { + spec->api = EMBED_SERVER_API_OPENAI; + } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) { + spec->api = EMBED_SERVER_API_PERPLEXITY; + } else if (!strcmp(val, "default")) { + spec->api = EMBED_SERVER_API_DEFAULT; + } else { + fprintf(stderr, "%s api must be openai or perplexity\n", flag); + return -1; + } + return 0; + } + if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) { + char *end = NULL; + long v = strtol(val, &end, 10); + if (!end || *end || v <= 0 || v > INT_MAX) { + fprintf(stderr, "%s min_dim must be a positive integer\n", flag); + return -1; + } + spec->min_dim = (int)v; + return 0; + } + fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt); + return -1; +} + +static int append_model_spec(model_specs_t *specs, + const char *arg, + embed_server_model_kind_t kind, + const char *flag) { const char *eq = strchr(arg, '='); if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "--model expects MODEL_ID=DIR\n"); + fprintf(stderr, "%s expects LABEL=DIR\n", flag); return -1; } if (specs->n == specs->cap) { @@ -673,18 +753,52 @@ static int append_model_spec(model_specs_t *specs, const char *arg) { specs->cap = new_cap; } size_t id_len = (size_t)(eq - arg); - char *id = (char *)malloc(id_len + 1); - char *path = strdup(eq + 1); - if (id) { - memcpy(id, arg, id_len); - id[id_len] = '\0'; + const char *path_start = eq + 1; + const char *opts = strchr(path_start, ':'); + size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start); + if (path_len == 0) { + fprintf(stderr, "%s expects a non-empty DIR\n", flag); + return -1; } - specs->v[specs->n].id = id; - specs->v[specs->n].path = path; - if (!specs->v[specs->n].id || !specs->v[specs->n].path) + embed_server_model_spec_t spec; + memset(&spec, 0, sizeof(spec)); + spec.id = dup_range(arg, id_len); + spec.path = dup_range(path_start, path_len); + spec.kind = kind; + if (!spec.id || !spec.path) { + free((char *)spec.id); + free((char *)spec.path); return -1; + } + const char *p = opts ? opts + 1 : NULL; + while (p && *p) { + if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) { + const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15); + spec.query_instruct = unescape_model_option(val); + if (!spec.query_instruct) + goto fail; + break; + } + const char *next = strchr(p, ':'); + size_t opt_len = next ? (size_t)(next - p) : strlen(p); + char *opt = dup_range(p, opt_len); + if (!opt) + goto fail; + int rc = parse_model_option(&spec, opt, flag); + free(opt); + if (rc != 0) + goto fail; + p = next ? next + 1 : NULL; + } + specs->v[specs->n] = spec; specs->n++; return 0; + +fail: + free((char *)spec.id); + free((char *)spec.path); + free((char *)spec.query_instruct); + return -1; } static void free_model_specs(model_specs_t *specs) { @@ -693,6 +807,7 @@ static void free_model_specs(model_specs_t *specs) { for (int i = 0; i < specs->n; i++) { free((char *)specs->v[i].id); free((char *)specs->v[i].path); + free((char *)specs->v[i].query_instruct); } free(specs->v); } @@ -732,7 +847,22 @@ int main(int argc, char *argv[]) { free_model_specs(&model_specs); return 0; } else if (!strcmp(f, "--model")) { - if (append_model_spec(&model_specs, argv[++arg]) != 0) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) != + 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--contextual-model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) != + 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--late-model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) { free_model_specs(&model_specs); return 1; } @@ -822,7 +952,7 @@ int main(int argc, char *argv[]) { return 1; } if (model_specs.n == 0) { - fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); + fprintf(stderr, "at least one --model LABEL=DIR is required\n"); print_usage(prog); free_model_specs(&model_specs); return 1; diff --git a/src/server.h b/src/server.h index 2a39b94..5ad2b12 100644 --- a/src/server.h +++ b/src/server.h @@ -14,9 +14,25 @@ #include <stdbool.h> +typedef enum { + EMBED_SERVER_MODEL_STANDARD = 0, + EMBED_SERVER_MODEL_CONTEXTUAL = 1, + EMBED_SERVER_MODEL_LATE = 2, +} embed_server_model_kind_t; + +typedef enum { + EMBED_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */ + EMBED_SERVER_API_OPENAI = 1, + EMBED_SERVER_API_PERPLEXITY = 2, +} embed_server_embedding_api_t; + typedef struct { - const char *id; + const char *id; /* served label; requests match this string exactly */ const char *path; + embed_server_model_kind_t kind; + embed_server_embedding_api_t api; + int min_dim; /* 0 means the loaded model's full hidden size */ + const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */ } embed_server_model_spec_t; typedef struct { diff --git a/src/server_encode.c b/src/server_encode.c index 7e86a74..ef4e3ec 100644 --- a/src/server_encode.c +++ b/src/server_encode.c @@ -11,11 +11,10 @@ /* ======================================================================== * Embedding output encoding * - * Turns a pooled float vector into the wire form the model's API family wants: - * base64 of int8 (tanh-quantized, Perplexity default), base64 of a binary sign - * bitmap, base64 of little-endian float32, or a plain JSON float array. The - * render_* functions assemble the full OpenAI-style response object from a - * prepared request and its computed embeddings. + * Turns a pooled float vector into the requested wire form. OpenAI-compatible + * models emit true float32 arrays by default. Perplexity-compatible models + * emit tanh-quantized int8 by default, and their "float" view is the decoded + * int8 value for SDK compatibility. * ======================================================================== */ signed char quantize_int8_tanh(float x) { @@ -66,10 +65,6 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) { sbuf_puts(b, "\"}"); } -/* Emit one embedding into the response, following the model's API family. - * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope - * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every - * other encoding renders a base64 string via encode_embedding(). */ void append_embedding_value( sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { if (!strcmp(encoding, "float")) { @@ -77,7 +72,8 @@ void append_embedding_value( for (int i = 0; i < dims; i++) { if (i) sbuf_putc(b, ','); - float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f; + float v = + api == EMBED_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i]; sbuf_printf(b, "%.9g", (double)v); } sbuf_puts(b, "]}"); diff --git a/src/server_handlers.c b/src/server_handlers.c index 95df1de..b3476d9 100644 --- a/src/server_handlers.c +++ b/src/server_handlers.c @@ -271,37 +271,31 @@ static int validate_common(cJSON *root, http_server *s) { cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); const char *model_id = NULL; + loaded_model *m = NULL; if (!model_item) { ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); } else { model_id = model_item->valuestring; - model_slot slot = model_slot_for_id(model_id); - if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) { + m = loaded_model_for_label(s, model_id); + if (!m) { ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", "enum"); - } else { - loaded_model *m = &s->models[slot]; - if (!m->info) { - *out_model = NULL; - } else { - *out_model = m; - } + } else if (m->info->kind != expected_kind) { + ve_add(detail, "[\"body\",\"model\"]", + "model is not available on this endpoint", "enum"); + m = NULL; } } - model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; - embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; - const char *enc = encoding_from_root(root, detail, api); - int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; - int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; + *out_model = m; + const char *enc = encoding_from_root(root, detail, m ? m->info->api : EMBED_API_OPENAI); + int min_dim = m ? m->info->min_dim : 128; + int max_dim = m ? m->info->dim : 2560; int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); *out_encoding = enc; *out_dims = dims; - return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && - !s->models[model_slot_for_id(model_id)].info - ? 503 - : 0; + return 0; } void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { @@ -313,22 +307,15 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); + validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; - /* Resolve request behavior from the model registry independent of load - * state. The query instruction, when requested, applies to every input. */ - cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); - model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring) - ? model_slot_for_id(model_item->valuestring) - : MODEL_UNKNOWN; - const char *query_instruct = - api_slot != MODEL_UNKNOWN && - text_type_is_query(root, detail, k_models[api_slot].query_instruct) - ? k_models[api_slot].query_instruct - : NULL; + /* The query instruction, when requested, applies to every input. */ + const char *query_instruct = m && text_type_is_query(root, detail, m->info->query_instruct) + ? m->info->query_instruct + : NULL; cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); int n_inputs = 0; @@ -363,12 +350,6 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re "type_error"); } - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - if (cJSON_GetArraySize(detail) == 0 && m && input) { out->n_inputs = n_inputs; out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); @@ -436,7 +417,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); + validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; @@ -497,12 +478,6 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ "value_error.list.max_items"); } - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - if (cJSON_GetArraySize(detail) == 0 && m) { out->n_docs = n_docs; out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); @@ -594,14 +569,18 @@ validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model * ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); return 0; } - model_slot slot = model_slot_for_id(item->valuestring); - if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { + loaded_model *m = loaded_model_for_label(s, item->valuestring); + if (!m) { ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", "enum"); return 0; } - *out_model = s->models[slot].info ? &s->models[slot] : NULL; - return *out_model ? 0 : 503; + if (m->info->kind != MODEL_KIND_LATE) { + ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum"); + return 0; + } + *out_model = m; + return 0; } void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { @@ -610,7 +589,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request out->root = root; cJSON *detail = cJSON_CreateArray(); - int unloaded = validate_rerank_model(root, detail, s, &out->model); + validate_rerank_model(root, detail, s, &out->model); cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query"); cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents"); /* query_str is set only once query passes every check; the tokenize step @@ -682,12 +661,6 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request out->return_documents = cJSON_IsTrue(return_documents); } - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - if (cJSON_GetArraySize(detail) == 0 && out->model) { out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); diff --git a/src/server_internal.h b/src/server_internal.h index 56bf25b..1f106ca 100644 --- a/src/server_internal.h +++ b/src/server_internal.h @@ -8,6 +8,7 @@ * the CLI and library do not include it. */ #include "embed.h" +#include "server.h" #include "sbuf.h" #include "tokenizer_bpe.h" #include "tokenizer_sentencepiece.h" @@ -47,18 +48,6 @@ #define EMBED_LATE_MASK_TOKEN_ID 151642 #define EMBED_LATE_QUERY_PREFIX_ID 151669 #define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 -/* `text_type: query` prepends the model's published retrieval instruction. - * Documents and the default pass through unchanged. Callers needing a custom - * instruction prepend it themselves and leave text_type at "document". */ -#define EMBED_QWEN3_QUERY_INSTRUCT \ - "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:" -#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ - "Instruct: Given a web search query, retrieve relevant passages that answer the " \ - "query\nQuery: " -#define EMBED_E5_QUERY_INSTRUCT \ - "Instruct: Given a web search query, retrieve relevant passages that answer the " \ - "query\nQuery: " -#define EMBED_SNOWFLAKE_QUERY_INSTRUCT "query: " /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ @@ -74,42 +63,15 @@ #define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 #define EMBED_MLX_RESIDENT_MULTIPLIER 2 -typedef enum { - MODEL_STD_06, - MODEL_STD_4, - MODEL_QWEN3_06, - MODEL_QWEN3_4, - MODEL_QWEN3_8, - MODEL_GTE_QWEN2_15, - MODEL_MINILM_L6, - MODEL_BGE_SMALL, - MODEL_BGE_BASE, - MODEL_BGE_LARGE, - MODEL_E5_MULTI_LARGE_INSTRUCT, - MODEL_SNOWFLAKE_ARCTIC_L_V2, - MODEL_CTX_06, - MODEL_CTX_4, - MODEL_LATE_06, - MODEL_COUNT, - MODEL_UNKNOWN -} model_slot; - typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; -/* Which embedding API a model speaks. pplx-embed models follow the Perplexity - * API: the canonical output is the tanh int8 quantization, so the default - * encoding is base64_int8 and "float" is the int8-decoded view (int8/128). - * Instruction embedding models follow the OpenAI-compatible API: "float" is - * the true float32 vector and "base64" is base64 of little-endian float32. - * int8 quantization underuses the range of L2-normalized vectors, so it is not - * used for these models. */ typedef enum { - EMBED_API_PERPLEXITY = 0, - EMBED_API_OPENAI = 1, + EMBED_API_OPENAI = 0, + EMBED_API_PERPLEXITY = 1, } embedding_api_t; typedef struct { - const char *id; + char *id; model_kind kind; int dim; int min_dim; @@ -118,11 +80,11 @@ typedef struct { embed_pooling_mode_t pooling_mode; int normalize_embeddings; embedding_api_t api; - const char *query_instruct; + char *query_instruct; } model_info; typedef struct { - const model_info *info; + model_info *info; char *path; embed_tokenizer_t *tok; embed_tokenizer_workspace_t *tok_ws; @@ -172,7 +134,8 @@ typedef struct { int max_batch_tokens; int batch_wait_us; char *api_key; - loaded_model models[MODEL_COUNT]; + loaded_model *models; + int n_models; int use_mlx; int use_cuda; int mlx_quantize_bits; @@ -347,9 +310,8 @@ void set_response_from_buf(job *j, sbuf *b); int render_embedding_response(embedding_request *r, const float *embs); void render_contextual_response(contextual_request *r, const float *embs); -/* ---- server_models.c: registry, load/lifecycle, tokenize/inference dispatch ---- */ -extern const model_info k_models[]; -model_slot model_slot_for_id(const char *id); +/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ +loaded_model *loaded_model_for_label(http_server *s, const char *label); void free_token_bufs(token_buf *t, int n); void embedding_request_free(embedding_request *r); void contextual_request_free(contextual_request *r); @@ -367,7 +329,7 @@ int model_embed_spans_batch(loaded_model *m, int inference_batch_accepts_input( const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim); -int load_one_model(http_server *s, model_slot slot, const char *path); +int load_one_model(http_server *s, const embed_server_model_spec_t *spec); void free_models(http_server *s); /* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ diff --git a/src/server_json.c b/src/server_json.c index e45c2c7..cca8fd3 100644 --- a/src/server_json.c +++ b/src/server_json.c @@ -143,10 +143,7 @@ bool cjson_is_integer(cJSON *item) { } const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) { - /* Default and accepted encodings follow the model's API family: Perplexity - * defaults to base64_int8 and accepts {base64_int8, base64_binary, float}; - * OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */ - const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8"; + const char *dflt = api == EMBED_API_PERPLEXITY ? "base64_int8" : "float"; cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); if (!encoding) return dflt; @@ -156,15 +153,16 @@ const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) return dflt; } const char *v = encoding->valuestring; - int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64")) - : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || - !strcmp(v, "float")); + int ok = api == EMBED_API_PERPLEXITY + ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || + !strcmp(v, "float")) + : (!strcmp(v, "float") || !strcmp(v, "base64")); if (!ok) { ve_add(detail, "[\"body\",\"encoding_format\"]", - api == EMBED_API_OPENAI - ? "value is not a valid enum member; permitted: 'float', 'base64'" - : "value is not a valid enum member; permitted: " - "'base64_int8', 'base64_binary', 'float'", + api == EMBED_API_PERPLEXITY + ? "value is not a valid enum member; permitted: " + "'base64_int8', 'base64_binary', 'float'" + : "value is not a valid enum member; permitted: 'float', 'base64'", "enum"); return dflt; } diff --git a/src/server_models.c b/src/server_models.c index 32199d9..320840b 100644 --- a/src/server_models.c +++ b/src/server_models.c @@ -1,6 +1,7 @@ #include "server_internal.h" #include "server_util.h" +#include <errno.h> #include <limits.h> #include <stdio.h> #include <stdlib.h> @@ -8,63 +9,137 @@ #include <unistd.h> /* ======================================================================== - * Model registry, load/lifecycle, and tokenize/inference dispatch + * Model load/lifecycle, and tokenize/inference dispatch * - * k_models is the static table of every model the server can serve (name, - * dimensions, attention/pooling mode, API family). load_one_model resolves a - * served id to a slot, picks the tokenizer by the files present (WordPiece - * vocab.txt, SentencePiece .model, else byte-level BPE vocab.json), and loads the CPU model - * (GPU backends load on the worker thread). The tokenize_* and model_embed_* + * Each loaded model owns resolved serving metadata for the operator-chosen + * label. Transformer behavior comes from the model files through embed_config; + * server-only facts absent from those files, such as output API family and + * Matryoshka floor, come from the load spec. The tokenize_* and model_embed_* * helpers are the single dispatch point the request handlers call, hiding the * CPU/MLX/CUDA backend choice behind one signature. * ======================================================================== */ -const model_info k_models[] = { - {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, - {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, - {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, - {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, - {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, - {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT}, - /* BERT encoders: WordPiece tokenizer, learned positions, no instruction - * prefix. all-MiniLM-L6-v2 mean-pools; bge-small-en-v1.5 pools the [CLS] - * token. Both emit L2-normalized float32, so they speak the OpenAI API. */ - {"all-MiniLM-L6-v2", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, NULL}, - {"bge-small-en-v1.5", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, - {"bge-base-en-v1.5", MODEL_KIND_STANDARD, 768, 768, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, - {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, - {"multilingual-e5-large-instruct", MODEL_KIND_STANDARD, 1024, 1024, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, - EMBED_E5_QUERY_INSTRUCT}, - {"snowflake-arctic-embed-l-v2.0", MODEL_KIND_STANDARD, 1024, 256, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, - EMBED_SNOWFLAKE_QUERY_INSTRUCT}, - {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, - {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, - {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, -}; +loaded_model *loaded_model_for_label(http_server *s, const char *label) { + if (!label) + return NULL; + for (int i = 0; i < s->n_models; i++) { + if (s->models[i].info && !strcmp(label, s->models[i].info->id)) + return &s->models[i]; + } + return NULL; +} + +static model_kind model_kind_from_public(embed_server_model_kind_t kind) { + switch (kind) { + case EMBED_SERVER_MODEL_CONTEXTUAL: + return MODEL_KIND_CONTEXTUAL; + case EMBED_SERVER_MODEL_LATE: + return MODEL_KIND_LATE; + case EMBED_SERVER_MODEL_STANDARD: + default: + return MODEL_KIND_STANDARD; + } +} + +static embedding_api_t embedding_api_from_public(embed_server_embedding_api_t api) { + return api == EMBED_SERVER_API_PERPLEXITY ? EMBED_API_PERPLEXITY : EMBED_API_OPENAI; +} + +static int read_optional_text_file(const char *model_dir, const char *relative, char **out) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); + if (n < 0 || (size_t)n >= sizeof(path)) { + server_log("embed-server: model path too long"); + return -1; + } + FILE *f = fopen(path, "rb"); + if (!f && errno == ENOENT) + return 0; + if (!f) { + server_log("embed-server: failed to open %s", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + free(buf); + fclose(f); + return -1; + } + fclose(f); + buf[sz] = '\0'; + *out = buf; + return 1; +} + +static int read_query_instruct_from_files(const char *model_dir, char **out) { + char *buf = NULL; + int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf); + if (rc <= 0) + return rc; + cJSON *root = cJSON_Parse(buf); + free(buf); + if (!root) { + server_log("embed-server: invalid config_sentence_transformers.json: %s", model_dir); + return -1; + } + cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts"); + cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL; + if (query && !cJSON_IsString(query)) { + cJSON_Delete(root); + server_log("embed-server: prompts.query must be a string: %s", model_dir); + return -1; + } + if (query && query->valuestring && query->valuestring[0]) { + *out = xstrdup(query->valuestring); + cJSON_Delete(root); + return *out ? 1 : -1; + } + cJSON_Delete(root); + return 0; +} -model_slot model_slot_for_id(const char *id) { - if (!id) - return MODEL_UNKNOWN; - for (int i = 0; i < MODEL_COUNT; i++) { - if (!strcmp(id, k_models[i].id)) - return (model_slot)i; +static int init_model_info(model_info **out, const embed_server_model_spec_t *spec) { + model_info *info = xcalloc(1, sizeof(*info)); + info->id = xstrdup(spec->id); + info->kind = model_kind_from_public(spec->kind); + info->api = embedding_api_from_public(spec->api); + info->min_dim = spec->min_dim; + if (spec->query_instruct) { + info->query_instruct = xstrdup(spec->query_instruct); + } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) { + free(info->id); + free(info); + return -1; + } + if (!info->id || (spec->query_instruct && !info->query_instruct)) { + free(info->query_instruct); + free(info->id); + free(info); + return -1; } - return MODEL_UNKNOWN; + *out = info; + return 0; +} + +static void free_model_info(model_info *info) { + if (!info) + return; + free(info->id); + free(info->query_instruct); + free(info); } void free_token_bufs(token_buf *t, int n) { @@ -307,14 +382,35 @@ int inference_batch_accepts_input( } int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) { - if (!m || !m->info || !config || config->hidden_size != m->info->dim || - token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode || - config->pooling_mode != m->info->pooling_mode || - config->normalize_embeddings != m->info->normalize_embeddings) { - if (m && m->info && config) - server_log("embed-server: model %s has incompatible config", m->info->id); + if (!m || !m->info || !config || config->hidden_size <= 0) { + if (m && m->info) + server_log("embed-server: model %s has invalid config", m->info->id); + return -1; + } + if (m->info->kind == MODEL_KIND_LATE) { + if (token_dim <= 0) { + server_log("embed-server: late model %s is missing token projection metadata", + m->info->id); + return -1; + } + } else if (token_dim != 0) { + server_log("embed-server: non-late model %s unexpectedly has token projection metadata", + m->info->id); return -1; } + + int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size; + if (min_dim <= 0 || min_dim > config->hidden_size) { + server_log("embed-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, + min_dim, config->hidden_size); + return -1; + } + m->info->dim = config->hidden_size; + m->info->min_dim = min_dim; + m->info->token_dim = token_dim; + m->info->attention_mode = config->attention_mode; + m->info->pooling_mode = config->pooling_mode; + m->info->normalize_embeddings = config->normalize_embeddings; m->append_terminal_token = config->append_terminal_token; /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ m->renormalize_truncated = @@ -322,9 +418,19 @@ int configure_loaded_model(loaded_model *m, const embed_config_t *config, int to return 0; } -int load_one_model(http_server *s, model_slot slot, const char *path) { - loaded_model *m = &s->models[slot]; - m->info = &k_models[slot]; +int load_one_model(http_server *s, const embed_server_model_spec_t *spec) { + /* Grow the model list by one entry. */ + loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models)); + if (!nm) + return -1; + s->models = nm; + loaded_model *m = &s->models[s->n_models]; + memset(m, 0, sizeof(*m)); + s->n_models++; + + if (init_model_info(&m->info, spec) != 0) + return -1; + const char *path = spec->path; m->path = xstrdup(path); m->terminal_token_id = -1; m->cls_id = -1; @@ -478,8 +584,9 @@ int load_one_model(http_server *s, model_slot slot, const char *path) { } void free_models(http_server *s) { - for (int i = 0; i < MODEL_COUNT; i++) { + for (int i = 0; i < s->n_models; i++) { loaded_model *m = &s->models[i]; + free_model_info(m->info); free(m->path); if (m->tok_ws) embed_tokenizer_workspace_free(m->tok_ws); @@ -514,4 +621,7 @@ void free_models(http_server *s) { embed_cuda_late_free(m->cuda_late_ctx); #endif } + free(s->models); + s->models = NULL; + s->n_models = 0; } |
