#ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include "build.h" #include "server.h" #include "server_internal.h" #include "server_util.h" #include "sbuf.h" #include "base64.h" #include "embed.h" #include "safetensors.h" #include "kernels.h" #include "tokenizer_bpe.h" #include "tokenizer_wordpiece.h" #ifdef USE_MLX #include "mlx.h" #endif #ifdef USE_CUDA #include "cuda.h" #endif #include "deps/ae/ae.h" #include "deps/ae/anet.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __APPLE__ #include #endif #include static const model_info k_models[] = { {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT}, /* BERT encoders: WordPiece tokenizer, learned positions, no instruction * prefix. all-MiniLM-L6-v2 mean-pools; bge-small-en-v1.5 pools the [CLS] * token. Both emit L2-normalized float32, so they speak the OpenAI API. */ {"all-MiniLM-L6-v2", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, NULL}, {"bge-small-en-v1.5", MODEL_KIND_STANDARD, 384, 384, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, {"bge-base-en-v1.5", MODEL_KIND_STANDARD, 768, 768, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, }; static model_slot model_slot_for_id(const char *id) { if (!id) return MODEL_UNKNOWN; for (int i = 0; i < MODEL_COUNT; i++) { if (!strcmp(id, k_models[i].id)) return (model_slot)i; } return MODEL_UNKNOWN; } static int queue_write(client *c); static void dispatch_request(client *c); static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); static volatile sig_atomic_t g_stop_requested = 0; static volatile sig_atomic_t g_signal_wfd = -1; static void stop_signal_handler(int sig) { (void)sig; if (g_stop_requested) _exit(130); int saved = errno; g_stop_requested = 1; int fd = (int)g_signal_wfd; if (fd >= 0) { const unsigned char byte = 1; if (write(fd, &byte, 1) < 0) { /* best-effort stop wakeup; a failed self-pipe write is ignored */ } } errno = saved; } static int set_nonblock(int fd) { int flags = fcntl(fd, F_GETFL, 0); if (flags < 0) return -1; return fcntl(fd, F_SETFL, flags | O_NONBLOCK); } static void http_request_free(http_request *r) { free(r->body); free(r->auth); memset(r, 0, sizeof(*r)); } static void client_free(client *c) { if (!c) return; http_request_free(&c->req); sbuf_free(&c->in); sbuf_free(&c->out); free(c); } static void client_incref(client *c) { pthread_mutex_lock(&c->srv->mu); c->refcount++; pthread_mutex_unlock(&c->srv->mu); } static void client_decref(client *c) { int free_now = 0; pthread_mutex_lock(&c->srv->mu); if (--c->refcount == 0) free_now = 1; pthread_mutex_unlock(&c->srv->mu); if (free_now) client_free(c); } static void client_decref_n(client *c, int n) { int free_now = 0; if (n <= 0) return; pthread_mutex_lock(&c->srv->mu); c->refcount -= n; if (c->refcount <= 0) free_now = 1; pthread_mutex_unlock(&c->srv->mu); if (free_now) client_free(c); } static void client_unlink(client *c) { http_server *s = c->srv; if (!c->linked) return; if (c->prev) c->prev->next = c->next; else s->clients = c->next; if (c->next) c->next->prev = c->prev; c->prev = c->next = NULL; c->linked = 0; if (s->n_clients > 0) s->n_clients--; } static int close_client_unlink(client *c) { if (!c || c->cancelled) return 0; c->cancelled = 1; if (c->fd >= 0) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE); close(c->fd); c->fd = -1; } client_unlink(c); return 1; } static void close_client(client *c) { if (!close_client_unlink(c)) return; client_decref(c); } static void append_http_response_ex(client *c, int status, const char *ctype, const char *extra_headers, const char *body, size_t body_len) { const char *reason = "OK"; if (status == 204) reason = "No Content"; else if (status == 400) reason = "Bad Request"; else if (status == 401) reason = "Unauthorized"; else if (status == 404) reason = "Not Found"; else if (status == 405) reason = "Method Not Allowed"; else if (status == 422) reason = "Unprocessable Entity"; else if (status == 503) reason = "Service Unavailable"; else if (status >= 500) reason = "Internal Server Error"; sbuf_printf(&c->out, "HTTP/1.1 %d %s\r\n" "Content-Length: %zu\r\n" "Connection: %s\r\n", status, reason, body_len, c->close_after_write ? "close" : "keep-alive"); if (ctype) sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); if (extra_headers) sbuf_puts(&c->out, extra_headers); sbuf_puts(&c->out, "\r\n"); if (body_len) sbuf_append(&c->out, body, body_len); } static void append_http_response(client *c, int status, const char *ctype, const char *body, size_t body_len) { append_http_response_ex(c, status, ctype, NULL, body, body_len); } static bool auth_ok(http_server *s, const char *auth) { if (!s->api_key || !s->api_key[0]) return true; if (!auth) return false; const char prefix[] = "Bearer "; return !strncmp(auth, prefix, sizeof(prefix) - 1) && !strcmp(auth + sizeof(prefix) - 1, s->api_key); } static ssize_t find_header_end(const char *p, size_t n) { for (size_t i = 3; i < n; i++) { if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n') return (ssize_t)(i + 1); } for (size_t i = 1; i < n; i++) { if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); } return -1; } static char *header_value_dup(const char *h, size_t n, const char *name) { size_t name_len = strlen(name); const char *p = h, *end = h + n; while (p < end) { const char *line = p; while (p < end && *p != '\n') p++; size_t len = (size_t)(p - line); if (len && line[len - 1] == '\r') len--; if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') { const char *v = line + name_len + 1; while (v < line + len && isspace((unsigned char)*v)) v++; const char *e = line + len; while (e > v && isspace((unsigned char)e[-1])) e--; return xstrndup(v, (size_t)(e - v)); } if (p < end) p++; } return NULL; } static int header_has_token(const char *value, const char *token) { size_t token_len = strlen(token); const char *p = value; while (p && *p) { while (*p == ',' || isspace((unsigned char)*p)) p++; const char *start = p; while (*p && *p != ',') p++; const char *end = p; while (end > start && isspace((unsigned char)end[-1])) end--; if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len)) return 1; } return 0; } static int parse_request_headers(client *c) { char line[512]; size_t i = 0; while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) { line[i] = c->in.ptr[i]; i++; } line[i] = '\0'; int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version); if (fields < 2) return -1; if (fields < 3) snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); char *q = strchr(c->req.path, '?'); if (q) *q = '\0'; c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); if (conn) { if (header_has_token(conn, "close")) c->close_after_write = 1; else if (header_has_token(conn, "keep-alive")) c->close_after_write = 0; free(conn); } char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length"); if (cl) { char *end = NULL; errno = 0; unsigned long long v = strtoull(cl, &end, 10); if (errno || !end || *end || v > EMBED_HTTP_MAX_BODY) { free(cl); return -1; } c->content_length = (size_t)v; free(cl); } c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization"); return 0; } static int complete_request(client *c) { if (!c->header_done) { ssize_t hend = find_header_end(c->in.ptr, c->in.len); if (hend < 0) { if (c->in.len > EMBED_HTTP_MAX_HEADER) return -1; return 0; } c->header_done = 1; c->header_len = (size_t)hend; if (parse_request_headers(c) != 0) return -1; } if (c->content_length > EMBED_HTTP_MAX_BODY) return -1; if (c->in.len < c->header_len + c->content_length) return 0; c->req.body_len = c->content_length; c->req.body = xmalloc(c->content_length + 1); memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); c->req.body[c->content_length] = '\0'; return 1; } static void send_bad_http_request(client *c) { size_t len; char *body = json_error_body("bad HTTP request", "invalid_request_error", &len); c->close_after_write = 1; append_http_response(c, 400, "application/json", body, len); free(body); aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); if (queue_write(c) != AE_OK) close_client(c); } static void reset_client_for_next_request(client *c) { size_t consumed = c->header_len + c->content_length; if (consumed > c->in.len) consumed = c->in.len; size_t remain = c->in.len - consumed; if (remain) memmove(c->in.ptr, c->in.ptr + consumed, remain); c->in.len = remain; if (c->in.ptr) c->in.ptr[remain] = '\0'; http_request_free(&c->req); sbuf_clear(&c->out); c->sent = 0; c->header_done = 0; c->header_len = 0; c->content_length = 0; c->close_after_write = 0; } static void arm_next_request(client *c) { reset_client_for_next_request(c); if (c->cancelled || c->fd < 0) return; if (c->in.len) { int done = complete_request(c); if (done < 0) { send_bad_http_request(c); return; } if (done > 0) { dispatch_request(c); return; } } if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK) close_client(c); } static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)loop; (void)mask; client *c = clientData; while (c->sent < c->out.len) { ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); if (n > 0) { c->sent += (size_t)n; c->last_active_ms = mstime(); continue; } if (n < 0 && errno == EINTR) continue; if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; close_client(c); return; } aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); if (c->close_after_write) close_client(c); else arm_next_request(c); } static int queue_write(client *c) { if (!c || c->cancelled || c->fd < 0) return -1; return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); } static void enqueue_done(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); if (s->done_tail) s->done_tail->next = j; else s->done_head = j; s->done_tail = j; pthread_mutex_unlock(&s->mu); const unsigned char byte = 1; if (write(s->completion_pipe[1], &byte, 1) < 0) { /* best-effort wakeup; the event loop also drains the done queue */ } } static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { s->job_head = j->next; if (!s->job_head) s->job_tail = NULL; j->next = NULL; } return j; } static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) { uint64_t now_ns = nstime(); if (now_ns >= deadline_ns) return ETIMEDOUT; uint64_t remaining_ns = deadline_ns - now_ns; struct timespec abs; if (clock_gettime(CLOCK_REALTIME, &abs) != 0) { struct timeval tv; gettimeofday(&tv, NULL); abs.tv_sec = tv.tv_sec; abs.tv_nsec = (long)tv.tv_usec * 1000L; } abs.tv_sec += (time_t)(remaining_ns / 1000000000u); abs.tv_nsec += (long)(remaining_ns % 1000000000u); if (abs.tv_nsec >= 1000000000L) { abs.tv_sec++; abs.tv_nsec -= 1000000000L; } return pthread_cond_timedwait(cv, mu, &abs); } static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { if (max_jobs <= 0) return 0; pthread_mutex_lock(&s->mu); while (!s->job_head && !s->stopping) pthread_cond_wait(&s->cv, &s->mu); job *first = pop_job_locked(s); if (!first) { pthread_mutex_unlock(&s->mu); return 0; } int n = 1; jobs[0] = first; int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") || !strcmp(first->path, "/v1/contextualizedembeddings")); if (!batchable) { pthread_mutex_unlock(&s->mu); return n; } /* Exact item and token budgets are applied after worker-side tokenization; * this queue stage only defines which arrival window may be grouped. */ uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns; while (n < max_jobs) { while (!s->job_head && !s->stopping && wait_ns > 0) { int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); if (rc == ETIMEDOUT) break; } if (!s->job_head) break; if (wait_ns > 0 && s->job_head->created_ns > deadline_ns) break; jobs[n++] = pop_job_locked(s); if (wait_ns == 0 && !s->job_head) break; } pthread_mutex_unlock(&s->mu); return n; } static void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); if (s->job_tail) s->job_tail->next = j; else s->job_head = j; s->job_tail = j; pthread_cond_signal(&s->cv); pthread_mutex_unlock(&s->mu); } static job *pop_done(http_server *s) { pthread_mutex_lock(&s->mu); job *j = s->done_head; if (j) { s->done_head = j->next; if (!s->done_head) s->done_tail = NULL; j->next = NULL; } pthread_mutex_unlock(&s->mu); return j; } static void job_free(job *j) { if (!j) return; free(j->body); free(j->auth); free(j->extra_headers); free(j->response); free(j); } static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)loop; (void)mask; http_server *s = clientData; unsigned char tmp[128]; while (read(fd, tmp, sizeof(tmp)) > 0) { } job *j; while ((j = pop_done(s)) != NULL) { client *c = j->c; int refs_to_drop = 1; /* job reference */ if (!c->cancelled) { append_http_response_ex(c, j->status, j->content_type, j->extra_headers, j->response ? j->response : "", j->response ? j->response_len : 0); if (queue_write(c) != AE_OK) refs_to_drop += close_client_unlink(c); } client_decref_n(c, refs_to_drop); job_free(j); } } static bool route_is_inference(const char *method, const char *path) { return !strcmp(method, "POST") && (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") || !strcmp(path, "/v1/rerank")); } static void dispatch_request(client *c) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); if (!strcmp(c->req.method, "OPTIONS")) { append_http_response(c, 204, NULL, "", 0); if (queue_write(c) != AE_OK) close_client(c); return; } if (!route_is_inference(c->req.method, c->req.path)) { size_t len; char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); append_http_response(c, 404, "application/json", body, len); free(body); if (queue_write(c) != AE_OK) close_client(c); return; } if (!auth_ok(c->srv, c->req.auth)) { size_t len; char *body = json_error_body("missing or invalid bearer token", "authentication_error", &len); append_http_response(c, 401, "application/json", body, len); free(body); if (queue_write(c) != AE_OK) close_client(c); return; } job *j = xcalloc(1, sizeof(*j)); j->srv = c->srv; j->c = c; snprintf(j->method, sizeof(j->method), "%s", c->req.method); snprintf(j->path, sizeof(j->path), "%s", c->req.path); j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); j->body_len = c->req.body_len; j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; j->created_ns = nstime(); client_incref(c); enqueue_job(j); } static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)loop; (void)mask; client *c = clientData; char tmp[EMBED_HTTP_IO_BUF]; for (;;) { ssize_t n = read(fd, tmp, sizeof(tmp)); if (n > 0) { sbuf_append(&c->in, tmp, (size_t)n); c->last_active_ms = mstime(); int done = complete_request(c); if (done < 0) { send_bad_http_request(c); return; } if (done > 0) { dispatch_request(c); return; } continue; } if (n == 0) { close_client(c); return; } if (errno == EINTR) continue; if (errno == EAGAIN || errno == EWOULDBLOCK) return; close_client(c); return; } } static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)loop; (void)mask; http_server *s = clientData; char cip[64]; int cport = 0; for (;;) { int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); if (cfd == ANET_ERR) { if (errno == EAGAIN || errno == EWOULDBLOCK) return; if (errno == EINTR) continue; return; } set_nonblock(cfd); client *c = xcalloc(1, sizeof(*c)); c->srv = s; c->fd = cfd; c->refcount = 1; c->last_active_ms = mstime(); c->linked = 1; c->next = s->clients; if (s->clients) s->clients->prev = c; s->clients = c; s->n_clients++; if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) close_client(c); } } static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { (void)loop; (void)id; http_server *s = clientData; uint64_t now = mstime(); client *c = s->clients; while (c) { client *next = c->next; /* refcount > 1 means a request from this client is queued or being * processed by the worker; long batch queues (e.g. 4B long-document * batches) must not be reaped as idle. */ if (!c->cancelled && c->refcount <= 1 && now - c->last_active_ms > EMBED_HTTP_CLIENT_TIMEOUT_MS) close_client(c); c = next; } return EMBED_HTTP_SWEEP_MS; } static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)fd; (void)mask; http_server *s = clientData; unsigned char tmp[128]; while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { } aeStop(loop); } static void job_set_timing_header(job *j) { if (!j || j->extra_headers || !j->created_ns) return; uint64_t done_ns = nstime(); uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0; uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0; char buf[384]; int n = snprintf(buf, sizeof(buf), "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " "worker;dur=%.3f\r\n", ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); if (n > 0 && (size_t)n < sizeof(buf)) j->extra_headers = xstrdup(buf); } static void free_token_bufs(token_buf *t, int n) { if (!t) return; for (int i = 0; i < n; i++) free(t[i].ids); } static void embedding_request_free(embedding_request *r) { if (!r) return; free_token_bufs(r->tokens, r->n_inputs); free(r->tokens); free(r->inputs); if (r->root) cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void contextual_request_free(contextual_request *r) { if (!r) return; for (int i = 0; i < r->n_docs; i++) { free(r->docs[i].ids); free(r->docs[i].spans); } free(r->docs); if (r->root) cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void late_tokens_free(late_tokens *t) { if (!t) return; free(t->ids); free(t->keep); memset(t, 0, sizeof(*t)); } static void rerank_request_free(rerank_request *r) { if (!r) return; late_tokens_free(&r->query); for (int i = 0; i < r->n_documents; i++) late_tokens_free(&r->documents[i]); free(r->documents); if (r->root) cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); uint64_t t0 = nstime(); if (m->wp_tok) { int n = 0; int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n); if (j) j->tokenize_ns += nstime() - t0; /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE * branch below appends its terminal token. A whitespace-only input * cleans to zero core tokens (core == NULL, n == 0); the bare * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the * returned buffer in place so only one allocation is touched. */ if (n < 0 || n > INT_MAX - 2) { free(core); memset(out, 0, sizeof(*out)); return -1; } int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); if (!ids) { free(core); memset(out, 0, sizeof(*out)); return -1; } memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); ids[0] = m->cls_id; ids[n + 1] = m->sep_id; out->ids = ids; out->n_tokens = n + 2; return 0; } out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); if (j) j->tokenize_ns += nstime() - t0; if (!out->ids || out->n_tokens <= 0) { free(out->ids); memset(out, 0, sizeof(*out)); return -1; } if (m->append_terminal_token) { if (out->n_tokens == INT_MAX) { free(out->ids); memset(out, 0, sizeof(*out)); return -1; } int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); if (!ids) { free(out->ids); memset(out, 0, sizeof(*out)); return -1; } out->ids = ids; out->ids[out->n_tokens++] = m->terminal_token_id; } return 0; } /* Tokenize one embedding input, optionally prepending the model's query * instruction. The instruction and input are tokenized in one pass. */ static int tokenize_input( loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { if (!query_instruct) return tokenize_one(m, j, text, out); size_t plen = strlen(query_instruct); size_t tlen = strlen(text); char *buf = (char *)malloc(plen + tlen + 1); if (!buf) { memset(out, 0, sizeof(*out)); return -1; } memcpy(buf, query_instruct, plen); memcpy(buf + plen, text, tlen + 1); int rc = tokenize_one(m, j, buf, out); free(buf); return rc; } static int late_id_is_skipped(const loaded_model *m, int id) { for (int i = 0; i < m->n_late_skip_ids; i++) { if (m->late_skip_ids[i] == id) return 1; } return 0; } static int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { token_buf raw = {0}; if (tokenize_one(m, j, text, &raw) != 0) return -1; int raw_tokens = raw.n_tokens; int target = raw_tokens + 1; if (is_query && target < EMBED_LATE_QUERY_TOKENS) target = EMBED_LATE_QUERY_TOKENS; out->ids = xmalloc((size_t)target * sizeof(*out->ids)); out->ids[0] = raw.ids[0]; out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id; if (raw_tokens > 1) { memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids)); } out->n_tokens = raw_tokens + 1; if (is_query) { while (out->n_tokens < EMBED_LATE_QUERY_TOKENS) out->ids[out->n_tokens++] = m->late_mask_id; out->n_keep = out->n_tokens; } else { out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep)); for (int i = 0; i < out->n_tokens; i++) { if (!late_id_is_skipped(m, out->ids[i])) out->keep[out->n_keep++] = i; } } free(raw.ids); return out->n_keep > 0 ? 0 : -1; } static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out); #endif #ifdef USE_CUDA if (m->cuda_ctx) return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out); #endif return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } static int model_embed_spans_batch(loaded_model *m, const embed_context_input_t *inputs, int batch, float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out); #endif #ifdef USE_CUDA if (m->cuda_ctx) return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out); #endif return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } static int model_uses_dense_batches(const loaded_model *m) { #ifdef USE_MLX return m->mlx_ctx != NULL; #else (void)m; return 0; #endif } /* Inputs are sorted by length before chunking. CPU packs real tokens while MLX * pads each dense row to the longest input in the chunk. */ static int inference_batch_accepts_input( const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { if (batch == 0) return 1; if (model_uses_dense_batches(m)) return next_tokens <= max_batch_tokens / (batch + 1); return next_tokens <= max_batch_tokens - packed_tokens; } static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } typedef struct { embed_input_t input; int output_index; } embedding_batch_item; static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; const embedding_batch_item *ib = b; if (ia->input.n_tokens < ib->input.n_tokens) return -1; if (ia->input.n_tokens > ib->input.n_tokens) return 1; return ia->output_index - ib->output_index; } static int embedding_request_fits_group(const embedding_request *r, int group_inputs, int group_tokens, int max_batch, int max_batch_tokens) { if (group_inputs == 0) return 1; return r->n_inputs <= max_batch - group_inputs && r->total_tokens <= max_batch_tokens - group_tokens; } static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { if (n_reqs <= 0) return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_inputs = 0; for (int i = 0; i < n_reqs; i++) total_inputs += reqs[i]->n_inputs; embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); int pos = 0; for (int i = 0; i < n_reqs; i++) { for (int k = 0; k < reqs[i]->n_inputs; k++) { items[pos].input = reqs[i]->inputs[k]; items[pos].output_index = pos; pos++; } } qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; if (max_batch > total_inputs) max_batch = total_inputs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); int failed = 0; uint64_t infer_ns = 0; for (int start = 0; start < total_inputs;) { int cur = 0; int tokens = 0; while (start + cur < total_inputs && cur < max_batch) { int n_tokens = items[start + cur].input.n_tokens; if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) break; inputs[cur] = items[start + cur].input; tokens += n_tokens; cur++; } uint64_t t0 = nstime(); int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); infer_ns += nstime() - t0; if (embed_rc != 0) { failed = 1; break; } for (int i = 0; i < cur; i++) memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); start += cur; } pos = 0; for (int i = 0; i < n_reqs; i++) { reqs[i]->j->infer_ns += infer_ns; if (failed) { job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); } else { uint64_t t0 = nstime(); if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); reqs[i]->j->encode_ns += nstime() - t0; } pos += reqs[i]->n_inputs; } free(batch_embs); free(inputs); free(embs); free(items); } static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } static int contextual_request_fits_group(const contextual_request *r, int group_docs, int group_tokens, int max_batch, int max_batch_tokens) { if (group_docs == 0) return 1; return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; } typedef struct { embed_context_input_t input; int output_span_index; int order; } contextual_batch_item; static int contextual_batch_item_cmp(const void *a, const void *b) { const contextual_batch_item *ia = a; const contextual_batch_item *ib = b; if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; return ia->order - ib->order; } static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { if (n_reqs <= 0) return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_docs = 0; int total_chunks = 0; for (int i = 0; i < n_reqs; i++) { if (total_docs > INT_MAX - reqs[i]->n_docs || total_chunks > INT_MAX - reqs[i]->total_chunks) { for (int k = 0; k < n_reqs; k++) job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); return; } total_docs += reqs[i]->n_docs; total_chunks += reqs[i]->total_chunks; } if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { for (int i = 0; i < n_reqs; i++) job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); return; } contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); int pos = 0; int span_offset = 0; for (int i = 0; i < n_reqs; i++) { for (int d = 0; d < reqs[i]->n_docs; d++) { contextual_doc *doc = &reqs[i]->docs[d]; items[pos].input.input.ids = doc->ids; items[pos].input.input.n_tokens = doc->n_tokens; items[pos].input.spans = doc->spans; items[pos].input.n_spans = doc->n_spans; items[pos].output_span_index = span_offset; items[pos].order = pos; span_offset += doc->n_spans; pos++; } } qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; if (max_batch > total_docs) max_batch = total_docs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc(sizeof(*batch_embs)); size_t batch_emb_cap = 0; int failed = 0; uint64_t infer_ns = 0; for (int start = 0; start < total_docs;) { int cur = 0; int tokens = 0; int chunks = 0; while (start + cur < total_docs && cur < max_batch) { const embed_context_input_t *input = &items[start + cur].input; if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, max_batch_tokens)) break; inputs[cur] = *input; tokens += input->input.n_tokens; chunks += input->n_spans; cur++; } size_t needed = (size_t)chunks * dim; if (needed > batch_emb_cap) { batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); batch_emb_cap = needed; } uint64_t t0 = nstime(); int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); infer_ns += nstime() - t0; if (embed_rc != 0) { failed = 1; break; } int batch_span = 0; for (int i = 0; i < cur; i++) { int n_spans = inputs[i].n_spans; memcpy(embs + (size_t)items[start + i].output_span_index * dim, batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); batch_span += n_spans; } start += cur; } span_offset = 0; for (int i = 0; i < n_reqs; i++) { reqs[i]->j->infer_ns += infer_ns; if (failed) { job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); } else { uint64_t t0 = nstime(); render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); reqs[i]->j->encode_ns += nstime() - t0; } span_offset += reqs[i]->total_chunks; } free(batch_embs); free(inputs); free(embs); free(items); } static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, loaded_model **out_model, int *out_dims, const char **out_encoding, http_server *s) { cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); const char *model_id = NULL; if (!model_item) { ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); } else { model_id = model_item->valuestring; model_slot slot = model_slot_for_id(model_id); if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) { ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", "enum"); } else { loaded_model *m = &s->models[slot]; if (!m->info) { *out_model = NULL; } else { *out_model = m; } } } model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; const char *enc = encoding_from_root(root, detail, api); int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); *out_encoding = enc; *out_dims = dims; return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && !s->models[model_slot_for_id(model_id)].info ? 503 : 0; } static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; cJSON *detail = cJSON_CreateArray(); loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; /* Resolve request behavior from the model registry independent of load * state. The query instruction, when requested, applies to every input. */ cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring) ? model_slot_for_id(model_item->valuestring) : MODEL_UNKNOWN; const char *query_instruct = api_slot != MODEL_UNKNOWN && text_type_is_query(root, detail, k_models[api_slot].query_instruct) ? k_models[api_slot].query_instruct : NULL; cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); int n_inputs = 0; if (!input) { ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (cJSON_IsString(input)) { if (!input->valuestring || input->valuestring[0] == '\0') ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); n_inputs = 1; } else if (cJSON_IsArray(input)) { n_inputs = cJSON_GetArraySize(input); if (n_inputs < 1) ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", "value_error.list.min_items"); else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", "value_error.list.max_items"); cJSON *item; int i = 0; cJSON_ArrayForEach(item, input) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); if (!cJSON_IsString(item)) { ve_add(detail, loc, "input item must be a string", "type_error.string"); } else if (!item->valuestring || item->valuestring[0] == '\0') { ve_add(detail, loc, "input item must not be empty", "value_error.empty"); } i++; } } else { ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", "type_error"); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } if (cJSON_GetArraySize(detail) == 0 && m && input) { out->n_inputs = n_inputs; out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); int idx = 0; if (input && cJSON_IsString(input) && input->valuestring) { if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) { ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", "value_error.tokenization"); } else { out->inputs[0].ids = out->tokens[0].ids; out->inputs[0].n_tokens = out->tokens[0].n_tokens; out->total_tokens = out->tokens[0].n_tokens; if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", "value_error.context_length"); } } else { cJSON *item; cJSON_ArrayForEach(item, input) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); if (!cJSON_IsString(item) || !item->valuestring) { ve_add(detail, loc, "input item must be a string", "type_error.string"); idx++; continue; } if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != 0) { ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { out->inputs[idx].ids = out->tokens[idx].ids; out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; out->total_tokens += out->tokens[idx].n_tokens; if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS) ve_add(detail, loc, "input exceeds 32768 token limit", "value_error.context_length"); } idx++; } } if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS) ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } if (cJSON_GetArraySize(detail) > 0) { job_set_422(j, detail); embedding_request_free(out); out->j = j; } else { out->ready = 1; } cJSON_Delete(detail); } static void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; cJSON *detail = cJSON_CreateArray(); loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); int n_docs = 0; int total_chunks = 0; if (!input) { ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (!cJSON_IsArray(input)) { ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", "type_error.array"); } else { n_docs = cJSON_GetArraySize(input); if (n_docs < 1) ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", "value_error.list.min_items"); else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", "value_error.list.max_items"); cJSON *doc_arr; int di = 0; cJSON_ArrayForEach(doc_arr, input) { if (!cJSON_IsArray(doc_arr)) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document must be an array of strings", "type_error.array"); di++; continue; } int doc_chunks = cJSON_GetArraySize(doc_arr); if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks) total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1; else total_chunks += doc_chunks; if (doc_chunks < 1) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document must contain at least 1 chunk", "value_error.list.min_items"); } cJSON *chunk; int ci = 0; cJSON_ArrayForEach(chunk, doc_arr) { char loc[80]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); if (!cJSON_IsString(chunk)) { ve_add(detail, loc, "chunk must be a string", "type_error.string"); } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); } ci++; } di++; } if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", "value_error.list.max_items"); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } if (cJSON_GetArraySize(detail) == 0 && m) { out->n_docs = n_docs; out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); int64_t request_tokens = 0; cJSON *doc_arr; int di = 0; cJSON_ArrayForEach(doc_arr, input) { contextual_doc *doc = &out->docs[di]; int n_chunks = cJSON_GetArraySize(doc_arr); token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); doc->n_spans = n_chunks; int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; int doc_valid = 1; cJSON *chunk; int ci = 0; cJSON_ArrayForEach(chunk, doc_arr) { char loc[80]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); doc_valid = 0; } else { doc_tokens += chunk_tokens[ci].n_tokens; } ci++; } if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document exceeds 32768 token limit", "value_error.context_length"); doc_valid = 0; } request_tokens += doc_tokens; if (doc_valid) { doc->n_tokens = (int)doc_tokens; doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); int pos = 0; for (ci = 0; ci < n_chunks; ci++) { doc->spans[ci].start = pos; doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; /* n_tokens > 0 implies ids != NULL (tokenize_one sets both * together), and the guard avoids a memcpy(dst, NULL, 0) for * an empty chunk that the static analyzer reports as UB. */ if (chunk_tokens[ci].n_tokens > 0) memcpy(doc->ids + pos, chunk_tokens[ci].ids, (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); pos += chunk_tokens[ci].n_tokens; if (ci + 1 < n_chunks) { doc->ids[pos++] = m->context_separator_id; } } } free_token_bufs(chunk_tokens, n_chunks); free(chunk_tokens); di++; } if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } else { out->total_tokens = (int)request_tokens; out->total_chunks = total_chunks; } } if (cJSON_GetArraySize(detail) > 0) { job_set_422(j, detail); contextual_request_free(out); out->j = j; } else { out->ready = 1; } cJSON_Delete(detail); } static int validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); if (!item) { ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); return 0; } if (!cJSON_IsString(item) || !item->valuestring) { ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); return 0; } model_slot slot = model_slot_for_id(item->valuestring); if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", "enum"); return 0; } *out_model = s->models[slot].info ? &s->models[slot] : NULL; return *out_model ? 0 : 503; } static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; cJSON *detail = cJSON_CreateArray(); int unloaded = validate_rerank_model(root, detail, s, &out->model); cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query"); cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents"); if (!query) { ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); } else if (!cJSON_IsString(query)) { ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); } else if (!query->valuestring || !query->valuestring[0]) { ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); } int n_documents = 0; if (!documents) { ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); } else if (!cJSON_IsArray(documents)) { ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", "type_error.array"); } else { n_documents = cJSON_GetArraySize(documents); if (n_documents < 1) ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", "value_error.list.min_items"); else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", "value_error.list.max_items"); cJSON *document; int i = 0; cJSON_ArrayForEach(document, documents) { char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); if (!cJSON_IsString(document)) ve_add(detail, loc, "document must be a string", "type_error.string"); else if (!document->valuestring || !document->valuestring[0]) ve_add(detail, loc, "document must not be empty", "value_error.empty"); } } cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); if (top_n && top_k) { ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", "value_error.conflict"); } cJSON *top = top_n ? top_n : top_k; out->top_n = n_documents; if (top) { if (!cjson_is_integer(top)) { ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", "top_n must be an integer", "type_error.integer"); } else { out->top_n = top->valueint; if (out->top_n < 1 || out->top_n > n_documents) ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", "top_n must be between 1 and the number of documents", "value_error.range"); } } cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); if (return_documents) { if (!cJSON_IsBool(return_documents)) ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", "type_error.bool"); else out->return_documents = cJSON_IsTrue(return_documents); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } if (cJSON_GetArraySize(detail) == 0 && out->model) { out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) { ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); } else { out->query_tokens = out->query.n_tokens; if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", "value_error.context_length"); } cJSON *document; int i = 0; cJSON_ArrayForEach(document, documents) { char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != 0) { ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { int n = out->documents[i].n_tokens; if (n > EMBED_API_MAX_ITEM_TOKENS) ve_add(detail, loc, "document exceeds 32768 token limit", "value_error.context_length"); if (out->document_tokens <= INT_MAX - n) out->document_tokens += n; else out->document_tokens = INT_MAX; } i++; } if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", "value_error.context_length"); } } if (cJSON_GetArraySize(detail) > 0) { job_set_422(j, detail); rerank_request_free(out); out->j = j; } else { out->ready = 1; } cJSON_Delete(detail); } typedef struct { int index; float score; } rerank_result; static int rerank_result_cmp(const void *a, const void *b) { const rerank_result *ra = a; const rerank_result *rb = b; if (ra->score > rb->score) return -1; if (ra->score < rb->score) return 1; return ra->index - rb->index; } static int execute_rerank_cpu(rerank_request *r, float *scores) { loaded_model *m = r->model; int dim = m->info->token_dim; int total_doc_vecs = 0; for (int i = 0; i < r->n_documents; i++) total_doc_vecs += r->documents[i].n_keep; float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); /* Encode every candidate in one block-diagonal forward. The encoder packs * each document's kept token vectors back-to-back and fills offsets, so the * cost of N short documents collapses to a single packed forward. */ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); for (int i = 0; i < r->n_documents; i++) { doc_ids[i] = r->documents[i].ids; keep[i] = r->documents[i].keep; n_tokens[i] = r->documents[i].n_tokens; n_keep[i] = r->documents[i].n_keep; } int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens, 1, query); if (rc == 0) rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, keep, n_keep, r->n_documents, 1, docs, offsets); if (rc == 0) rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, scores); free(n_keep); free(n_tokens); free(keep); free(doc_ids); free(offsets); free(docs); free(query); return rc; } #ifdef USE_MLX static int execute_rerank_mlx(rerank_request *r, float *scores) { loaded_model *m = r->model; embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); /* Encode every candidate in one padded transformer pass. The encoder packs * each document's kept token vectors back-to-back and fills offsets, so the * cost of N short documents collapses to a single batched forward. */ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); for (int i = 0; i < r->n_documents; i++) { doc_ids[i] = r->documents[i].ids; keep[i] = r->documents[i].keep; n_tokens[i] = r->documents[i].n_tokens; n_keep[i] = r->documents[i].n_keep; } embed_mlx_late_vectors_t *packed = query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep, r->n_documents, 1, offsets) : NULL; int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, r->n_documents, scores) : -1; embed_mlx_late_vectors_free(packed); embed_mlx_late_vectors_free(query); free(offsets); free(n_keep); free(n_tokens); free(keep); free(doc_ids); return rc; } #endif #ifdef USE_CUDA static int execute_rerank_cuda(rerank_request *r, float *scores) { loaded_model *m = r->model; int dim = m->info->token_dim; int total_doc_vecs = 0; for (int i = 0; i < r->n_documents; i++) total_doc_vecs += r->documents[i].n_keep; float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); /* GPU encodes the query and every candidate (one packed forward); MaxSim * runs on the host, where the grouped-GEMM scorer beats a device graph. */ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); for (int i = 0; i < r->n_documents; i++) { doc_ids[i] = r->documents[i].ids; keep[i] = r->documents[i].keep; n_tokens[i] = r->documents[i].n_tokens; n_keep[i] = r->documents[i].n_keep; } int rc = embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query); if (rc == 0) rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep, r->n_documents, 1, docs, offsets); if (rc == 0) rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, scores); free(n_keep); free(n_tokens); free(keep); free(doc_ids); free(offsets); free(docs); free(query); return rc; } #endif static void execute_rerank_request(rerank_request *r) { if (!r || !r->model) { if (r && r->j) job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); return; } float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); uint64_t t0 = nstime(); #if defined(USE_MLX) int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); #elif defined(USE_CUDA) int rc = r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores); #else int rc = execute_rerank_cpu(r, scores); #endif r->j->infer_ns += nstime() - t0; if (rc != 0) { free(scores); job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); return; } rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); for (int i = 0; i < r->n_documents; i++) { ranked[i].index = i; ranked[i].score = scores[i]; } qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp); t0 = nstime(); sbuf b = {0}; sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); for (int i = 0; i < r->top_n; i++) { if (i) sbuf_putc(&b, ','); sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, (double)ranked[i].score); if (r->return_documents) { cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); sbuf_puts(&b, ",\"document\":"); append_json_string(&b, document->valuestring); } sbuf_putc(&b, '}'); } sbuf_printf(&b, "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," "\"total_tokens\":%d}}", r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); set_response_from_buf(r->j, &b); r->j->encode_ns += nstime() - t0; free(ranked); free(scores); } static int parse_job_root(job *j, cJSON **out_root) { cJSON *detail = cJSON_CreateArray(); cJSON *root = parse_json_body(j, detail); if (!root) { job_set_422(j, detail); cJSON_Delete(detail); *out_root = NULL; return -1; } cJSON_Delete(detail); *out_root = root; return 0; } static void process_unknown_job(job *j) { cJSON *root = NULL; if (!j->started_ns) j->started_ns = nstime(); uint64_t t0 = nstime(); int rc = parse_job_root(j, &root); j->parse_ns += nstime() - t0; if (rc != 0) return; job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); cJSON_Delete(root); } static void process_job_group(http_server *s, job **jobs, int n_jobs) { embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); int *done = xcalloc((size_t)n_jobs, sizeof(*done)); embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); for (int i = 0; i < n_jobs; i++) { cJSON *root = NULL; if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime(); if (!strcmp(jobs[i]->path, "/v1/embeddings")) { kind[i] = 1; } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { kind[i] = 2; } else if (!strcmp(jobs[i]->path, "/v1/rerank")) { kind[i] = 3; } else { continue; } uint64_t t0 = nstime(); int parse_rc = parse_job_root(jobs[i], &root); jobs[i]->parse_ns += nstime() - t0; if (parse_rc != 0) continue; if (kind[i] == 1) prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); else if (kind[i] == 2) prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); else prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]); } for (int i = 0; i < n_jobs; i++) { if (done[i]) continue; if (!kind[i]) { process_unknown_job(jobs[i]); job_set_timing_header(jobs[i]); enqueue_done(jobs[i]); done[i] = 1; continue; } if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) || (kind[i] == 3 && !rerank_reqs[i].ready)) { job_set_timing_header(jobs[i]); enqueue_done(jobs[i]); done[i] = 1; continue; } if (kind[i] == 3) { execute_rerank_request(&rerank_reqs[i]); job_set_timing_header(jobs[i]); enqueue_done(jobs[i]); done[i] = 1; } else if (kind[i] == 1) { int group_n = 1; int group_inputs = std_reqs[i].n_inputs; int group_tokens = std_reqs[i].total_tokens; std_group[0] = &std_reqs[i]; for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 1 && std_reqs[k].ready && embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens, s->batch_size, s->max_batch_tokens)) { std_group[group_n++] = &std_reqs[k]; group_inputs += std_reqs[k].n_inputs; group_tokens += std_reqs[k].total_tokens; } } execute_embedding_request_list(std_group, group_n); for (int k = 0; k < group_n; k++) { int idx = (int)(std_group[k] - std_reqs); job_set_timing_header(jobs[idx]); enqueue_done(jobs[idx]); done[idx] = 1; } } else { int group_n = 1; int group_docs = ctx_reqs[i].n_docs; int group_tokens = ctx_reqs[i].total_tokens; ctx_group[0] = &ctx_reqs[i]; for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens, s->batch_size, s->max_batch_tokens)) { ctx_group[group_n++] = &ctx_reqs[k]; group_docs += ctx_reqs[k].n_docs; group_tokens += ctx_reqs[k].total_tokens; } } execute_contextual_request_list(ctx_group, group_n); for (int k = 0; k < group_n; k++) { int idx = (int)(ctx_group[k] - ctx_reqs); job_set_timing_header(jobs[idx]); enqueue_done(jobs[idx]); done[idx] = 1; } } } for (int i = 0; i < n_jobs; i++) { embedding_request_free(&std_reqs[i]); contextual_request_free(&ctx_reqs[i]); rerank_request_free(&rerank_reqs[i]); } free(ctx_group); free(std_group); free(done); free(kind); free(rerank_reqs); free(ctx_reqs); free(std_reqs); } static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) { if (!m || !m->info || !config || config->hidden_size != m->info->dim || token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode || config->pooling_mode != m->info->pooling_mode || config->normalize_embeddings != m->info->normalize_embeddings) { if (m && m->info && config) server_log("embed-server: model %s has incompatible config", m->info->id); return -1; } m->append_terminal_token = config->append_terminal_token; /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ m->renormalize_truncated = config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN; return 0; } static void *worker_main(void *arg) { http_server *s = arg; #ifdef USE_MLX if (s->use_mlx) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (!m->info) continue; embed_mlx_options_t mlx_opts = { .quantize_bits = s->mlx_quantize_bits, .quantize_group_size = s->mlx_quantize_group_size, }; if (m->info->kind == MODEL_KIND_LATE) m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts); else m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts); if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) || (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) { server_log("embed-server: failed to load MLX model on worker: %s", m->path); rc = 1; break; } const embed_config_t *config = m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_config(m->mlx_late_ctx) : embed_mlx_config(m->mlx_ctx); int token_dim = m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; if (configure_loaded_model(m, config, token_dim) != 0 || (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size))) { rc = 1; break; } } pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; if (rc) s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (m->mlx_ctx) { embed_mlx_free(m->mlx_ctx); m->mlx_ctx = NULL; } if (m->mlx_late_ctx) { embed_mlx_late_free(m->mlx_late_ctx); m->mlx_late_ctx = NULL; } } return NULL; } } else #endif #ifdef USE_CUDA if (s->use_cuda) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (!m->info) continue; if (m->info->kind == MODEL_KIND_LATE) m->cuda_late_ctx = embed_cuda_late_load(m->path); else m->cuda_ctx = embed_cuda_load(m->path); if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) || (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) { server_log("embed-server: failed to load CUDA model on worker: %s", m->path); rc = 1; break; } const embed_config_t *config = m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_config(m->cuda_late_ctx) : embed_cuda_config(m->cuda_ctx); int token_dim = m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0; if (configure_loaded_model(m, config, token_dim) != 0 || (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size))) { rc = 1; break; } } pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; if (rc) s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (m->cuda_ctx) { embed_cuda_free(m->cuda_ctx); m->cuda_ctx = NULL; } if (m->cuda_late_ctx) { embed_cuda_late_free(m->cuda_late_ctx); m->cuda_late_ctx = NULL; } } return NULL; } } else #endif { pthread_mutex_lock(&s->mu); s->worker_ready = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); } for (;;) { job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); if (n_jobs == 0) break; process_job_group(s, jobs, n_jobs); } #ifdef USE_MLX if (s->use_mlx) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (m->mlx_ctx) { embed_mlx_free(m->mlx_ctx); m->mlx_ctx = NULL; } if (m->mlx_late_ctx) { embed_mlx_late_free(m->mlx_late_ctx); m->mlx_late_ctx = NULL; } } } #endif #ifdef USE_CUDA if (s->use_cuda) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; if (m->cuda_ctx) { embed_cuda_free(m->cuda_ctx); m->cuda_ctx = NULL; } if (m->cuda_late_ctx) { embed_cuda_late_free(m->cuda_late_ctx); m->cuda_late_ctx = NULL; } } } #endif return NULL; } static int listen_on(const char *host, int port) { char portbuf[32]; snprintf(portbuf, sizeof(portbuf), "%d", port); char err[ANET_ERR_LEN]; int fd = anetTcpServer(err, port, (char *)host, EMBED_HTTP_BACKLOG); (void)portbuf; if (fd == ANET_ERR) { server_log("embed-server: listen failed on %s:%d: %s", host, port, err); return -1; } set_nonblock(fd); return fd; } static int load_one_model(http_server *s, model_slot slot, const char *path) { loaded_model *m = &s->models[slot]; m->info = &k_models[slot]; m->path = xstrdup(path); m->terminal_token_id = -1; m->cls_id = -1; m->sep_id = -1; /* Pick the tokenizer by the files present: a WordPiece vocab.txt marks a * BERT encoder; otherwise the Qwen byte-level BPE vocab.json. The two never * coexist in a model directory, so the probe is unambiguous. */ char wp_path[1024]; snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path); if (access(wp_path, R_OK) == 0) { m->wp_tok = wordpiece_tokenizer_load(path); if (!m->wp_tok) { server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path); return -1; } m->wp_tok_ws = wordpiece_workspace_new(); if (!m->wp_tok_ws) { server_log("embed-server: failed to allocate WordPiece workspace: %s", path); return -1; } /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */ m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]"); m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]"); if (m->cls_id < 0 || m->sep_id < 0) { server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path); return -1; } } else { char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); m->tok = embed_tokenizer_load(vocab_path); if (!m->tok) { server_log("embed-server: failed to load tokenizer: %s", vocab_path); return -1; } m->tok_ws = embed_tokenizer_workspace_new(); if (!m->tok_ws) { server_log("embed-server: failed to allocate tokenizer workspace: %s", path); return -1; } /* The contextual chunk separator is the tokenizer's <|endoftext|>. The * released models keep added special tokens out of vocab.json, so the * family constant is the normal case; a vocab that does define the * token (e.g. small test models) takes precedence so the id stays * inside that model's embedding table. */ int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>"); m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> * suffix - the same token as the separator, not the model's chat * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ m->terminal_token_id = m->context_separator_id; if (m->info->kind == MODEL_KIND_LATE) { int id = embed_tokenizer_token_id(m->tok, "[MASK]"); m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; id = embed_tokenizer_token_id(m->tok, "[Q]"); m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; id = embed_tokenizer_token_id(m->tok, "[D]"); m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; for (const char *p = punct; *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0])); p++) { char text[2] = {*p, '\0'}; int n_ids = 0; int *ids = embed_tokenizer_encode(m->tok, text, &n_ids); if (ids && n_ids == 1) m->late_skip_ids[m->n_late_skip_ids++] = ids[0]; free(ids); } } } #ifdef USE_MLX if (s->use_mlx) { /* MLX streams are thread-local. The inference worker loads the MLX * model so all MLX arrays/streams are created, used, and freed by the * same thread. */ return 0; } #endif #ifdef USE_CUDA if (s->use_cuda) { return 0; } #endif const embed_config_t *config = NULL; int token_dim = 0; if (m->info->kind == MODEL_KIND_LATE) { m->cpu_late_model = embed_late_model_load(path); if (!m->cpu_late_model) { server_log("embed-server: failed to load late model: %s", path); return -1; } m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model); if (!m->cpu_late_ws) { server_log("embed-server: failed to allocate late workspace: %s", path); return -1; } config = embed_late_model_config(m->cpu_late_model); token_dim = embed_late_model_token_dim(m->cpu_late_model); } else { m->cpu_model = embed_model_load(path); if (!m->cpu_model) { server_log("embed-server: failed to load model: %s", path); return -1; } m->cpu_ws = embed_workspace_new(m->cpu_model); if (!m->cpu_ws) { server_log("embed-server: failed to allocate workspace: %s", path); return -1; } config = embed_model_config(m->cpu_model); } if (configure_loaded_model(m, config, token_dim) != 0) { return -1; } if (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size)) { server_log("embed-server: late special-token ids exceed model vocab"); return -1; } return 0; } static void free_models(http_server *s) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; free(m->path); if (m->tok_ws) embed_tokenizer_workspace_free(m->tok_ws); if (m->tok) embed_tokenizer_free(m->tok); if (m->wp_tok_ws) wordpiece_workspace_free(m->wp_tok_ws); if (m->wp_tok) wordpiece_tokenizer_free(m->wp_tok); if (m->cpu_ws) embed_workspace_free(m->cpu_ws); if (m->cpu_model) embed_model_free(m->cpu_model); if (m->cpu_late_ws) embed_late_workspace_free(m->cpu_late_ws); if (m->cpu_late_model) embed_late_model_free(m->cpu_late_model); #ifdef USE_MLX if (m->mlx_ctx) embed_mlx_free(m->mlx_ctx); if (m->mlx_late_ctx) embed_mlx_late_free(m->mlx_late_ctx); #endif #ifdef USE_CUDA if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx); if (m->cuda_late_ctx) embed_cuda_late_free(m->cuda_late_ctx); #endif } } static uint64_t physical_memory_nbytes(void) { #ifdef __APPLE__ uint64_t bytes = 0; size_t len = sizeof(bytes); if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) return bytes; #endif long pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) return 0; return (uint64_t)pages * (uint64_t)page_size; } static int validate_model_specs(const embed_server_config_t *cfg) { int seen[MODEL_COUNT] = {0}; for (int i = 0; i < cfg->n_models; i++) { const embed_server_model_spec_t *spec = &cfg->models[i]; model_slot slot = model_slot_for_id(spec->id); if (slot == MODEL_UNKNOWN) { fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : ""); return -1; } if (!spec->path || !spec->path[0]) { fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); return -1; } if (seen[slot]) { fprintf(stderr, "embed-server: duplicate model id: %s\n", spec->id); return -1; } seen[slot] = 1; } return 0; } static int mlx_memory_preflight(const embed_server_config_t *cfg) { if (!cfg->use_mlx) return 0; uint64_t payload = 0; for (int i = 0; i < cfg->n_models; i++) { multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path); size_t model_payload = 0; if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { multi_safetensors_close(ms); fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path); return -1; } multi_safetensors_close(ms); if ((uint64_t)model_payload > UINT64_MAX - payload) { fprintf(stderr, "embed-server: MLX model payload size overflow\n"); return -1; } payload += (uint64_t)model_payload; } if (payload > UINT64_MAX / EMBED_MLX_RESIDENT_MULTIPLIER) { fprintf(stderr, "embed-server: MLX resident-memory estimate overflow\n"); return -1; } uint64_t estimated = payload * EMBED_MLX_RESIDENT_MULTIPLIER; uint64_t physical = physical_memory_nbytes(); if (physical == 0) { server_log("embed-server: warning: could not determine physical memory; " "skipping MLX memory preflight"); return 0; } double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; uint64_t budget = (uint64_t)((double)physical * utilization); const double gib = 1024.0 * 1024.0 * 1024.0; server_log("embed-server: MLX memory preflight: %.1f GiB tensors, " "%.1f GiB estimated resident, %.1f GiB budget " "(%.2f of physical memory)", (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); if (cfg->mlx_quantize_bits) { server_log("embed-server: MLX %d-bit quantization enabled; preflight " "uses source payload as a conservative peak estimate", cfg->mlx_quantize_bits); } if (estimated <= budget) return 0; server_log("embed-server: refusing MLX model set above the host-memory " "budget"); server_log("embed-server: use BF16 artifacts, load fewer models, or raise " "--memory-utilization (values above 1.0 overcommit)"); return -1; } int embed_run_server(const embed_server_config_t *cfg) { if (!cfg || !cfg->models || cfg->n_models <= 0) { fprintf(stderr, "embed-server: at least one --model MODEL_ID=PATH is required\n"); return 1; } if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) return 1; http_server s; memset(&s, 0, sizeof(s)); s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; s.port = cfg->port > 0 ? cfg->port : 8000; s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; s.max_batch_tokens = cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US : EMBED_SERVER_BATCH_WAIT_US); s.use_mlx = cfg->use_mlx; s.use_cuda = cfg->use_cuda; s.mlx_quantize_bits = cfg->mlx_quantize_bits; s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); else { const char *env_key = getenv("EMBED_API_KEY"); if (env_key && env_key[0]) s.api_key = xstrdup(env_key); } pthread_mutex_init(&s.mu, NULL); pthread_cond_init(&s.cv, NULL); for (int i = 0; i < cfg->n_models; i++) { model_slot slot = model_slot_for_id(cfg->models[i].id); if (slot == MODEL_UNKNOWN) { fprintf(stderr, "embed-server: unknown model id: %s\n", cfg->models[i].id); free_models(&s); return 1; } if (load_one_model(&s, slot, cfg->models[i].path) != 0) { free_models(&s); return 1; } } if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { perror("pipe"); free_models(&s); return 1; } set_nonblock(s.completion_pipe[0]); set_nonblock(s.completion_pipe[1]); set_nonblock(s.signal_pipe[0]); set_nonblock(s.signal_pipe[1]); struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = stop_signal_handler; sigemptyset(&sa.sa_mask); sigaction(SIGINT, &sa, NULL); sigaction(SIGTERM, &sa, NULL); signal(SIGPIPE, SIG_IGN); g_signal_wfd = s.signal_pipe[1]; s.loop = aeCreateEventLoop(EMBED_HTTP_SETSIZE); if (!s.loop) { fprintf(stderr, "embed-server: failed to create event loop\n"); free_models(&s); return 1; } if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { fprintf(stderr, "embed-server: failed to start worker\n"); aeDeleteEventLoop(s.loop); free_models(&s); return 1; } pthread_mutex_lock(&s.mu); while (!s.worker_ready) pthread_cond_wait(&s.cv, &s.mu); int worker_init_rc = s.worker_init_rc; pthread_mutex_unlock(&s.mu); if (worker_init_rc) { pthread_join(s.worker, NULL); aeDeleteEventLoop(s.loop); free_models(&s); free(s.api_key); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; } s.listen_fd = listen_on(s.host, s.port); if (s.listen_fd < 0) { pthread_mutex_lock(&s.mu); s.stopping = 1; pthread_cond_signal(&s.cv); pthread_mutex_unlock(&s.mu); pthread_join(s.worker, NULL); aeDeleteEventLoop(s.loop); free_models(&s); return 1; } aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); aeCreateTimeEvent(s.loop, EMBED_HTTP_SWEEP_MS, timeout_cb, &s, NULL); server_log("embed-server: listening on http://%s:%d", s.host, s.port); aeMain(s.loop); server_log("embed-server: shutdown requested"); if (s.listen_fd >= 0) close(s.listen_fd); client *c = s.clients; while (c) { client *next = c->next; close_client(c); c = next; } pthread_mutex_lock(&s.mu); s.stopping = 1; pthread_cond_signal(&s.cv); pthread_mutex_unlock(&s.mu); pthread_join(s.worker, NULL); aeDeleteEventLoop(s.loop); close(s.completion_pipe[0]); close(s.completion_pipe[1]); close(s.signal_pipe[0]); close(s.signal_pipe[1]); free_models(&s); free(s.api_key); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 0; } /* ==================================================================== * embed-server entry point * ==================================================================== */ #ifndef EMBED_SERVER_TEST static void print_usage(const char *prog) { fprintf(stderr, "Usage: %s --model ID=DIR [options]\n" "\n" "Options:\n" " --model ID=DIR Model to serve (repeatable)\n" #ifdef USE_MLX " --mlx Use Apple MLX GPU backend\n" #endif #ifdef USE_CUDA " --cuda Use NVIDIA CUDA GPU backend\n" #endif " --host HOST Bind host (default: 127.0.0.1)\n" " --port N Bind port (default: 8000)\n" " --api-key K Require Authorization: Bearer K\n" " -b, --batch-size N Max texts or documents per inference batch\n" " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" #ifdef USE_CUDA " (default: CUDA 1000; CPU 0)\n" #else " (default: 0)\n" #endif #ifdef USE_MLX " --memory-utilization F Fraction of physical memory the MLX model-set\n" " preflight may plan for (default: 0.90; values\n" " above 1.0 overcommit)\n" " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" #endif #ifdef USE_CUDA " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" " (default: f32, exact)\n" " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" " f32); bf16 halves weight memory\n" #endif " -t, --threads N CPU threads (default: available cores, cgroup-aware)\n" " -v, --verbose Verbose (-vv for debug)\n" " -V, --version Print version and exit\n" " --build-info Print build details and exit\n" " -h, --help Show this help\n" "\n" "Examples:\n" " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" #ifdef USE_MLX " %s --mlx --mlx-quant-bits 8 \\\n" " --model pplx-embed-v1-4b=./model-4b-bf16\n", prog, prog, prog); #else , prog, prog); #endif } typedef struct { embed_server_model_spec_t *v; int n; int cap; } model_specs_t; static int append_model_spec(model_specs_t *specs, const char *arg) { const char *eq = strchr(arg, '='); if (!eq || eq == arg || !eq[1]) { fprintf(stderr, "--model expects MODEL_ID=DIR\n"); return -1; } if (specs->n == specs->cap) { int new_cap = specs->cap ? specs->cap * 2 : 4; void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); if (!p) return -1; specs->v = p; specs->cap = new_cap; } size_t id_len = (size_t)(eq - arg); char *id = (char *)malloc(id_len + 1); char *path = strdup(eq + 1); if (id) { memcpy(id, arg, id_len); id[id_len] = '\0'; } specs->v[specs->n].id = id; specs->v[specs->n].path = path; if (!specs->v[specs->n].id || !specs->v[specs->n].path) return -1; specs->n++; return 0; } static void free_model_specs(model_specs_t *specs) { if (!specs) return; for (int i = 0; i < specs->n; i++) { free((char *)specs->v[i].id); free((char *)specs->v[i].path); } free(specs->v); } int main(int argc, char *argv[]) { int n_threads = 0; int verbose = 0; int batch_size = 0; int max_batch_tokens = 0; int batch_wait_us = -1; #ifdef USE_MLX int use_mlx = 0; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; double memory_utilization = 0.0; #endif #ifdef USE_CUDA int use_cuda = 0; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; #endif const char *host = "127.0.0.1"; const char *api_key = NULL; int port = 8000; model_specs_t model_specs = {0}; const char *prog = embed_prog_name(argv[0]); int arg = 1; while (arg < argc && argv[arg][0] == '-') { const char *f = argv[arg]; if (!strcmp(f, "-V") || !strcmp(f, "--version")) { embed_print_version(prog); free_model_specs(&model_specs); return 0; } else if (!strcmp(f, "--build-info")) { embed_print_build_info(prog); free_model_specs(&model_specs); return 0; } else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg]) != 0) { free_model_specs(&model_specs); return 1; } } #ifdef USE_MLX else if (!strcmp(f, "--mlx")) { use_mlx = 1; } else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); free_model_specs(&model_specs); return 1; } } else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg]); if (mlx_quantize_group_size <= 0) { fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); free_model_specs(&model_specs); return 1; } } else if (!strcmp(f, "--memory-utilization")) { memory_utilization = atof(argv[++arg]); if (memory_utilization <= 0.0) { fprintf(stderr, "--memory-utilization must be > 0\n"); free_model_specs(&model_specs); return 1; } } #endif #ifdef USE_CUDA else if (!strcmp(f, "--cuda")) { use_cuda = 1; } else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg]; } else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg]; } #endif else if (!strcmp(f, "--host")) { host = argv[++arg]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg]); } else if (!strcmp(f, "--max-batch-tokens")) { max_batch_tokens = atoi(argv[++arg]); if (max_batch_tokens <= 0) { fprintf(stderr, "--max-batch-tokens must be > 0\n"); free_model_specs(&model_specs); return 1; } } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { fprintf(stderr, "--batch-wait-us must be >= 0\n"); free_model_specs(&model_specs); return 1; } } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { n_threads = atoi(argv[++arg]); } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(prog); free_model_specs(&model_specs); return 0; } else { fprintf(stderr, "unknown option: %s\n", f); print_usage(prog); free_model_specs(&model_specs); return 1; } arg++; } embed_verbose = verbose; if (arg < argc) { fprintf(stderr, "unexpected argument: %s\n", argv[arg]); print_usage(prog); free_model_specs(&model_specs); return 1; } if (model_specs.n == 0) { fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); print_usage(prog); free_model_specs(&model_specs); return 1; } #ifdef USE_MLX if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); free_model_specs(&model_specs); return 1; } #endif #ifdef USE_CUDA if (cuda_fast_gemm) { if (!use_cuda) { fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); free_model_specs(&model_specs); return 1; } if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); free_model_specs(&model_specs); return 1; } } if (cuda_weights) { if (!use_cuda) { fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); free_model_specs(&model_specs); return 1; } if (!strcmp(cuda_weights, "bf16")) { embed_cuda_set_weights_bf16(1); } else if (!strcmp(cuda_weights, "f32")) { embed_cuda_set_weights_bf16(0); } else { fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); free_model_specs(&model_specs); return 1; } } #endif #ifdef USE_MLX if (use_mlx) { if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); } else #endif #ifdef USE_CUDA if (use_cuda) { if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); } else #endif { if (n_threads <= 0) n_threads = embed_get_num_cpus(); embed_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } embed_server_config_t scfg = { .models = model_specs.v, .n_models = model_specs.n, .host = host, .port = port, .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE, .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, #ifdef USE_MLX .use_mlx = use_mlx, .mlx_quantize_bits = mlx_quantize_bits, .mlx_quantize_group_size = mlx_quantize_group_size, .memory_utilization = memory_utilization, #endif #ifdef USE_CUDA .use_cuda = use_cuda, #endif .api_key = api_key, }; int rc = embed_run_server(&scfg); free_model_specs(&model_specs); return rc; } #endif /* EMBED_SERVER_TEST */