diff options
Diffstat (limited to 'embed_server.c')
| -rw-r--r-- | embed_server.c | 1300 |
1 files changed, 664 insertions, 636 deletions
diff --git a/embed_server.c b/embed_server.c index 8c7743c..38da125 100644 --- a/embed_server.c +++ b/embed_server.c @@ -47,39 +47,39 @@ #include <cjson/cJSON.h> -#define EMBED_HTTP_IO_BUF 8192 -#define EMBED_HTTP_MAX_HEADER (64u * 1024u) -#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define EMBED_HTTP_MAX_PATH 255u -#define EMBED_HTTP_BACKLOG 128 -#define EMBED_HTTP_SETSIZE 10240 +#define EMBED_HTTP_IO_BUF 8192 +#define EMBED_HTTP_MAX_HEADER (64u * 1024u) +#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define EMBED_HTTP_MAX_PATH 255u +#define EMBED_HTTP_BACKLOG 128 +#define EMBED_HTTP_SETSIZE 10240 #define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 -#define EMBED_HTTP_SWEEP_MS 1000 +#define EMBED_HTTP_SWEEP_MS 1000 -#define EMBED_API_MAX_STANDARD_INPUTS 512 -#define EMBED_API_MAX_CONTEXT_DOCS 512 -#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 +#define EMBED_API_MAX_STANDARD_INPUTS 512 +#define EMBED_API_MAX_CONTEXT_DOCS 512 +#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 #define EMBED_API_MAX_RERANK_DOCUMENTS 1000 -#define EMBED_API_MAX_ITEM_TOKENS 32768 -#define EMBED_API_MAX_TOTAL_TOKENS 120000 -#define EMBED_LATE_QUERY_TOKENS 32 -#define EMBED_LATE_MASK_TOKEN_ID 151642 -#define EMBED_LATE_QUERY_PREFIX_ID 151669 -#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 +#define EMBED_API_MAX_ITEM_TOKENS 32768 +#define EMBED_API_MAX_TOTAL_TOKENS 120000 +#define EMBED_LATE_QUERY_TOKENS 32 +#define EMBED_LATE_MASK_TOKEN_ID 151642 +#define EMBED_LATE_QUERY_PREFIX_ID 151669 +#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ -#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 +#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 #define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 /* Microseconds the worker waits for more requests before dispatching a batch. * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU * busy; MLX and CPU dispatch immediately, where waiting only adds latency. * Override with --batch-wait-us. */ -#define EMBED_SERVER_BATCH_WAIT_US 0 -#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 +#define EMBED_SERVER_BATCH_WAIT_US 0 +#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 #define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 -#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 -#define EMBED_MLX_RESIDENT_MULTIPLIER 2 +#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 +#define EMBED_MLX_RESIDENT_MULTIPLIER 2 typedef struct { char *ptr; @@ -94,19 +94,22 @@ static void die_oom(void) { static void *xmalloc(size_t n) { void *p = malloc(n ? n : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } static void *xcalloc(size_t n, size_t sz) { void *p = calloc(n ? n : 1, sz ? sz : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } static void *xrealloc(void *p, size_t n) { p = realloc(p, n ? n : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } @@ -142,9 +145,7 @@ static uint64_t nstime(void) { return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u; } -static double ns_to_ms(uint64_t ns) { - return (double)ns / 1000000.0; -} +static double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; } static void server_log(const char *fmt, ...) { char ts[32]; @@ -161,12 +162,15 @@ static void server_log(const char *fmt, ...) { } static void sbuf_reserve(sbuf *b, size_t add) { - if (add > SIZE_MAX - b->len - 1) die_oom(); + if (add > SIZE_MAX - b->len - 1) + die_oom(); size_t need = b->len + add + 1; - if (need <= b->cap) return; + if (need <= b->cap) + return; size_t cap = b->cap ? b->cap * 2 : 256; while (cap < need) { - if (cap > SIZE_MAX / 2) die_oom(); + if (cap > SIZE_MAX / 2) + die_oom(); cap *= 2; } b->ptr = xrealloc(b->ptr, cap); @@ -185,7 +189,8 @@ static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } static void sbuf_clear(sbuf *b) { b->len = 0; - if (b->ptr) b->ptr[0] = '\0'; + if (b->ptr) + b->ptr[0] = '\0'; } static void sbuf_printf(sbuf *b, const char *fmt, ...) { @@ -194,11 +199,13 @@ static void sbuf_printf(sbuf *b, const char *fmt, ...) { va_copy(ap2, ap); int n = vsnprintf(NULL, 0, fmt, ap); va_end(ap); - if (n < 0) die_oom(); + if (n < 0) + die_oom(); sbuf_reserve(b, (size_t)n); int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2); va_end(ap2); - if (n2 < 0 || n2 != n) die_oom(); + if (n2 < 0 || n2 != n) + die_oom(); b->len += (size_t)n; } @@ -220,11 +227,7 @@ typedef enum { MODEL_UNKNOWN } model_slot; -typedef enum { - MODEL_KIND_STANDARD, - MODEL_KIND_CONTEXTUAL, - MODEL_KIND_LATE -} model_kind; +typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; /* Which embedding API a model speaks. pplx-embed models follow the Perplexity * API: the canonical output is the tanh int8 quantization, so the default @@ -251,28 +254,30 @@ typedef struct { } model_info; static const model_info k_models[] = { - {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, }; static model_slot model_slot_for_id(const char *id) { - if (!id) return MODEL_UNKNOWN; + if (!id) + return MODEL_UNKNOWN; for (int i = 0; i < MODEL_COUNT; i++) { - if (!strcmp(id, k_models[i].id)) return (model_slot)i; + if (!strcmp(id, k_models[i].id)) + return (model_slot)i; } return MODEL_UNKNOWN; } @@ -398,7 +403,8 @@ static volatile sig_atomic_t g_signal_wfd = -1; static void stop_signal_handler(int sig) { (void)sig; - if (g_stop_requested) _exit(130); + if (g_stop_requested) + _exit(130); int saved = errno; g_stop_requested = 1; int fd = (int)g_signal_wfd; @@ -411,7 +417,8 @@ static void stop_signal_handler(int sig) { static int set_nonblock(int fd) { int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0) return -1; + if (flags < 0) + return -1; return fcntl(fd, F_SETFL, flags | O_NONBLOCK); } @@ -422,7 +429,8 @@ static void http_request_free(http_request *r) { } static void client_free(client *c) { - if (!c) return; + if (!c) + return; http_request_free(&c->req); sbuf_free(&c->in); sbuf_free(&c->out); @@ -438,34 +446,45 @@ static void client_incref(client *c) { static void client_decref(client *c) { int free_now = 0; pthread_mutex_lock(&c->srv->mu); - if (--c->refcount == 0) free_now = 1; + if (--c->refcount == 0) + free_now = 1; pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); + if (free_now) + client_free(c); } static void client_decref_n(client *c, int n) { int free_now = 0; - if (n <= 0) return; + if (n <= 0) + return; pthread_mutex_lock(&c->srv->mu); c->refcount -= n; - if (c->refcount <= 0) free_now = 1; + if (c->refcount <= 0) + free_now = 1; pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); + if (free_now) + client_free(c); } static void client_unlink(client *c) { http_server *s = c->srv; - if (!c->linked) return; - if (c->prev) c->prev->next = c->next; - else s->clients = c->next; - if (c->next) c->next->prev = c->prev; + if (!c->linked) + return; + if (c->prev) + c->prev->next = c->next; + else + s->clients = c->next; + if (c->next) + c->next->prev = c->prev; c->prev = c->next = NULL; c->linked = 0; - if (s->n_clients > 0) s->n_clients--; + if (s->n_clients > 0) + s->n_clients--; } static int close_client_unlink(client *c) { - if (!c || c->cancelled) return 0; + if (!c || c->cancelled) + return 0; c->cancelled = 1; if (c->fd >= 0) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); @@ -478,40 +497,52 @@ static int close_client_unlink(client *c) { } static void close_client(client *c) { - if (!close_client_unlink(c)) return; + if (!close_client_unlink(c)) + return; client_decref(c); } -static void append_http_response_ex(client *c, int status, const char *ctype, +static void append_http_response_ex(client *c, + int status, + const char *ctype, const char *extra_headers, - const char *body, size_t body_len) { + const char *body, + size_t body_len) { const char *reason = "OK"; - if (status == 204) reason = "No Content"; - else if (status == 400) reason = "Bad Request"; - else if (status == 401) reason = "Unauthorized"; - else if (status == 404) reason = "Not Found"; - else if (status == 405) reason = "Method Not Allowed"; - else if (status == 422) reason = "Unprocessable Entity"; - else if (status == 503) reason = "Service Unavailable"; - else if (status >= 500) reason = "Internal Server Error"; + if (status == 204) + reason = "No Content"; + else if (status == 400) + reason = "Bad Request"; + else if (status == 401) + reason = "Unauthorized"; + else if (status == 404) + reason = "Not Found"; + else if (status == 405) + reason = "Method Not Allowed"; + else if (status == 422) + reason = "Unprocessable Entity"; + else if (status == 503) + reason = "Service Unavailable"; + else if (status >= 500) + reason = "Internal Server Error"; sbuf_printf(&c->out, "HTTP/1.1 %d %s\r\n" "Content-Length: %zu\r\n" "Connection: %s\r\n", - status, reason, body_len, - c->close_after_write ? "close" : "keep-alive"); + status, reason, body_len, c->close_after_write ? "close" : "keep-alive"); if (ctype) sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); if (extra_headers) sbuf_puts(&c->out, extra_headers); sbuf_puts(&c->out, "\r\n"); - if (body_len) sbuf_append(&c->out, body, body_len); + if (body_len) + sbuf_append(&c->out, body, body_len); } -static void append_http_response(client *c, int status, const char *ctype, - const char *body, size_t body_len) { +static void +append_http_response(client *c, int status, const char *ctype, const char *body, size_t body_len) { append_http_response_ex(c, status, ctype, NULL, body, body_len); } @@ -534,13 +565,27 @@ static void append_json_string(sbuf *b, const char *s) { sbuf_putc(b, '"'); for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) { switch (*p) { - case '"': sbuf_puts(b, "\\\""); break; - case '\\': sbuf_puts(b, "\\\\"); break; - case '\b': sbuf_puts(b, "\\b"); break; - case '\f': sbuf_puts(b, "\\f"); break; - case '\n': sbuf_puts(b, "\\n"); break; - case '\r': sbuf_puts(b, "\\r"); break; - case '\t': sbuf_puts(b, "\\t"); break; + case '"': + sbuf_puts(b, "\\\""); + break; + case '\\': + sbuf_puts(b, "\\\\"); + break; + case '\b': + sbuf_puts(b, "\\b"); + break; + case '\f': + sbuf_puts(b, "\\f"); + break; + case '\n': + sbuf_puts(b, "\\n"); + break; + case '\r': + sbuf_puts(b, "\\r"); + break; + case '\t': + sbuf_puts(b, "\\t"); + break; default: if (*p < 0x20) sbuf_printf(b, "\\u%04x", (unsigned)*p); @@ -566,8 +611,10 @@ static void job_set_error(job *j, int status, const char *message, const char *t } static bool auth_ok(http_server *s, const char *auth) { - if (!s->api_key || !s->api_key[0]) return true; - if (!auth) return false; + if (!s->api_key || !s->api_key[0]) + return true; + if (!auth) + return false; const char prefix[] = "Bearer "; return !strncmp(auth, prefix, sizeof(prefix) - 1) && !strcmp(auth + sizeof(prefix) - 1, s->api_key); @@ -575,12 +622,12 @@ static bool auth_ok(http_server *s, const char *auth) { static ssize_t find_header_end(const char *p, size_t n) { for (size_t i = 3; i < n; i++) { - if (p[i - 3] == '\r' && p[i - 2] == '\n' && - p[i - 1] == '\r' && p[i] == '\n') + if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n') return (ssize_t)(i + 1); } for (size_t i = 1; i < n; i++) { - if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); + if (p[i - 1] == '\n' && p[i] == '\n') + return (ssize_t)(i + 1); } return -1; } @@ -590,18 +637,22 @@ static char *header_value_dup(const char *h, size_t n, const char *name) { const char *p = h, *end = h + n; while (p < end) { const char *line = p; - while (p < end && *p != '\n') p++; + while (p < end && *p != '\n') + p++; size_t len = (size_t)(p - line); - if (len && line[len - 1] == '\r') len--; - if (len > name_len && !strncasecmp(line, name, name_len) && - line[name_len] == ':') { + if (len && line[len - 1] == '\r') + len--; + if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') { const char *v = line + name_len + 1; - while (v < line + len && isspace((unsigned char)*v)) v++; + while (v < line + len && isspace((unsigned char)*v)) + v++; const char *e = line + len; - while (e > v && isspace((unsigned char)e[-1])) e--; + while (e > v && isspace((unsigned char)e[-1])) + e--; return xstrndup(v, (size_t)(e - v)); } - if (p < end) p++; + if (p < end) + p++; } return NULL; } @@ -610,13 +661,15 @@ static int header_has_token(const char *value, const char *token) { size_t token_len = strlen(token); const char *p = value; while (p && *p) { - while (*p == ',' || isspace((unsigned char)*p)) p++; + while (*p == ',' || isspace((unsigned char)*p)) + p++; const char *start = p; - while (*p && *p != ',') p++; + while (*p && *p != ',') + p++; const char *end = p; - while (end > start && isspace((unsigned char)end[-1])) end--; - if ((size_t)(end - start) == token_len && - !strncasecmp(start, token, token_len)) + while (end > start && isspace((unsigned char)end[-1])) + end--; + if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len)) return 1; } return 0; @@ -630,14 +683,14 @@ static int parse_request_headers(client *c) { i++; } line[i] = '\0'; - int fields = sscanf(line, "%7s %255s %15s", - c->req.method, c->req.path, c->req.version); + int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version); if (fields < 2) return -1; if (fields < 3) snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); char *q = strchr(c->req.path, '?'); - if (q) *q = '\0'; + if (q) + *q = '\0'; c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); @@ -669,15 +722,19 @@ static int complete_request(client *c) { if (!c->header_done) { ssize_t hend = find_header_end(c->in.ptr, c->in.len); if (hend < 0) { - if (c->in.len > EMBED_HTTP_MAX_HEADER) return -1; + if (c->in.len > EMBED_HTTP_MAX_HEADER) + return -1; return 0; } c->header_done = 1; c->header_len = (size_t)hend; - if (parse_request_headers(c) != 0) return -1; + if (parse_request_headers(c) != 0) + return -1; } - if (c->content_length > EMBED_HTTP_MAX_BODY) return -1; - if (c->in.len < c->header_len + c->content_length) return 0; + if (c->content_length > EMBED_HTTP_MAX_BODY) + return -1; + if (c->in.len < c->header_len + c->content_length) + return 0; c->req.body_len = c->content_length; c->req.body = xmalloc(c->content_length + 1); memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); @@ -687,23 +744,25 @@ static int complete_request(client *c) { static void send_bad_http_request(client *c) { size_t len; - char *body = json_error_body("bad HTTP request", "invalid_request_error", - &len); + char *body = json_error_body("bad HTTP request", "invalid_request_error", &len); c->close_after_write = 1; append_http_response(c, 400, "application/json", body, len); free(body); aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); } static void reset_client_for_next_request(client *c) { size_t consumed = c->header_len + c->content_length; - if (consumed > c->in.len) consumed = c->in.len; + if (consumed > c->in.len) + consumed = c->in.len; size_t remain = c->in.len - consumed; if (remain) memmove(c->in.ptr, c->in.ptr + consumed, remain); c->in.len = remain; - if (c->in.ptr) c->in.ptr[remain] = '\0'; + if (c->in.ptr) + c->in.ptr[remain] = '\0'; http_request_free(&c->req); sbuf_clear(&c->out); @@ -716,7 +775,8 @@ static void reset_client_for_next_request(client *c) { static void arm_next_request(client *c) { reset_client_for_next_request(c); - if (c->cancelled || c->fd < 0) return; + if (c->cancelled || c->fd < 0) + return; if (c->in.len) { int done = complete_request(c); @@ -735,7 +795,8 @@ static void arm_next_request(client *c) { } static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; client *c = clientData; while (c->sent < c->out.len) { ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); @@ -744,26 +805,33 @@ static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { c->last_active_ms = mstime(); continue; } - if (n < 0 && errno == EINTR) continue; - if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; + if (n < 0 && errno == EINTR) + continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + return; close_client(c); return; } aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); - if (c->close_after_write) close_client(c); - else arm_next_request(c); + if (c->close_after_write) + close_client(c); + else + arm_next_request(c); } static int queue_write(client *c) { - if (!c || c->cancelled || c->fd < 0) return -1; + if (!c || c->cancelled || c->fd < 0) + return -1; return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); } static void enqueue_done(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); - if (s->done_tail) s->done_tail->next = j; - else s->done_head = j; + if (s->done_tail) + s->done_tail->next = j; + else + s->done_head = j; s->done_tail = j; pthread_mutex_unlock(&s->mu); const unsigned char byte = 1; @@ -774,16 +842,17 @@ static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { s->job_head = j->next; - if (!s->job_head) s->job_tail = NULL; + if (!s->job_head) + s->job_tail = NULL; j->next = NULL; } return j; } -static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, - uint64_t deadline_ns) { +static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) { uint64_t now_ns = nstime(); - if (now_ns >= deadline_ns) return ETIMEDOUT; + if (now_ns >= deadline_ns) + return ETIMEDOUT; uint64_t remaining_ns = deadline_ns - now_ns; struct timespec abs; @@ -803,7 +872,8 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, } static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { - if (max_jobs <= 0) return 0; + if (max_jobs <= 0) + return 0; pthread_mutex_lock(&s->mu); while (!s->job_head && !s->stopping) @@ -817,9 +887,8 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { int n = 1; jobs[0] = first; - int batchable = s->batch_size > 1 && - (!strcmp(first->path, "/v1/embeddings") || - !strcmp(first->path, "/v1/contextualizedembeddings")); + int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") || + !strcmp(first->path, "/v1/contextualizedembeddings")); if (!batchable) { pthread_mutex_unlock(&s->mu); return n; @@ -828,12 +897,13 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { /* Exact item and token budgets are applied after worker-side tokenization; * this queue stage only defines which arrival window may be grouped. */ uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; - uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns - ? UINT64_MAX : first->created_ns + wait_ns; + uint64_t deadline_ns = + first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns; while (n < max_jobs) { while (!s->job_head && !s->stopping && wait_ns > 0) { int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); - if (rc == ETIMEDOUT) break; + if (rc == ETIMEDOUT) + break; } if (!s->job_head) break; @@ -850,8 +920,10 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { static void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); - if (s->job_tail) s->job_tail->next = j; - else s->job_head = j; + if (s->job_tail) + s->job_tail->next = j; + else + s->job_head = j; s->job_tail = j; pthread_cond_signal(&s->cv); pthread_mutex_unlock(&s->mu); @@ -862,7 +934,8 @@ static job *pop_done(http_server *s) { job *j = s->done_head; if (j) { s->done_head = j->next; - if (!s->done_head) s->done_tail = NULL; + if (!s->done_head) + s->done_tail = NULL; j->next = NULL; } pthread_mutex_unlock(&s->mu); @@ -870,7 +943,8 @@ static job *pop_done(http_server *s) { } static void job_free(job *j) { - if (!j) return; + if (!j) + return; free(j->body); free(j->auth); free(j->extra_headers); @@ -879,18 +953,19 @@ static void job_free(job *j) { } static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; http_server *s = clientData; unsigned char tmp[128]; - while (read(fd, tmp, sizeof(tmp)) > 0) {} + while (read(fd, tmp, sizeof(tmp)) > 0) { + } job *j; while ((j = pop_done(s)) != NULL) { client *c = j->c; int refs_to_drop = 1; /* job reference */ if (!c->cancelled) { - append_http_response_ex(c, j->status, j->content_type, - j->extra_headers, + append_http_response_ex(c, j->status, j->content_type, j->extra_headers, j->response ? j->response : "", j->response ? j->response_len : 0); if (queue_write(c) != AE_OK) @@ -903,8 +978,7 @@ static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) static bool route_is_inference(const char *method, const char *path) { return !strcmp(method, "POST") && - (!strcmp(path, "/v1/embeddings") || - !strcmp(path, "/v1/contextualizedembeddings") || + (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") || !strcmp(path, "/v1/rerank")); } @@ -913,7 +987,8 @@ static void dispatch_request(client *c) { if (!strcmp(c->req.method, "OPTIONS")) { append_http_response(c, 204, NULL, "", 0); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } @@ -922,17 +997,19 @@ static void dispatch_request(client *c) { char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); append_http_response(c, 404, "application/json", body, len); free(body); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } if (!auth_ok(c->srv, c->req.auth)) { size_t len; - char *body = json_error_body("missing or invalid bearer token", - "authentication_error", &len); + char *body = + json_error_body("missing or invalid bearer token", "authentication_error", &len); append_http_response(c, 401, "application/json", body, len); free(body); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } @@ -950,7 +1027,8 @@ static void dispatch_request(client *c) { } static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; client *c = clientData; char tmp[EMBED_HTTP_IO_BUF]; for (;;) { @@ -973,23 +1051,28 @@ static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { close_client(c); return; } - if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EWOULDBLOCK) return; + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; close_client(c); return; } } static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; http_server *s = clientData; char cip[64]; int cport = 0; for (;;) { int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); if (cfd == ANET_ERR) { - if (errno == EAGAIN || errno == EWOULDBLOCK) return; - if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + if (errno == EINTR) + continue; return; } set_nonblock(cfd); @@ -1000,7 +1083,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { c->last_active_ms = mstime(); c->linked = 1; c->next = s->clients; - if (s->clients) s->clients->prev = c; + if (s->clients) + s->clients->prev = c; s->clients = c; s->n_clients++; if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) @@ -1009,7 +1093,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { } static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { - (void)loop; (void)id; + (void)loop; + (void)id; http_server *s = clientData; uint64_t now = mstime(); client *c = s->clients; @@ -1027,15 +1112,16 @@ static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { } static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)fd; (void)mask; + (void)fd; + (void)mask; http_server *s = clientData; unsigned char tmp[128]; - while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {} + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { + } aeStop(loop); } -static bool ve_add(cJSON *detail, const char *loc_json, - const char *msg, const char *type) { +static bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) { cJSON *obj = cJSON_CreateObject(); cJSON *loc = cJSON_Parse(loc_json); if (!obj || !loc) { @@ -1067,21 +1153,19 @@ static void job_set_422(job *j, cJSON *detail) { } static void job_set_timing_header(job *j) { - if (!j || j->extra_headers || !j->created_ns) return; + if (!j || j->extra_headers || !j->created_ns) + return; uint64_t done_ns = nstime(); - uint64_t queue_ns = j->started_ns > j->created_ns - ? j->started_ns - j->created_ns : 0; - uint64_t worker_ns = j->started_ns && done_ns > j->started_ns - ? done_ns - j->started_ns : 0; + uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0; + uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0; char buf[384]; int n = snprintf(buf, sizeof(buf), - "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " - "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " - "worker;dur=%.3f\r\n", - ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), - ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns), - ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); + "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " + "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " + "worker;dur=%.3f\r\n", + ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns), + ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); if (n > 0 && (size_t)n < sizeof(buf)) j->extra_headers = xstrdup(buf); } @@ -1098,7 +1182,8 @@ static cJSON *parse_json_body(job *j, cJSON *detail) { return NULL; } const char *body_end = j->body + j->body_len; - while (end && end < body_end && isspace((unsigned char)*end)) end++; + while (end && end < body_end && isspace((unsigned char)*end)) + end++; if (!end || end != body_end) { ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); cJSON_Delete(root); @@ -1113,30 +1198,30 @@ static cJSON *parse_json_body(job *j, cJSON *detail) { } static bool cjson_is_integer(cJSON *item) { - if (!cJSON_IsNumber(item)) return false; + if (!cJSON_IsNumber(item)) + return false; double d = item->valuedouble; int i = item->valueint; return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; } -static const char *encoding_from_root(cJSON *root, cJSON *detail, - embedding_api_t api) { +static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) { /* Default and accepted encodings follow the model's API family: Perplexity * defaults to base64_int8 and accepts {base64_int8, base64_binary, float}; * OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */ const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8"; cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); - if (!encoding) return dflt; + if (!encoding) + return dflt; if (!cJSON_IsString(encoding) || !encoding->valuestring) { - ve_add(detail, "[\"body\",\"encoding_format\"]", - "encoding_format must be a string", "type_error.string"); + ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string", + "type_error.string"); return dflt; } const char *v = encoding->valuestring; - int ok = api == EMBED_API_OPENAI - ? (!strcmp(v, "float") || !strcmp(v, "base64")) - : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || - !strcmp(v, "float")); + int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64")) + : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || + !strcmp(v, "float")); if (!ok) { ve_add(detail, "[\"body\",\"encoding_format\"]", api == EMBED_API_OPENAI @@ -1149,8 +1234,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail, return v; } -static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, - int max_dim, const char *encoding) { +static int +dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) { int dims = max_dim; cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); if (dimensions) { @@ -1161,31 +1246,30 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, dims = dimensions->valueint; if (dims < min_dim || dims > max_dim) { char msg[96]; - snprintf(msg, sizeof(msg), - "dimensions must be between %d and %d", - min_dim, max_dim); + snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, + max_dim); ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); } } } if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) ve_add(detail, "[\"body\",\"dimensions\"]", - "dimensions must be divisible by 8 for base64_binary", - "value_error.divisible"); + "dimensions must be divisible by 8 for base64_binary", "value_error.divisible"); return dims; } static char *base64_encode(const unsigned char *src, size_t n) { - static const char tbl[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; size_t out_len = ((n + 2) / 3) * 4; char *out = xmalloc(out_len + 1); size_t o = 0; for (size_t i = 0; i < n; i += 3) { unsigned v = (unsigned)src[i] << 16; int remain = (int)(n - i); - if (remain > 1) v |= (unsigned)src[i + 1] << 8; - if (remain > 2) v |= (unsigned)src[i + 2]; + if (remain > 1) + v |= (unsigned)src[i + 1] << 8; + if (remain > 2) + v |= (unsigned)src[i + 2]; out[o++] = tbl[(v >> 18) & 63]; out[o++] = tbl[(v >> 12) & 63]; out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '='; @@ -1197,8 +1281,10 @@ static char *base64_encode(const unsigned char *src, size_t n) { static signed char quantize_int8_tanh(float x) { int v = (int)lrintf(tanhf(x) * 127.0f); - if (v > 127) v = 127; - if (v < -128) v = -128; + if (v > 127) + v = 127; + if (v < -128) + v = -128; return (signed char)v; } @@ -1295,54 +1381,62 @@ typedef struct { } rerank_request; static void free_token_bufs(token_buf *t, int n) { - if (!t) return; - for (int i = 0; i < n; i++) free(t[i].ids); + if (!t) + return; + for (int i = 0; i < n; i++) + free(t[i].ids); } static void embedding_request_free(embedding_request *r) { - if (!r) return; + if (!r) + return; free_token_bufs(r->tokens, r->n_inputs); free(r->tokens); free(r->inputs); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void contextual_request_free(contextual_request *r) { - if (!r) return; + if (!r) + return; for (int i = 0; i < r->n_docs; i++) { free(r->docs[i].ids); free(r->docs[i].spans); } free(r->docs); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void late_tokens_free(late_tokens *t) { - if (!t) return; + if (!t) + return; free(t->ids); free(t->keep); memset(t, 0, sizeof(*t)); } static void rerank_request_free(rerank_request *r) { - if (!r) return; + if (!r) + return; late_tokens_free(&r->query); for (int i = 0; i < r->n_documents; i++) late_tokens_free(&r->documents[i]); free(r->documents); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } -static int tokenize_one(loaded_model *m, job *j, const char *text, - token_buf *out) { +static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); uint64_t t0 = nstime(); - out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, - text, &out->n_tokens); - if (j) j->tokenize_ns += nstime() - t0; + out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); + if (j) + j->tokenize_ns += nstime() - t0; if (!out->ids || out->n_tokens <= 0) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1354,8 +1448,7 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, memset(out, 0, sizeof(*out)); return -1; } - int *ids = realloc( - out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); + int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); if (!ids) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1369,15 +1462,17 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, static int late_id_is_skipped(const loaded_model *m, int id) { for (int i = 0; i < m->n_late_skip_ids; i++) { - if (m->late_skip_ids[i] == id) return 1; + if (m->late_skip_ids[i] == id) + return 1; } return 0; } -static int tokenize_late_text(loaded_model *m, job *j, const char *text, - int is_query, late_tokens *out) { +static int +tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { token_buf raw = {0}; - if (tokenize_one(m, j, text, &raw) != 0) return -1; + if (tokenize_one(m, j, text, &raw) != 0) + return -1; int raw_tokens = raw.n_tokens; int target = raw_tokens + 1; @@ -1386,11 +1481,9 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text, out->ids = xmalloc((size_t)target * sizeof(*out->ids)); out->ids[0] = raw.ids[0]; - out->ids[1] = is_query ? m->late_query_prefix_id - : m->late_document_prefix_id; + out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id; if (raw_tokens > 1) { - memcpy(out->ids + 2, raw.ids + 1, - (size_t)(raw_tokens - 1) * sizeof(*out->ids)); + memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids)); } out->n_tokens = raw_tokens + 1; if (is_query) { @@ -1408,8 +1501,7 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text, return out->n_keep > 0 ? 0 : -1; } -static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, - int batch, float *out) { +static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out); @@ -1423,7 +1515,8 @@ static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, static int model_embed_spans_batch(loaded_model *m, const embed_context_input_t *inputs, - int batch, float *out) { + int batch, + float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out); @@ -1432,8 +1525,7 @@ static int model_embed_spans_batch(loaded_model *m, if (m->cuda_ctx) return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out); #endif - return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, - out); + return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } static int model_uses_dense_batches(const loaded_model *m) { @@ -1447,10 +1539,10 @@ static int model_uses_dense_batches(const loaded_model *m) { /* Inputs are sorted by length before chunking. CPU packs real tokens while MLX * pads each dense row to the longest input in the chunk. */ -static int inference_batch_accepts_input(const loaded_model *m, int batch, - int packed_tokens, int next_tokens, - int max_batch_tokens) { - if (batch == 0) return 1; +static int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { + if (batch == 0) + return 1; if (model_uses_dense_batches(m)) return next_tokens <= max_batch_tokens / (batch + 1); return next_tokens <= max_batch_tokens - packed_tokens; @@ -1466,17 +1558,14 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) { * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every * other encoding renders a base64 string via encode_embedding(). */ -static void append_embedding_value(sbuf *b, int index, const float *emb, - int dims, const char *encoding, - embedding_api_t api) { +static void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { if (!strcmp(encoding, "float")) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", - index); + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); for (int i = 0; i < dims; i++) { - if (i) sbuf_putc(b, ','); - float v = api == EMBED_API_OPENAI - ? emb[i] - : (float)quantize_int8_tanh(emb[i]) / 128.0f; + if (i) + sbuf_putc(b, ','); + float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f; sbuf_printf(b, "%.9g", (double)v); } sbuf_puts(b, "]}"); @@ -1500,12 +1589,10 @@ static int render_embedding_response(embedding_request *r, const float *embs) { const float *render_embs = embs; float *truncated = NULL; if (r->model->renormalize_truncated && r->dims < full_dim) { - truncated = xmalloc( - (size_t)r->n_inputs * r->dims * sizeof(*truncated)); + truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated)); for (int i = 0; i < r->n_inputs; i++) { float *dst = truncated + (size_t)i * r->dims; - memcpy(dst, embs + (size_t)i * full_dim, - (size_t)r->dims * sizeof(*dst)); + memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); if (embed_l2_normalize(dst, r->dims) != 0) { free(truncated); return -1; @@ -1519,25 +1606,23 @@ static int render_embedding_response(embedding_request *r, const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); for (int i = 0; i < r->n_inputs; i++) { - if (i) sbuf_putc(&b, ','); - append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, - r->dims, r->encoding, api); + if (i) + sbuf_putc(&b, ','); + append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, + api); } sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); set_response_from_buf(r->j, &b); free(truncated); return 0; } -static int embedding_request_compatible(const embedding_request *a, - const embedding_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && +static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } @@ -1549,21 +1634,27 @@ typedef struct { static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; const embedding_batch_item *ib = b; - if (ia->input.n_tokens < ib->input.n_tokens) return -1; - if (ia->input.n_tokens > ib->input.n_tokens) return 1; + if (ia->input.n_tokens < ib->input.n_tokens) + return -1; + if (ia->input.n_tokens > ib->input.n_tokens) + return 1; return ia->output_index - ib->output_index; } static int embedding_request_fits_group(const embedding_request *r, - int group_inputs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_inputs == 0) return 1; + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_inputs == 0) + return 1; return r->n_inputs <= max_batch - group_inputs && r->total_tokens <= max_batch_tokens - group_tokens; } static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { - if (n_reqs <= 0) return; + if (n_reqs <= 0) + return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_inputs = 0; @@ -1583,9 +1674,9 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) } qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_inputs; - if (max_batch > total_inputs) max_batch = total_inputs; + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) + max_batch = total_inputs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); @@ -1597,8 +1688,7 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) int tokens = 0; while (start + cur < total_inputs && cur < max_batch) { int n_tokens = items[start + cur].input.n_tokens; - if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, - max_batch_tokens)) + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) break; inputs[cur] = items[start + cur].input; tokens += n_tokens; @@ -1612,8 +1702,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) break; } for (int i = 0; i < cur; i++) - memcpy(embs + (size_t)items[start + i].output_index * dim, - batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); + memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, + (size_t)dim * sizeof(float)); start += cur; } @@ -1624,10 +1714,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); } else { uint64_t t0 = nstime(); - if (render_embedding_response( - reqs[i], embs + (size_t)pos * dim) != 0) - job_set_error(reqs[i]->j, 500, "embedding normalization failed", - "server_error"); + if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) + job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); reqs[i]->j->encode_ns += nstime() - t0; } pos += reqs[i]->n_inputs; @@ -1639,41 +1727,40 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) free(items); } -static void render_contextual_response(contextual_request *r, - const float *embs) { +static void render_contextual_response(contextual_request *r, const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); int span_offset = 0; for (int di = 0; di < r->n_docs; di++) { - if (di) sbuf_putc(&b, ','); + if (di) + sbuf_putc(&b, ','); sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); for (int ci = 0; ci < r->docs[di].n_spans; ci++) { - if (ci) sbuf_putc(&b, ','); + if (ci) + sbuf_putc(&b, ','); const float *emb = embs + (size_t)span_offset * r->model->info->dim; - append_embedding_value(&b, ci, emb, r->dims, r->encoding, - r->model->info->api); + append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api); span_offset++; } sbuf_puts(&b, "]}"); } - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); + sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); set_response_from_buf(r->j, &b); } -static int contextual_request_compatible(const contextual_request *a, - const contextual_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && +static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } static int contextual_request_fits_group(const contextual_request *r, - int group_docs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_docs == 0) return 1; + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_docs == 0) + return 1; return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; } @@ -1687,14 +1774,16 @@ typedef struct { static int contextual_batch_item_cmp(const void *a, const void *b) { const contextual_batch_item *ia = a; const contextual_batch_item *ib = b; - if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; - if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) + return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) + return 1; return ia->order - ib->order; } -static void execute_contextual_request_list(contextual_request **reqs, - int n_reqs) { - if (n_reqs <= 0) return; +static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_docs = 0; @@ -1703,8 +1792,7 @@ static void execute_contextual_request_list(contextual_request **reqs, if (total_docs > INT_MAX - reqs[i]->n_docs || total_chunks > INT_MAX - reqs[i]->total_chunks) { for (int k = 0; k < n_reqs; k++) - job_set_error(reqs[k]->j, 500, "contextual batch is too large", - "server_error"); + job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); return; } total_docs += reqs[i]->n_docs; @@ -1712,13 +1800,11 @@ static void execute_contextual_request_list(contextual_request **reqs, } if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { for (int i = 0; i < n_reqs; i++) - job_set_error(reqs[i]->j, 500, "contextual batch is too large", - "server_error"); + job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); return; } - contextual_batch_item *items = - xmalloc((size_t)total_docs * sizeof(*items)); + contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); int pos = 0; int span_offset = 0; @@ -1737,12 +1823,11 @@ static void execute_contextual_request_list(contextual_request **reqs, } qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_docs; - if (max_batch > total_docs) max_batch = total_docs; + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) + max_batch = total_docs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_context_input_t *inputs = - xmalloc((size_t)max_batch * sizeof(*inputs)); + embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc(sizeof(*batch_embs)); size_t batch_emb_cap = 0; @@ -1754,8 +1839,7 @@ static void execute_contextual_request_list(contextual_request **reqs, int chunks = 0; while (start + cur < total_docs && cur < max_batch) { const embed_context_input_t *input = &items[start + cur].input; - if (!inference_batch_accepts_input(m, cur, tokens, - input->input.n_tokens, + if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, max_batch_tokens)) break; inputs[cur] = *input; @@ -1779,8 +1863,7 @@ static void execute_contextual_request_list(contextual_request **reqs, for (int i = 0; i < cur; i++) { int n_spans = inputs[i].n_spans; memcpy(embs + (size_t)items[start + i].output_span_index * dim, - batch_embs + (size_t)batch_span * dim, - (size_t)n_spans * dim * sizeof(*embs)); + batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); batch_span += n_spans; } start += cur; @@ -1790,12 +1873,10 @@ static void execute_contextual_request_list(contextual_request **reqs, for (int i = 0; i < n_reqs; i++) { reqs[i]->j->infer_ns += infer_ns; if (failed) { - job_set_error(reqs[i]->j, 500, "contextual embedding failed", - "server_error"); + job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); } else { uint64_t t0 = nstime(); - render_contextual_response(reqs[i], - embs + (size_t)span_offset * dim); + render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); reqs[i]->j->encode_ns += nstime() - t0; } span_offset += reqs[i]->total_chunks; @@ -1807,9 +1888,13 @@ static void execute_contextual_request_list(contextual_request **reqs, free(items); } -static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, - loaded_model **out_model, int *out_dims, - const char **out_encoding, http_server *s) { +static int validate_common(cJSON *root, + cJSON *detail, + model_kind expected_kind, + loaded_model **out_model, + int *out_dims, + const char **out_encoding, + http_server *s) { cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); const char *model_id = NULL; if (!model_item) { @@ -1832,8 +1917,7 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, } } model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; - embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api - : EMBED_API_PERPLEXITY; + embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; const char *enc = encoding_from_root(root, detail, api); int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; @@ -1841,11 +1925,12 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, *out_encoding = enc; *out_dims = dims; return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && - !s->models[model_slot_for_id(model_id)].info ? 503 : 0; + !s->models[model_slot_for_id(model_id)].info + ? 503 + : 0; } -static void prepare_embedding_request(job *j, cJSON *root, http_server *s, - embedding_request *out) { +static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -1854,8 +1939,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, - &m, &dims, &encoding, s); + int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; @@ -1866,8 +1950,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (cJSON_IsString(input)) { if (!input->valuestring || input->valuestring[0] == '\0') - ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", - "value_error.empty"); + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); n_inputs = 1; } else if (cJSON_IsArray(input)) { n_inputs = cJSON_GetArraySize(input); @@ -1890,13 +1973,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, i++; } } else { - ve_add(detail, "[\"body\",\"input\"]", - "input must be a string or an array of strings", "type_error"); + ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", + "type_error"); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -1916,8 +1998,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, out->inputs[0].n_tokens = out->tokens[0].n_tokens; out->total_tokens = out->tokens[0].n_tokens; if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", - "input exceeds 32768 token limit", + ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", "value_error.context_length"); } } else { @@ -1926,15 +2007,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, loc, "input item must be a string", - "type_error.string"); + ve_add(detail, loc, "input item must be a string", "type_error.string"); idx++; continue; } - if (tokenize_one(m, j, item->valuestring, - &out->tokens[idx]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_one(m, j, item->valuestring, &out->tokens[idx]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { out->inputs[idx].ids = out->tokens[idx].ids; out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; @@ -1962,8 +2040,8 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, cJSON_Delete(detail); } -static void prepare_contextual_request(job *j, cJSON *root, http_server *s, - contextual_request *out) { +static void +prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -1972,8 +2050,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, - &m, &dims, &encoding, s); + int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; @@ -1984,8 +2061,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, if (!input) { ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (!cJSON_IsArray(input)) { - ve_add(detail, "[\"body\",\"input\"]", - "input must be an array of document chunk arrays", "type_error.array"); + ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", + "type_error.array"); } else { n_docs = cJSON_GetArraySize(input); if (n_docs < 1) @@ -2000,8 +2077,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, if (!cJSON_IsArray(doc_arr)) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must be an array of strings", - "type_error.array"); + ve_add(detail, loc, "document must be an array of strings", "type_error.array"); di++; continue; } @@ -2036,8 +2112,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -2051,8 +2126,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(doc_arr, input) { contextual_doc *doc = &out->docs[di]; int n_chunks = cJSON_GetArraySize(doc_arr); - token_buf *chunk_tokens = - xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); doc->n_spans = n_chunks; int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; @@ -2063,10 +2137,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(chunk, doc_arr) { char loc[80]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (tokenize_one(m, j, chunk->valuestring, - &chunk_tokens[ci]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); doc_valid = 0; } else { doc_tokens += chunk_tokens[ci].n_tokens; @@ -2103,8 +2175,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, di++; } if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { - ve_add(detail, "[\"body\",\"input\"]", - "request exceeds 120000 token limit", + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } else { out->total_tokens = (int)request_tokens; @@ -2122,30 +2193,28 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_Delete(detail); } -static int validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, - loaded_model **out_model) { +static int +validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); if (!item) { ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); return 0; } if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, "[\"body\",\"model\"]", "model must be a string", - "type_error.string"); + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); return 0; } model_slot slot = model_slot_for_id(item->valuestring); if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { - ve_add(detail, "[\"body\",\"model\"]", - "value is not a valid enum member for this endpoint", "enum"); + ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", + "enum"); return 0; } *out_model = s->models[slot].info ? &s->models[slot] : NULL; return *out_model ? 0 : 503; } -static void prepare_rerank_request(job *j, cJSON *root, http_server *s, - rerank_request *out) { +static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -2157,28 +2226,24 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, if (!query) { ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); } else if (!cJSON_IsString(query)) { - ve_add(detail, "[\"body\",\"query\"]", "query must be a string", - "type_error.string"); + ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); } else if (!query->valuestring || !query->valuestring[0]) { - ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", - "value_error.empty"); + ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); } int n_documents = 0; if (!documents) { ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); } else if (!cJSON_IsArray(documents)) { - ve_add(detail, "[\"body\",\"documents\"]", - "documents must be an array of strings", "type_error.array"); + ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", + "type_error.array"); } else { n_documents = cJSON_GetArraySize(documents); if (n_documents < 1) - ve_add(detail, "[\"body\",\"documents\"]", - "documents must contain at least 1 item", + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", "value_error.list.min_items"); else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) - ve_add(detail, "[\"body\",\"documents\"]", - "documents must contain at most 1000 items", + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", "value_error.list.max_items"); cJSON *document; int i = 0; @@ -2186,51 +2251,43 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); if (!cJSON_IsString(document)) - ve_add(detail, loc, "document must be a string", - "type_error.string"); + ve_add(detail, loc, "document must be a string", "type_error.string"); else if (!document->valuestring || !document->valuestring[0]) - ve_add(detail, loc, "document must not be empty", - "value_error.empty"); + ve_add(detail, loc, "document must not be empty", "value_error.empty"); } } cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); if (top_n && top_k) { - ve_add(detail, "[\"body\",\"top_n\"]", - "top_n and top_k are aliases; provide only one", + ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", "value_error.conflict"); } cJSON *top = top_n ? top_n : top_k; out->top_n = n_documents; if (top) { if (!cjson_is_integer(top)) { - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" - : "[\"body\",\"top_k\"]", + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", "top_n must be an integer", "type_error.integer"); } else { out->top_n = top->valueint; if (out->top_n < 1 || out->top_n > n_documents) - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" - : "[\"body\",\"top_k\"]", - "top_n must be between 1 and the number of documents", - "value_error.range"); + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be between 1 and the number of documents", "value_error.range"); } } - cJSON *return_documents = - cJSON_GetObjectItemCaseSensitive(root, "return_documents"); + cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); if (return_documents) { if (!cJSON_IsBool(return_documents)) - ve_add(detail, "[\"body\",\"return_documents\"]", - "return_documents must be a boolean", "type_error.bool"); + ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", + "type_error.bool"); else out->return_documents = cJSON_IsTrue(return_documents); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -2238,15 +2295,13 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, if (cJSON_GetArraySize(detail) == 0 && out->model) { out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); - if (tokenize_late_text(out->model, j, query->valuestring, 1, - &out->query) != 0) { + if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) { ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); } else { out->query_tokens = out->query.n_tokens; if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"query\"]", - "query exceeds 32768 token limit", + ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", "value_error.context_length"); } @@ -2255,10 +2310,9 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(document, documents) { char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); - if (tokenize_late_text(out->model, j, document->valuestring, 0, - &out->documents[i]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != + 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { int n = out->documents[i].n_tokens; if (n > EMBED_API_MAX_ITEM_TOKENS) @@ -2271,10 +2325,8 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, } i++; } - if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - - out->document_tokens) { - ve_add(detail, "[\"body\",\"documents\"]", - "request exceeds 120000 token limit", + if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { + ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", "value_error.context_length"); } } @@ -2297,8 +2349,10 @@ typedef struct { static int rerank_result_cmp(const void *a, const void *b) { const rerank_result *ra = a; const rerank_result *rb = b; - if (ra->score > rb->score) return -1; - if (ra->score < rb->score) return 1; + if (ra->score > rb->score) + return -1; + if (ra->score < rb->score) + return 1; return ra->index - rb->index; } @@ -2317,26 +2371,24 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - int rc = embed_late_model_encode_tokens( - m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens, - 1, query); + int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, + r->query.n_tokens, 1, query); int pos = 0; offsets[0] = 0; for (int i = 0; rc == 0 && i < r->n_documents; i++) { late_tokens *doc = &r->documents[i]; - rc = embed_late_model_encode_tokens( - m->cpu_late_model, m->cpu_late_ws, doc->ids, doc->n_tokens, 1, all); + rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids, + doc->n_tokens, 1, all); for (int k = 0; rc == 0 && k < doc->n_keep; k++) { - memcpy(docs + (size_t)pos * dim, - all + (size_t)doc->keep[k] * dim, + memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim, (size_t)dim * sizeof(*docs)); pos++; } offsets[i + 1] = pos; } if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, - r->n_documents, dim, scores); + rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, + scores); free(offsets); free(all); free(docs); @@ -2347,25 +2399,22 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { #ifdef USE_MLX static int execute_rerank_mlx(rerank_request *r, float *scores) { loaded_model *m = r->model; - embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device( - m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); - embed_mlx_late_vectors_t **docs = - xcalloc((size_t)r->n_documents, sizeof(*docs)); - const embed_mlx_late_vectors_t **doc_refs = - xcalloc((size_t)r->n_documents, sizeof(*doc_refs)); + embed_mlx_late_vectors_t *query = + embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); + embed_mlx_late_vectors_t **docs = xcalloc((size_t)r->n_documents, sizeof(*docs)); + const embed_mlx_late_vectors_t **doc_refs = xcalloc((size_t)r->n_documents, sizeof(*doc_refs)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); int rc = query ? 0 : -1; offsets[0] = 0; for (int i = 0; rc == 0 && i < r->n_documents; i++) { late_tokens *doc = &r->documents[i]; - embed_mlx_late_vectors_t *all = embed_mlx_late_encode_tokens_device( - m->mlx_late_ctx, doc->ids, doc->n_tokens, 1); + embed_mlx_late_vectors_t *all = + embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, doc->ids, doc->n_tokens, 1); if (!all) { rc = -1; break; } - docs[i] = embed_mlx_late_vectors_select( - m->mlx_late_ctx, all, doc->keep, doc->n_keep); + docs[i] = embed_mlx_late_vectors_select(m->mlx_late_ctx, all, doc->keep, doc->n_keep); embed_mlx_late_vectors_free(all); if (!docs[i]) { rc = -1; @@ -2374,14 +2423,13 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { doc_refs[i] = docs[i]; offsets[i + 1] = offsets[i] + doc->n_keep; } - embed_mlx_late_vectors_t *packed = rc == 0 - ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, - r->n_documents) - : NULL; - if (!packed) rc = -1; + embed_mlx_late_vectors_t *packed = + rc == 0 ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, r->n_documents) : NULL; + if (!packed) + rc = -1; if (rc == 0) - rc = embed_mlx_late_maxsim_batch_device( - m->mlx_late_ctx, query, packed, offsets, r->n_documents, scores); + rc = embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, + r->n_documents, scores); embed_mlx_late_vectors_free(packed); for (int i = 0; i < r->n_documents; i++) embed_mlx_late_vectors_free(docs[i]); @@ -2396,29 +2444,24 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { static void execute_rerank_request(rerank_request *r) { if (!r || !r->model) { if (r && r->j) - job_set_error(r->j, 500, "late-interaction model is unavailable", - "server_error"); + job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); return; } float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); uint64_t t0 = nstime(); #ifdef USE_MLX - int rc = r->model->mlx_late_ctx - ? execute_rerank_mlx(r, scores) - : execute_rerank_cpu(r, scores); + int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); #else int rc = execute_rerank_cpu(r, scores); #endif r->j->infer_ns += nstime() - t0; if (rc != 0) { free(scores); - job_set_error(r->j, 500, "late-interaction reranking failed", - "server_error"); + job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); return; } - rerank_result *ranked = - xmalloc((size_t)r->n_documents * sizeof(*ranked)); + rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); for (int i = 0; i < r->n_documents; i++) { ranked[i].index = i; ranked[i].score = scores[i]; @@ -2427,14 +2470,13 @@ static void execute_rerank_request(rerank_request *r) { t0 = nstime(); sbuf b = {0}; - sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", - r->model->info->id); - cJSON *documents = - cJSON_GetObjectItemCaseSensitive(r->root, "documents"); + sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); for (int i = 0; i < r->top_n; i++) { - if (i) sbuf_putc(&b, ','); - sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", - ranked[i].index, (double)ranked[i].score); + if (i) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, + (double)ranked[i].score); if (r->return_documents) { cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); sbuf_puts(&b, ",\"document\":"); @@ -2443,10 +2485,9 @@ static void execute_rerank_request(rerank_request *r) { sbuf_putc(&b, '}'); } sbuf_printf(&b, - "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," - "\"total_tokens\":%d}}", - r->query_tokens, r->document_tokens, - r->query_tokens + r->document_tokens); + "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," + "\"total_tokens\":%d}}", + r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); set_response_from_buf(r->j, &b); r->j->encode_ns += nstime() - t0; free(ranked); @@ -2469,7 +2510,8 @@ static int parse_job_root(job *j, cJSON **out_root) { static void process_unknown_job(job *j) { cJSON *root = NULL; - if (!j->started_ns) j->started_ns = nstime(); + if (!j->started_ns) + j->started_ns = nstime(); uint64_t t0 = nstime(); int rc = parse_job_root(j, &root); j->parse_ns += nstime() - t0; @@ -2482,8 +2524,7 @@ static void process_unknown_job(job *j) { static void process_job_group(http_server *s, job **jobs, int n_jobs) { embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); - rerank_request *rerank_reqs = - xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); + rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); int *done = xcalloc((size_t)n_jobs, sizeof(*done)); embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); @@ -2491,7 +2532,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int i = 0; i < n_jobs; i++) { cJSON *root = NULL; - if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime(); + if (!jobs[i]->started_ns) + jobs[i]->started_ns = nstime(); if (!strcmp(jobs[i]->path, "/v1/embeddings")) { kind[i] = 1; } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { @@ -2526,8 +2568,7 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { continue; } - if ((kind[i] == 1 && !std_reqs[i].ready) || - (kind[i] == 2 && !ctx_reqs[i].ready) || + if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) || (kind[i] == 3 && !rerank_reqs[i].ready)) { job_set_timing_header(jobs[i]); enqueue_done(jobs[i]); @@ -2548,8 +2589,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 1 && std_reqs[k].ready && embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && - embedding_request_fits_group(&std_reqs[k], group_inputs, - group_tokens, s->batch_size, s->max_batch_tokens)) { + embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens, + s->batch_size, s->max_batch_tokens)) { std_group[group_n++] = &std_reqs[k]; group_inputs += std_reqs[k].n_inputs; group_tokens += std_reqs[k].total_tokens; @@ -2570,8 +2611,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && - contextual_request_fits_group(&ctx_reqs[k], group_docs, - group_tokens, s->batch_size, s->max_batch_tokens)) { + contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens, + s->batch_size, s->max_batch_tokens)) { ctx_group[group_n++] = &ctx_reqs[k]; group_docs += ctx_reqs[k].n_docs; group_tokens += ctx_reqs[k].total_tokens; @@ -2601,26 +2642,19 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { free(std_reqs); } -static int configure_loaded_model(loaded_model *m, - const embed_config_t *config, - int token_dim) -{ - if (!m || !m->info || !config || - config->hidden_size != m->info->dim || - token_dim != m->info->token_dim || - config->attention_mode != m->info->attention_mode || +static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) { + if (!m || !m->info || !config || config->hidden_size != m->info->dim || + token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode || config->pooling_mode != m->info->pooling_mode || config->normalize_embeddings != m->info->normalize_embeddings) { if (m && m->info && config) - server_log("embed-server: model %s has incompatible config", - m->info->id); + server_log("embed-server: model %s has incompatible config", m->info->id); return -1; } m->append_terminal_token = config->append_terminal_token; /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ m->renormalize_truncated = - config->normalize_embeddings && - config->pooling_mode == EMBED_POOL_LAST_TOKEN; + config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN; return 0; } @@ -2631,28 +2665,27 @@ static void *worker_main(void *arg) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; - if (!m->info) continue; + if (!m->info) + continue; embed_mlx_options_t mlx_opts = { .quantize_bits = s->mlx_quantize_bits, .quantize_group_size = s->mlx_quantize_group_size, }; if (m->info->kind == MODEL_KIND_LATE) - m->mlx_late_ctx = - embed_mlx_late_load_with_options(m->path, &mlx_opts); + m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts); else m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts); if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) || (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load MLX model on worker: %s", - m->path); + server_log("embed-server: failed to load MLX model on worker: %s", m->path); rc = 1; break; } const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_config(m->mlx_late_ctx) - : embed_mlx_config(m->mlx_ctx); - int token_dim = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; + ? embed_mlx_late_config(m->mlx_late_ctx) + : embed_mlx_config(m->mlx_ctx); + int token_dim = + m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; if (configure_loaded_model(m, config, token_dim) != 0 || (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || @@ -2665,7 +2698,8 @@ static void *worker_main(void *arg) { pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; - if (rc) s->stopping = 1; + if (rc) + s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { @@ -2685,20 +2719,19 @@ static void *worker_main(void *arg) { } else #endif #ifdef USE_CUDA - if (s->use_cuda) { + if (s->use_cuda) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; - if (!m->info) continue; + if (!m->info) + continue; m->cuda_ctx = embed_cuda_load(m->path); if (!m->cuda_ctx) { - server_log("embed-server: failed to load CUDA model on worker: %s", - m->path); + server_log("embed-server: failed to load CUDA model on worker: %s", m->path); rc = 1; break; } - if (configure_loaded_model( - m, embed_cuda_config(m->cuda_ctx), 0) != 0) { + if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) { rc = 1; break; } @@ -2706,7 +2739,8 @@ static void *worker_main(void *arg) { pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; - if (rc) s->stopping = 1; + if (rc) + s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { @@ -2730,9 +2764,9 @@ static void *worker_main(void *arg) { for (;;) { job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = collect_job_batch( - s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); - if (n_jobs == 0) break; + int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); + if (n_jobs == 0) + break; process_job_group(s, jobs, n_jobs); } #ifdef USE_MLX @@ -2792,8 +2826,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } m->tok_ws = qwen_tokenizer_workspace_new(); if (!m->tok_ws) { - server_log("embed-server: failed to allocate tokenizer workspace: %s", - path); + server_log("embed-server: failed to allocate tokenizer workspace: %s", path); return -1; } /* The contextual chunk separator is the tokenizer's <|endoftext|>. The @@ -2802,8 +2835,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { * token (e.g. small test models) takes precedence so the id stays * inside that model's embedding table. */ int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>"); - m->context_separator_id = sep_id >= 0 - ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> * suffix - the same token as the separator, not the model's chat * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ @@ -2812,16 +2844,13 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { int id = qwen_tokenizer_token_id(m->tok, "[MASK]"); m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; id = qwen_tokenizer_token_id(m->tok, "[Q]"); - m->late_query_prefix_id = - id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; + m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; id = qwen_tokenizer_token_id(m->tok, "[D]"); - m->late_document_prefix_id = - id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; + m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; - for (const char *p = punct; - *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / - sizeof(m->late_skip_ids[0])); + for (const char *p = punct; *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / + sizeof(m->late_skip_ids[0])); p++) { char text[2] = {*p, '\0'}; int n_ids = 0; @@ -2854,8 +2883,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model); if (!m->cpu_late_ws) { - server_log("embed-server: failed to allocate late workspace: %s", - path); + server_log("embed-server: failed to allocate late workspace: %s", path); return -1; } config = embed_late_model_config(m->cpu_late_model); @@ -2877,8 +2905,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { return -1; } if (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || + (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size)) { server_log("embed-server: late special-token ids exceed model vocab"); return -1; @@ -2890,18 +2917,27 @@ static void free_models(http_server *s) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; free(m->path); - if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); - if (m->tok) qwen_tokenizer_free(m->tok); - if (m->cpu_ws) embed_workspace_free(m->cpu_ws); - if (m->cpu_model) embed_model_free(m->cpu_model); - if (m->cpu_late_ws) embed_late_workspace_free(m->cpu_late_ws); - if (m->cpu_late_model) embed_late_model_free(m->cpu_late_model); + if (m->tok_ws) + qwen_tokenizer_workspace_free(m->tok_ws); + if (m->tok) + qwen_tokenizer_free(m->tok); + if (m->cpu_ws) + embed_workspace_free(m->cpu_ws); + if (m->cpu_model) + embed_model_free(m->cpu_model); + if (m->cpu_late_ws) + embed_late_workspace_free(m->cpu_late_ws); + if (m->cpu_late_model) + embed_late_model_free(m->cpu_late_model); #ifdef USE_MLX - if (m->mlx_ctx) embed_mlx_free(m->mlx_ctx); - if (m->mlx_late_ctx) embed_mlx_late_free(m->mlx_late_ctx); + if (m->mlx_ctx) + embed_mlx_free(m->mlx_ctx); + if (m->mlx_late_ctx) + embed_mlx_late_free(m->mlx_late_ctx); #endif #ifdef USE_CUDA - if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx); + if (m->cuda_ctx) + embed_cuda_free(m->cuda_ctx); #endif } } @@ -2910,14 +2946,12 @@ static uint64_t physical_memory_nbytes(void) { #ifdef __APPLE__ uint64_t bytes = 0; size_t len = sizeof(bytes); - if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && - len == sizeof(bytes) && bytes > 0) + if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) return bytes; #endif long pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); - if (pages <= 0 || page_size <= 0 || - (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) + if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) return 0; return (uint64_t)pages * (uint64_t)page_size; } @@ -2928,13 +2962,11 @@ static int validate_model_specs(const embed_server_config_t *cfg) { const embed_server_model_spec_t *spec = &cfg->models[i]; model_slot slot = model_slot_for_id(spec->id); if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "embed-server: unknown model id: %s\n", - spec->id ? spec->id : "<null>"); + fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : "<null>"); return -1; } if (!spec->path || !spec->path[0]) { - fprintf(stderr, "embed-server: model %s has an empty path\n", - spec->id); + fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); return -1; } if (seen[slot]) { @@ -2942,8 +2974,7 @@ static int validate_model_specs(const embed_server_config_t *cfg) { return -1; } if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) { - fprintf(stderr, - "embed-server: model %s has no CUDA late-interaction backend\n", + fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n", spec->id); return -1; } @@ -2953,17 +2984,16 @@ static int validate_model_specs(const embed_server_config_t *cfg) { } static int mlx_memory_preflight(const embed_server_config_t *cfg) { - if (!cfg->use_mlx) return 0; + if (!cfg->use_mlx) + return 0; uint64_t payload = 0; for (int i = 0; i < cfg->n_models; i++) { - multi_safetensors_t *ms = - multi_safetensors_open(cfg->models[i].path); + multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path); size_t model_payload = 0; if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { multi_safetensors_close(ms); - fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", - cfg->models[i].path); + fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path); return -1; } multi_safetensors_close(ms); @@ -2985,22 +3015,21 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) { "skipping MLX memory preflight"); return 0; } - double utilization = cfg->memory_utilization > 0.0 - ? cfg->memory_utilization - : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; + double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization + : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; uint64_t budget = (uint64_t)((double)physical * utilization); const double gib = 1024.0 * 1024.0 * 1024.0; server_log("embed-server: MLX memory preflight: %.1f GiB tensors, " "%.1f GiB estimated resident, %.1f GiB budget " "(%.2f of physical memory)", - (double)payload / gib, (double)estimated / gib, - (double)budget / gib, utilization); + (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); if (cfg->mlx_quantize_bits) { server_log("embed-server: MLX %d-bit quantization enabled; preflight " "uses source payload as a conservative peak estimate", cfg->mlx_quantize_bits); } - if (estimated <= budget) return 0; + if (estimated <= budget) + return 0; server_log("embed-server: refusing MLX model set above the host-memory " "budget"); server_log("embed-server: use BF16 artifacts, load fewer models, or raise " @@ -3020,24 +3049,23 @@ int embed_run_server(const embed_server_config_t *cfg) { memset(&s, 0, sizeof(s)); s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; s.port = cfg->port > 0 ? cfg->port : 8000; - s.batch_size = cfg->batch_size > 0 - ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; - s.max_batch_tokens = cfg->max_batch_tokens > 0 - ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; - s.batch_wait_us = cfg->batch_wait_us >= 0 - ? cfg->batch_wait_us - : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US - : EMBED_SERVER_BATCH_WAIT_US); + s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = + cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us + : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US + : EMBED_SERVER_BATCH_WAIT_US); s.use_mlx = cfg->use_mlx; s.use_cuda = cfg->use_cuda; s.mlx_quantize_bits = cfg->mlx_quantize_bits; - s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 - ? cfg->mlx_quantize_group_size : 64; + s.mlx_quantize_group_size = + cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); else { const char *env_key = getenv("EMBED_API_KEY"); - if (env_key && env_key[0]) s.api_key = xstrdup(env_key); + if (env_key && env_key[0]) + s.api_key = xstrdup(env_key); } pthread_mutex_init(&s.mu, NULL); @@ -3124,7 +3152,8 @@ int embed_run_server(const embed_server_config_t *cfg) { aeMain(s.loop); server_log("embed-server: shutdown requested"); - if (s.listen_fd >= 0) close(s.listen_fd); + if (s.listen_fd >= 0) + close(s.listen_fd); client *c = s.clients; while (c) { client *next = c->next; @@ -3155,59 +3184,58 @@ int embed_run_server(const embed_server_config_t *cfg) { * ==================================================================== */ #ifndef EMBED_SERVER_TEST -static void print_usage(const char *prog) -{ +static void print_usage(const char *prog) { fprintf(stderr, - "Usage: %s --model ID=DIR [options]\n" - "\n" - "Options:\n" - " --model ID=DIR Model to serve (repeatable)\n" + "Usage: %s --model ID=DIR [options]\n" + "\n" + "Options:\n" + " --model ID=DIR Model to serve (repeatable)\n" #ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" + " --mlx Use Apple MLX GPU backend\n" #endif #ifdef USE_CUDA - " --cuda Use NVIDIA CUDA GPU backend\n" + " --cuda Use NVIDIA CUDA GPU backend\n" #endif - " --host HOST Bind host (default: 127.0.0.1)\n" - " --port N Bind port (default: 8000)\n" - " --api-key K Require Authorization: Bearer K\n" - " -b, --batch-size N Max texts or documents per inference batch\n" - " (default: 32)\n" - " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N First-arrival micro-batch deadline in us\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 32)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N First-arrival micro-batch deadline in us\n" #ifdef USE_CUDA - " (default: CUDA 1000; CPU 0)\n" + " (default: CUDA 1000; CPU 0)\n" #else - " (default: 0)\n" + " (default: 0)\n" #endif #ifdef USE_MLX - " --memory-utilization F Fraction of physical memory the MLX model-set\n" - " preflight may plan for (default: 0.90; values\n" - " above 1.0 overcommit)\n" - " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" - " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" + " --memory-utilization F Fraction of physical memory the MLX model-set\n" + " preflight may plan for (default: 0.90; values\n" + " above 1.0 overcommit)\n" + " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" + " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" #endif #ifdef USE_CUDA - " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" - " (default: f32, exact)\n" - " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" - " f32); bf16 halves weight memory\n" + " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n" #endif - " -t, --threads N CPU threads (default: all cores)\n" - " -v, --verbose Verbose (-vv for debug)\n" - " -V, --version Print version and exit\n" - " --build-info Print build details and exit\n" - " -h, --help Show this help\n" - "\n" - "Examples:\n" - " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" + " -t, --threads N CPU threads (default: all cores)\n" + " -v, --verbose Verbose (-vv for debug)\n" + " -V, --version Print version and exit\n" + " --build-info Print build details and exit\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" #ifdef USE_MLX - " %s --mlx --mlx-quant-bits 8 \\\n" - " --model pplx-embed-v1-4b=./model-4b-bf16\n", - prog, prog, prog); + " %s --mlx --mlx-quant-bits 8 \\\n" + " --model pplx-embed-v1-4b=./model-4b-bf16\n", + prog, prog, prog); #else - , - prog, prog); + , + prog, prog); #endif } @@ -3217,8 +3245,7 @@ typedef struct { int cap; } model_specs_t; -static int append_model_spec(model_specs_t *specs, const char *arg) -{ +static int append_model_spec(model_specs_t *specs, const char *arg) { const char *eq = strchr(arg, '='); if (!eq || eq == arg || !eq[1]) { fprintf(stderr, "--model expects MODEL_ID=DIR\n"); @@ -3227,7 +3254,8 @@ static int append_model_spec(model_specs_t *specs, const char *arg) if (specs->n == specs->cap) { int new_cap = specs->cap ? specs->cap * 2 : 4; void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) return -1; + if (!p) + return -1; specs->v = p; specs->cap = new_cap; } @@ -3246,9 +3274,9 @@ static int append_model_spec(model_specs_t *specs, const char *arg) return 0; } -static void free_model_specs(model_specs_t *specs) -{ - if (!specs) return; +static void free_model_specs(model_specs_t *specs) { + if (!specs) + return; for (int i = 0; i < specs->n; i++) { free((char *)specs->v[i].id); free((char *)specs->v[i].path); @@ -3256,8 +3284,7 @@ static void free_model_specs(model_specs_t *specs) free(specs->v); } -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]) { int n_threads = 0; int verbose = 0; int batch_size = 0; @@ -3287,37 +3314,34 @@ int main(int argc, char *argv[]) embed_print_version(prog); free_model_specs(&model_specs); return 0; - } - else if (!strcmp(f, "--build-info")) { + } else if (!strcmp(f, "--build-info")) { embed_print_build_info(prog); free_model_specs(&model_specs); return 0; - } - else if (!strcmp(f, "--model")) { + } else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg]) != 0) { free_model_specs(&model_specs); return 1; } } #ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { use_mlx = 1; } - else if (!strcmp(f, "--mlx-quant-bits")) { + else if (!strcmp(f, "--mlx")) { + use_mlx = 1; + } else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { + } else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg]); if (mlx_quantize_group_size <= 0) { fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--memory-utilization")) { + } else if (!strcmp(f, "--memory-utilization")) { memory_utilization = atof(argv[++arg]); if (memory_utilization <= 0.0) { fprintf(stderr, "--memory-utilization must be > 0\n"); @@ -3327,47 +3351,47 @@ int main(int argc, char *argv[]) } #endif #ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { use_cuda = 1; } - else if (!strcmp(f, "--cuda-gemm-mode")) { + else if (!strcmp(f, "--cuda")) { + use_cuda = 1; + } else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { + } else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg]; } #endif - else if (!strcmp(f, "--host")) { host = argv[++arg]; } - else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } - else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } - else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + else if (!strcmp(f, "--host")) { + host = argv[++arg]; + } else if (!strcmp(f, "--port")) { + port = atoi(argv[++arg]); + } else if (!strcmp(f, "--api-key")) { + api_key = argv[++arg]; + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg]); - } - else if (!strcmp(f, "--max-batch-tokens")) { + } else if (!strcmp(f, "--max-batch-tokens")) { max_batch_tokens = atoi(argv[++arg]); if (max_batch_tokens <= 0) { fprintf(stderr, "--max-batch-tokens must be > 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--batch-wait-us")) { + } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { fprintf(stderr, "--batch-wait-us must be >= 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { n_threads = atoi(argv[++arg]); - } - else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } - else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(prog); free_model_specs(&model_specs); return 0; - } - else { + } else { fprintf(stderr, "unknown option: %s\n", f); print_usage(prog); free_model_specs(&model_specs); @@ -3431,18 +3455,22 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (use_mlx) { - if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + if (verbose >= 1) + fprintf(stderr, "Using MLX GPU backend\n"); } else #endif #ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); + if (use_cuda) { + if (verbose >= 1) + fprintf(stderr, "Using CUDA GPU backend\n"); } else #endif { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + if (n_threads <= 0) + n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + if (verbose >= 1) + fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } embed_server_config_t scfg = { |
