aboutsummaryrefslogtreecommitdiff
path: root/embed_server.c
diff options
context:
space:
mode:
Diffstat (limited to 'embed_server.c')
-rw-r--r--embed_server.c1300
1 files changed, 664 insertions, 636 deletions
diff --git a/embed_server.c b/embed_server.c
index 8c7743c..38da125 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -47,39 +47,39 @@
#include <cjson/cJSON.h>
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
+#define EMBED_HTTP_IO_BUF 8192
+#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
+#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define EMBED_HTTP_MAX_PATH 255u
+#define EMBED_HTTP_BACKLOG 128
+#define EMBED_HTTP_SETSIZE 10240
#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
+#define EMBED_HTTP_SWEEP_MS 1000
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
+#define EMBED_API_MAX_STANDARD_INPUTS 512
+#define EMBED_API_MAX_CONTEXT_DOCS 512
+#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
+#define EMBED_API_MAX_ITEM_TOKENS 32768
+#define EMBED_API_MAX_TOTAL_TOKENS 120000
+#define EMBED_LATE_QUERY_TOKENS 32
+#define EMBED_LATE_MASK_TOKEN_ID 151642
+#define EMBED_LATE_QUERY_PREFIX_ID 151669
+#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
+#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
/* Microseconds the worker waits for more requests before dispatching a batch.
* CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
* busy; MLX and CPU dispatch immediately, where waiting only adds latency.
* Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
+#define EMBED_SERVER_BATCH_WAIT_US 0
+#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
+#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
+#define EMBED_MLX_RESIDENT_MULTIPLIER 2
typedef struct {
char *ptr;
@@ -94,19 +94,22 @@ static void die_oom(void) {
static void *xmalloc(size_t n) {
void *p = malloc(n ? n : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
static void *xcalloc(size_t n, size_t sz) {
void *p = calloc(n ? n : 1, sz ? sz : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
static void *xrealloc(void *p, size_t n) {
p = realloc(p, n ? n : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
@@ -142,9 +145,7 @@ static uint64_t nstime(void) {
return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u;
}
-static double ns_to_ms(uint64_t ns) {
- return (double)ns / 1000000.0;
-}
+static double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; }
static void server_log(const char *fmt, ...) {
char ts[32];
@@ -161,12 +162,15 @@ static void server_log(const char *fmt, ...) {
}
static void sbuf_reserve(sbuf *b, size_t add) {
- if (add > SIZE_MAX - b->len - 1) die_oom();
+ if (add > SIZE_MAX - b->len - 1)
+ die_oom();
size_t need = b->len + add + 1;
- if (need <= b->cap) return;
+ if (need <= b->cap)
+ return;
size_t cap = b->cap ? b->cap * 2 : 256;
while (cap < need) {
- if (cap > SIZE_MAX / 2) die_oom();
+ if (cap > SIZE_MAX / 2)
+ die_oom();
cap *= 2;
}
b->ptr = xrealloc(b->ptr, cap);
@@ -185,7 +189,8 @@ static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); }
static void sbuf_clear(sbuf *b) {
b->len = 0;
- if (b->ptr) b->ptr[0] = '\0';
+ if (b->ptr)
+ b->ptr[0] = '\0';
}
static void sbuf_printf(sbuf *b, const char *fmt, ...) {
@@ -194,11 +199,13 @@ static void sbuf_printf(sbuf *b, const char *fmt, ...) {
va_copy(ap2, ap);
int n = vsnprintf(NULL, 0, fmt, ap);
va_end(ap);
- if (n < 0) die_oom();
+ if (n < 0)
+ die_oom();
sbuf_reserve(b, (size_t)n);
int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2);
va_end(ap2);
- if (n2 < 0 || n2 != n) die_oom();
+ if (n2 < 0 || n2 != n)
+ die_oom();
b->len += (size_t)n;
}
@@ -220,11 +227,7 @@ typedef enum {
MODEL_UNKNOWN
} model_slot;
-typedef enum {
- MODEL_KIND_STANDARD,
- MODEL_KIND_CONTEXTUAL,
- MODEL_KIND_LATE
-} model_kind;
+typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
* API: the canonical output is the tanh int8 quantization, so the default
@@ -251,28 +254,30 @@ typedef struct {
} model_info;
static const model_info k_models[] = {
- {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
- {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
- {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
{"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
};
static model_slot model_slot_for_id(const char *id) {
- if (!id) return MODEL_UNKNOWN;
+ if (!id)
+ return MODEL_UNKNOWN;
for (int i = 0; i < MODEL_COUNT; i++) {
- if (!strcmp(id, k_models[i].id)) return (model_slot)i;
+ if (!strcmp(id, k_models[i].id))
+ return (model_slot)i;
}
return MODEL_UNKNOWN;
}
@@ -398,7 +403,8 @@ static volatile sig_atomic_t g_signal_wfd = -1;
static void stop_signal_handler(int sig) {
(void)sig;
- if (g_stop_requested) _exit(130);
+ if (g_stop_requested)
+ _exit(130);
int saved = errno;
g_stop_requested = 1;
int fd = (int)g_signal_wfd;
@@ -411,7 +417,8 @@ static void stop_signal_handler(int sig) {
static int set_nonblock(int fd) {
int flags = fcntl(fd, F_GETFL, 0);
- if (flags < 0) return -1;
+ if (flags < 0)
+ return -1;
return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
@@ -422,7 +429,8 @@ static void http_request_free(http_request *r) {
}
static void client_free(client *c) {
- if (!c) return;
+ if (!c)
+ return;
http_request_free(&c->req);
sbuf_free(&c->in);
sbuf_free(&c->out);
@@ -438,34 +446,45 @@ static void client_incref(client *c) {
static void client_decref(client *c) {
int free_now = 0;
pthread_mutex_lock(&c->srv->mu);
- if (--c->refcount == 0) free_now = 1;
+ if (--c->refcount == 0)
+ free_now = 1;
pthread_mutex_unlock(&c->srv->mu);
- if (free_now) client_free(c);
+ if (free_now)
+ client_free(c);
}
static void client_decref_n(client *c, int n) {
int free_now = 0;
- if (n <= 0) return;
+ if (n <= 0)
+ return;
pthread_mutex_lock(&c->srv->mu);
c->refcount -= n;
- if (c->refcount <= 0) free_now = 1;
+ if (c->refcount <= 0)
+ free_now = 1;
pthread_mutex_unlock(&c->srv->mu);
- if (free_now) client_free(c);
+ if (free_now)
+ client_free(c);
}
static void client_unlink(client *c) {
http_server *s = c->srv;
- if (!c->linked) return;
- if (c->prev) c->prev->next = c->next;
- else s->clients = c->next;
- if (c->next) c->next->prev = c->prev;
+ if (!c->linked)
+ return;
+ if (c->prev)
+ c->prev->next = c->next;
+ else
+ s->clients = c->next;
+ if (c->next)
+ c->next->prev = c->prev;
c->prev = c->next = NULL;
c->linked = 0;
- if (s->n_clients > 0) s->n_clients--;
+ if (s->n_clients > 0)
+ s->n_clients--;
}
static int close_client_unlink(client *c) {
- if (!c || c->cancelled) return 0;
+ if (!c || c->cancelled)
+ return 0;
c->cancelled = 1;
if (c->fd >= 0) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -478,40 +497,52 @@ static int close_client_unlink(client *c) {
}
static void close_client(client *c) {
- if (!close_client_unlink(c)) return;
+ if (!close_client_unlink(c))
+ return;
client_decref(c);
}
-static void append_http_response_ex(client *c, int status, const char *ctype,
+static void append_http_response_ex(client *c,
+ int status,
+ const char *ctype,
const char *extra_headers,
- const char *body, size_t body_len) {
+ const char *body,
+ size_t body_len) {
const char *reason = "OK";
- if (status == 204) reason = "No Content";
- else if (status == 400) reason = "Bad Request";
- else if (status == 401) reason = "Unauthorized";
- else if (status == 404) reason = "Not Found";
- else if (status == 405) reason = "Method Not Allowed";
- else if (status == 422) reason = "Unprocessable Entity";
- else if (status == 503) reason = "Service Unavailable";
- else if (status >= 500) reason = "Internal Server Error";
+ if (status == 204)
+ reason = "No Content";
+ else if (status == 400)
+ reason = "Bad Request";
+ else if (status == 401)
+ reason = "Unauthorized";
+ else if (status == 404)
+ reason = "Not Found";
+ else if (status == 405)
+ reason = "Method Not Allowed";
+ else if (status == 422)
+ reason = "Unprocessable Entity";
+ else if (status == 503)
+ reason = "Service Unavailable";
+ else if (status >= 500)
+ reason = "Internal Server Error";
sbuf_printf(&c->out,
"HTTP/1.1 %d %s\r\n"
"Content-Length: %zu\r\n"
"Connection: %s\r\n",
- status, reason, body_len,
- c->close_after_write ? "close" : "keep-alive");
+ status, reason, body_len, c->close_after_write ? "close" : "keep-alive");
if (ctype)
sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype);
if (extra_headers)
sbuf_puts(&c->out, extra_headers);
sbuf_puts(&c->out, "\r\n");
- if (body_len) sbuf_append(&c->out, body, body_len);
+ if (body_len)
+ sbuf_append(&c->out, body, body_len);
}
-static void append_http_response(client *c, int status, const char *ctype,
- const char *body, size_t body_len) {
+static void
+append_http_response(client *c, int status, const char *ctype, const char *body, size_t body_len) {
append_http_response_ex(c, status, ctype, NULL, body, body_len);
}
@@ -534,13 +565,27 @@ static void append_json_string(sbuf *b, const char *s) {
sbuf_putc(b, '"');
for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) {
switch (*p) {
- case '"': sbuf_puts(b, "\\\""); break;
- case '\\': sbuf_puts(b, "\\\\"); break;
- case '\b': sbuf_puts(b, "\\b"); break;
- case '\f': sbuf_puts(b, "\\f"); break;
- case '\n': sbuf_puts(b, "\\n"); break;
- case '\r': sbuf_puts(b, "\\r"); break;
- case '\t': sbuf_puts(b, "\\t"); break;
+ case '"':
+ sbuf_puts(b, "\\\"");
+ break;
+ case '\\':
+ sbuf_puts(b, "\\\\");
+ break;
+ case '\b':
+ sbuf_puts(b, "\\b");
+ break;
+ case '\f':
+ sbuf_puts(b, "\\f");
+ break;
+ case '\n':
+ sbuf_puts(b, "\\n");
+ break;
+ case '\r':
+ sbuf_puts(b, "\\r");
+ break;
+ case '\t':
+ sbuf_puts(b, "\\t");
+ break;
default:
if (*p < 0x20)
sbuf_printf(b, "\\u%04x", (unsigned)*p);
@@ -566,8 +611,10 @@ static void job_set_error(job *j, int status, const char *message, const char *t
}
static bool auth_ok(http_server *s, const char *auth) {
- if (!s->api_key || !s->api_key[0]) return true;
- if (!auth) return false;
+ if (!s->api_key || !s->api_key[0])
+ return true;
+ if (!auth)
+ return false;
const char prefix[] = "Bearer ";
return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
@@ -575,12 +622,12 @@ static bool auth_ok(http_server *s, const char *auth) {
static ssize_t find_header_end(const char *p, size_t n) {
for (size_t i = 3; i < n; i++) {
- if (p[i - 3] == '\r' && p[i - 2] == '\n' &&
- p[i - 1] == '\r' && p[i] == '\n')
+ if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n')
return (ssize_t)(i + 1);
}
for (size_t i = 1; i < n; i++) {
- if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1);
+ if (p[i - 1] == '\n' && p[i] == '\n')
+ return (ssize_t)(i + 1);
}
return -1;
}
@@ -590,18 +637,22 @@ static char *header_value_dup(const char *h, size_t n, const char *name) {
const char *p = h, *end = h + n;
while (p < end) {
const char *line = p;
- while (p < end && *p != '\n') p++;
+ while (p < end && *p != '\n')
+ p++;
size_t len = (size_t)(p - line);
- if (len && line[len - 1] == '\r') len--;
- if (len > name_len && !strncasecmp(line, name, name_len) &&
- line[name_len] == ':') {
+ if (len && line[len - 1] == '\r')
+ len--;
+ if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') {
const char *v = line + name_len + 1;
- while (v < line + len && isspace((unsigned char)*v)) v++;
+ while (v < line + len && isspace((unsigned char)*v))
+ v++;
const char *e = line + len;
- while (e > v && isspace((unsigned char)e[-1])) e--;
+ while (e > v && isspace((unsigned char)e[-1]))
+ e--;
return xstrndup(v, (size_t)(e - v));
}
- if (p < end) p++;
+ if (p < end)
+ p++;
}
return NULL;
}
@@ -610,13 +661,15 @@ static int header_has_token(const char *value, const char *token) {
size_t token_len = strlen(token);
const char *p = value;
while (p && *p) {
- while (*p == ',' || isspace((unsigned char)*p)) p++;
+ while (*p == ',' || isspace((unsigned char)*p))
+ p++;
const char *start = p;
- while (*p && *p != ',') p++;
+ while (*p && *p != ',')
+ p++;
const char *end = p;
- while (end > start && isspace((unsigned char)end[-1])) end--;
- if ((size_t)(end - start) == token_len &&
- !strncasecmp(start, token, token_len))
+ while (end > start && isspace((unsigned char)end[-1]))
+ end--;
+ if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len))
return 1;
}
return 0;
@@ -630,14 +683,14 @@ static int parse_request_headers(client *c) {
i++;
}
line[i] = '\0';
- int fields = sscanf(line, "%7s %255s %15s",
- c->req.method, c->req.path, c->req.version);
+ int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version);
if (fields < 2)
return -1;
if (fields < 3)
snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0");
char *q = strchr(c->req.path, '?');
- if (q) *q = '\0';
+ if (q)
+ *q = '\0';
c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0;
char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection");
@@ -669,15 +722,19 @@ static int complete_request(client *c) {
if (!c->header_done) {
ssize_t hend = find_header_end(c->in.ptr, c->in.len);
if (hend < 0) {
- if (c->in.len > EMBED_HTTP_MAX_HEADER) return -1;
+ if (c->in.len > EMBED_HTTP_MAX_HEADER)
+ return -1;
return 0;
}
c->header_done = 1;
c->header_len = (size_t)hend;
- if (parse_request_headers(c) != 0) return -1;
+ if (parse_request_headers(c) != 0)
+ return -1;
}
- if (c->content_length > EMBED_HTTP_MAX_BODY) return -1;
- if (c->in.len < c->header_len + c->content_length) return 0;
+ if (c->content_length > EMBED_HTTP_MAX_BODY)
+ return -1;
+ if (c->in.len < c->header_len + c->content_length)
+ return 0;
c->req.body_len = c->content_length;
c->req.body = xmalloc(c->content_length + 1);
memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length);
@@ -687,23 +744,25 @@ static int complete_request(client *c) {
static void send_bad_http_request(client *c) {
size_t len;
- char *body = json_error_body("bad HTTP request", "invalid_request_error",
- &len);
+ char *body = json_error_body("bad HTTP request", "invalid_request_error", &len);
c->close_after_write = 1;
append_http_response(c, 400, "application/json", body, len);
free(body);
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
}
static void reset_client_for_next_request(client *c) {
size_t consumed = c->header_len + c->content_length;
- if (consumed > c->in.len) consumed = c->in.len;
+ if (consumed > c->in.len)
+ consumed = c->in.len;
size_t remain = c->in.len - consumed;
if (remain)
memmove(c->in.ptr, c->in.ptr + consumed, remain);
c->in.len = remain;
- if (c->in.ptr) c->in.ptr[remain] = '\0';
+ if (c->in.ptr)
+ c->in.ptr[remain] = '\0';
http_request_free(&c->req);
sbuf_clear(&c->out);
@@ -716,7 +775,8 @@ static void reset_client_for_next_request(client *c) {
static void arm_next_request(client *c) {
reset_client_for_next_request(c);
- if (c->cancelled || c->fd < 0) return;
+ if (c->cancelled || c->fd < 0)
+ return;
if (c->in.len) {
int done = complete_request(c);
@@ -735,7 +795,8 @@ static void arm_next_request(client *c) {
}
static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
client *c = clientData;
while (c->sent < c->out.len) {
ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent);
@@ -744,26 +805,33 @@ static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
c->last_active_ms = mstime();
continue;
}
- if (n < 0 && errno == EINTR) continue;
- if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return;
+ if (n < 0 && errno == EINTR)
+ continue;
+ if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
+ return;
close_client(c);
return;
}
aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE);
- if (c->close_after_write) close_client(c);
- else arm_next_request(c);
+ if (c->close_after_write)
+ close_client(c);
+ else
+ arm_next_request(c);
}
static int queue_write(client *c) {
- if (!c || c->cancelled || c->fd < 0) return -1;
+ if (!c || c->cancelled || c->fd < 0)
+ return -1;
return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c);
}
static void enqueue_done(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
- if (s->done_tail) s->done_tail->next = j;
- else s->done_head = j;
+ if (s->done_tail)
+ s->done_tail->next = j;
+ else
+ s->done_head = j;
s->done_tail = j;
pthread_mutex_unlock(&s->mu);
const unsigned char byte = 1;
@@ -774,16 +842,17 @@ static job *pop_job_locked(http_server *s) {
job *j = s->job_head;
if (j) {
s->job_head = j->next;
- if (!s->job_head) s->job_tail = NULL;
+ if (!s->job_head)
+ s->job_tail = NULL;
j->next = NULL;
}
return j;
}
-static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu,
- uint64_t deadline_ns) {
+static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) {
uint64_t now_ns = nstime();
- if (now_ns >= deadline_ns) return ETIMEDOUT;
+ if (now_ns >= deadline_ns)
+ return ETIMEDOUT;
uint64_t remaining_ns = deadline_ns - now_ns;
struct timespec abs;
@@ -803,7 +872,8 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu,
}
static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
- if (max_jobs <= 0) return 0;
+ if (max_jobs <= 0)
+ return 0;
pthread_mutex_lock(&s->mu);
while (!s->job_head && !s->stopping)
@@ -817,9 +887,8 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
int n = 1;
jobs[0] = first;
- int batchable = s->batch_size > 1 &&
- (!strcmp(first->path, "/v1/embeddings") ||
- !strcmp(first->path, "/v1/contextualizedembeddings"));
+ int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") ||
+ !strcmp(first->path, "/v1/contextualizedembeddings"));
if (!batchable) {
pthread_mutex_unlock(&s->mu);
return n;
@@ -828,12 +897,13 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
/* Exact item and token budgets are applied after worker-side tokenization;
* this queue stage only defines which arrival window may be grouped. */
uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
- uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns
- ? UINT64_MAX : first->created_ns + wait_ns;
+ uint64_t deadline_ns =
+ first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
while (n < max_jobs) {
while (!s->job_head && !s->stopping && wait_ns > 0) {
int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
- if (rc == ETIMEDOUT) break;
+ if (rc == ETIMEDOUT)
+ break;
}
if (!s->job_head)
break;
@@ -850,8 +920,10 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
static void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
- if (s->job_tail) s->job_tail->next = j;
- else s->job_head = j;
+ if (s->job_tail)
+ s->job_tail->next = j;
+ else
+ s->job_head = j;
s->job_tail = j;
pthread_cond_signal(&s->cv);
pthread_mutex_unlock(&s->mu);
@@ -862,7 +934,8 @@ static job *pop_done(http_server *s) {
job *j = s->done_head;
if (j) {
s->done_head = j->next;
- if (!s->done_head) s->done_tail = NULL;
+ if (!s->done_head)
+ s->done_tail = NULL;
j->next = NULL;
}
pthread_mutex_unlock(&s->mu);
@@ -870,7 +943,8 @@ static job *pop_done(http_server *s) {
}
static void job_free(job *j) {
- if (!j) return;
+ if (!j)
+ return;
free(j->body);
free(j->auth);
free(j->extra_headers);
@@ -879,18 +953,19 @@ static void job_free(job *j) {
}
static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
http_server *s = clientData;
unsigned char tmp[128];
- while (read(fd, tmp, sizeof(tmp)) > 0) {}
+ while (read(fd, tmp, sizeof(tmp)) > 0) {
+ }
job *j;
while ((j = pop_done(s)) != NULL) {
client *c = j->c;
int refs_to_drop = 1; /* job reference */
if (!c->cancelled) {
- append_http_response_ex(c, j->status, j->content_type,
- j->extra_headers,
+ append_http_response_ex(c, j->status, j->content_type, j->extra_headers,
j->response ? j->response : "",
j->response ? j->response_len : 0);
if (queue_write(c) != AE_OK)
@@ -903,8 +978,7 @@ static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask)
static bool route_is_inference(const char *method, const char *path) {
return !strcmp(method, "POST") &&
- (!strcmp(path, "/v1/embeddings") ||
- !strcmp(path, "/v1/contextualizedembeddings") ||
+ (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") ||
!strcmp(path, "/v1/rerank"));
}
@@ -913,7 +987,8 @@ static void dispatch_request(client *c) {
if (!strcmp(c->req.method, "OPTIONS")) {
append_http_response(c, 204, NULL, "", 0);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
@@ -922,17 +997,19 @@ static void dispatch_request(client *c) {
char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
append_http_response(c, 404, "application/json", body, len);
free(body);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
if (!auth_ok(c->srv, c->req.auth)) {
size_t len;
- char *body = json_error_body("missing or invalid bearer token",
- "authentication_error", &len);
+ char *body =
+ json_error_body("missing or invalid bearer token", "authentication_error", &len);
append_http_response(c, 401, "application/json", body, len);
free(body);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
@@ -950,7 +1027,8 @@ static void dispatch_request(client *c) {
}
static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
client *c = clientData;
char tmp[EMBED_HTTP_IO_BUF];
for (;;) {
@@ -973,23 +1051,28 @@ static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
close_client(c);
return;
}
- if (errno == EINTR) continue;
- if (errno == EAGAIN || errno == EWOULDBLOCK) return;
+ if (errno == EINTR)
+ continue;
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
close_client(c);
return;
}
}
static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
http_server *s = clientData;
char cip[64];
int cport = 0;
for (;;) {
int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
if (cfd == ANET_ERR) {
- if (errno == EAGAIN || errno == EWOULDBLOCK) return;
- if (errno == EINTR) continue;
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
+ if (errno == EINTR)
+ continue;
return;
}
set_nonblock(cfd);
@@ -1000,7 +1083,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
c->last_active_ms = mstime();
c->linked = 1;
c->next = s->clients;
- if (s->clients) s->clients->prev = c;
+ if (s->clients)
+ s->clients->prev = c;
s->clients = c;
s->n_clients++;
if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
@@ -1009,7 +1093,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
}
static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
- (void)loop; (void)id;
+ (void)loop;
+ (void)id;
http_server *s = clientData;
uint64_t now = mstime();
client *c = s->clients;
@@ -1027,15 +1112,16 @@ static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
}
static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)fd; (void)mask;
+ (void)fd;
+ (void)mask;
http_server *s = clientData;
unsigned char tmp[128];
- while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {}
+ while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
+ }
aeStop(loop);
}
-static bool ve_add(cJSON *detail, const char *loc_json,
- const char *msg, const char *type) {
+static bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) {
cJSON *obj = cJSON_CreateObject();
cJSON *loc = cJSON_Parse(loc_json);
if (!obj || !loc) {
@@ -1067,21 +1153,19 @@ static void job_set_422(job *j, cJSON *detail) {
}
static void job_set_timing_header(job *j) {
- if (!j || j->extra_headers || !j->created_ns) return;
+ if (!j || j->extra_headers || !j->created_ns)
+ return;
uint64_t done_ns = nstime();
- uint64_t queue_ns = j->started_ns > j->created_ns
- ? j->started_ns - j->created_ns : 0;
- uint64_t worker_ns = j->started_ns && done_ns > j->started_ns
- ? done_ns - j->started_ns : 0;
+ uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0;
+ uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0;
char buf[384];
int n = snprintf(buf, sizeof(buf),
- "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
- "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
- "worker;dur=%.3f\r\n",
- ns_to_ms(queue_ns), ns_to_ms(j->parse_ns),
- ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns),
- ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
+ "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
+ "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
+ "worker;dur=%.3f\r\n",
+ ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns),
+ ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
if (n > 0 && (size_t)n < sizeof(buf))
j->extra_headers = xstrdup(buf);
}
@@ -1098,7 +1182,8 @@ static cJSON *parse_json_body(job *j, cJSON *detail) {
return NULL;
}
const char *body_end = j->body + j->body_len;
- while (end && end < body_end && isspace((unsigned char)*end)) end++;
+ while (end && end < body_end && isspace((unsigned char)*end))
+ end++;
if (!end || end != body_end) {
ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
cJSON_Delete(root);
@@ -1113,30 +1198,30 @@ static cJSON *parse_json_body(job *j, cJSON *detail) {
}
static bool cjson_is_integer(cJSON *item) {
- if (!cJSON_IsNumber(item)) return false;
+ if (!cJSON_IsNumber(item))
+ return false;
double d = item->valuedouble;
int i = item->valueint;
return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i;
}
-static const char *encoding_from_root(cJSON *root, cJSON *detail,
- embedding_api_t api) {
+static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
/* Default and accepted encodings follow the model's API family: Perplexity
* defaults to base64_int8 and accepts {base64_int8, base64_binary, float};
* OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */
const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8";
cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
- if (!encoding) return dflt;
+ if (!encoding)
+ return dflt;
if (!cJSON_IsString(encoding) || !encoding->valuestring) {
- ve_add(detail, "[\"body\",\"encoding_format\"]",
- "encoding_format must be a string", "type_error.string");
+ ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string",
+ "type_error.string");
return dflt;
}
const char *v = encoding->valuestring;
- int ok = api == EMBED_API_OPENAI
- ? (!strcmp(v, "float") || !strcmp(v, "base64"))
- : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
- !strcmp(v, "float"));
+ int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64"))
+ : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
+ !strcmp(v, "float"));
if (!ok) {
ve_add(detail, "[\"body\",\"encoding_format\"]",
api == EMBED_API_OPENAI
@@ -1149,8 +1234,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail,
return v;
}
-static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim,
- int max_dim, const char *encoding) {
+static int
+dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
int dims = max_dim;
cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
if (dimensions) {
@@ -1161,31 +1246,30 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim,
dims = dimensions->valueint;
if (dims < min_dim || dims > max_dim) {
char msg[96];
- snprintf(msg, sizeof(msg),
- "dimensions must be between %d and %d",
- min_dim, max_dim);
+ snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim,
+ max_dim);
ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
}
}
}
if (!strcmp(encoding, "base64_binary") && dims % 8 != 0)
ve_add(detail, "[\"body\",\"dimensions\"]",
- "dimensions must be divisible by 8 for base64_binary",
- "value_error.divisible");
+ "dimensions must be divisible by 8 for base64_binary", "value_error.divisible");
return dims;
}
static char *base64_encode(const unsigned char *src, size_t n) {
- static const char tbl[] =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
size_t out_len = ((n + 2) / 3) * 4;
char *out = xmalloc(out_len + 1);
size_t o = 0;
for (size_t i = 0; i < n; i += 3) {
unsigned v = (unsigned)src[i] << 16;
int remain = (int)(n - i);
- if (remain > 1) v |= (unsigned)src[i + 1] << 8;
- if (remain > 2) v |= (unsigned)src[i + 2];
+ if (remain > 1)
+ v |= (unsigned)src[i + 1] << 8;
+ if (remain > 2)
+ v |= (unsigned)src[i + 2];
out[o++] = tbl[(v >> 18) & 63];
out[o++] = tbl[(v >> 12) & 63];
out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '=';
@@ -1197,8 +1281,10 @@ static char *base64_encode(const unsigned char *src, size_t n) {
static signed char quantize_int8_tanh(float x) {
int v = (int)lrintf(tanhf(x) * 127.0f);
- if (v > 127) v = 127;
- if (v < -128) v = -128;
+ if (v > 127)
+ v = 127;
+ if (v < -128)
+ v = -128;
return (signed char)v;
}
@@ -1295,54 +1381,62 @@ typedef struct {
} rerank_request;
static void free_token_bufs(token_buf *t, int n) {
- if (!t) return;
- for (int i = 0; i < n; i++) free(t[i].ids);
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
}
static void embedding_request_free(embedding_request *r) {
- if (!r) return;
+ if (!r)
+ return;
free_token_bufs(r->tokens, r->n_inputs);
free(r->tokens);
free(r->inputs);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
static void contextual_request_free(contextual_request *r) {
- if (!r) return;
+ if (!r)
+ return;
for (int i = 0; i < r->n_docs; i++) {
free(r->docs[i].ids);
free(r->docs[i].spans);
}
free(r->docs);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
static void late_tokens_free(late_tokens *t) {
- if (!t) return;
+ if (!t)
+ return;
free(t->ids);
free(t->keep);
memset(t, 0, sizeof(*t));
}
static void rerank_request_free(rerank_request *r) {
- if (!r) return;
+ if (!r)
+ return;
late_tokens_free(&r->query);
for (int i = 0; i < r->n_documents; i++)
late_tokens_free(&r->documents[i]);
free(r->documents);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
-static int tokenize_one(loaded_model *m, job *j, const char *text,
- token_buf *out) {
+static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
memset(out, 0, sizeof(*out));
uint64_t t0 = nstime();
- out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws,
- text, &out->n_tokens);
- if (j) j->tokenize_ns += nstime() - t0;
+ out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
if (!out->ids || out->n_tokens <= 0) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -1354,8 +1448,7 @@ static int tokenize_one(loaded_model *m, job *j, const char *text,
memset(out, 0, sizeof(*out));
return -1;
}
- int *ids = realloc(
- out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
if (!ids) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -1369,15 +1462,17 @@ static int tokenize_one(loaded_model *m, job *j, const char *text,
static int late_id_is_skipped(const loaded_model *m, int id) {
for (int i = 0; i < m->n_late_skip_ids; i++) {
- if (m->late_skip_ids[i] == id) return 1;
+ if (m->late_skip_ids[i] == id)
+ return 1;
}
return 0;
}
-static int tokenize_late_text(loaded_model *m, job *j, const char *text,
- int is_query, late_tokens *out) {
+static int
+tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
token_buf raw = {0};
- if (tokenize_one(m, j, text, &raw) != 0) return -1;
+ if (tokenize_one(m, j, text, &raw) != 0)
+ return -1;
int raw_tokens = raw.n_tokens;
int target = raw_tokens + 1;
@@ -1386,11 +1481,9 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text,
out->ids = xmalloc((size_t)target * sizeof(*out->ids));
out->ids[0] = raw.ids[0];
- out->ids[1] = is_query ? m->late_query_prefix_id
- : m->late_document_prefix_id;
+ out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
if (raw_tokens > 1) {
- memcpy(out->ids + 2, raw.ids + 1,
- (size_t)(raw_tokens - 1) * sizeof(*out->ids));
+ memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
}
out->n_tokens = raw_tokens + 1;
if (is_query) {
@@ -1408,8 +1501,7 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text,
return out->n_keep > 0 ? 0 : -1;
}
-static int model_embed_batch(loaded_model *m, const embed_input_t *inputs,
- int batch, float *out) {
+static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
#ifdef USE_MLX
if (m->mlx_ctx)
return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
@@ -1423,7 +1515,8 @@ static int model_embed_batch(loaded_model *m, const embed_input_t *inputs,
static int model_embed_spans_batch(loaded_model *m,
const embed_context_input_t *inputs,
- int batch, float *out) {
+ int batch,
+ float *out) {
#ifdef USE_MLX
if (m->mlx_ctx)
return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
@@ -1432,8 +1525,7 @@ static int model_embed_spans_batch(loaded_model *m,
if (m->cuda_ctx)
return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
#endif
- return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch,
- out);
+ return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
}
static int model_uses_dense_batches(const loaded_model *m) {
@@ -1447,10 +1539,10 @@ static int model_uses_dense_batches(const loaded_model *m) {
/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
* pads each dense row to the longest input in the chunk. */
-static int inference_batch_accepts_input(const loaded_model *m, int batch,
- int packed_tokens, int next_tokens,
- int max_batch_tokens) {
- if (batch == 0) return 1;
+static int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
if (model_uses_dense_batches(m))
return next_tokens <= max_batch_tokens / (batch + 1);
return next_tokens <= max_batch_tokens - packed_tokens;
@@ -1466,17 +1558,14 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) {
* "float" renders a JSON array: the true float32 vector for OpenAI/DashScope
* (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every
* other encoding renders a base64 string via encode_embedding(). */
-static void append_embedding_value(sbuf *b, int index, const float *emb,
- int dims, const char *encoding,
- embedding_api_t api) {
+static void append_embedding_value(
+ sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) {
if (!strcmp(encoding, "float")) {
- sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[",
- index);
+ sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index);
for (int i = 0; i < dims; i++) {
- if (i) sbuf_putc(b, ',');
- float v = api == EMBED_API_OPENAI
- ? emb[i]
- : (float)quantize_int8_tanh(emb[i]) / 128.0f;
+ if (i)
+ sbuf_putc(b, ',');
+ float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f;
sbuf_printf(b, "%.9g", (double)v);
}
sbuf_puts(b, "]}");
@@ -1500,12 +1589,10 @@ static int render_embedding_response(embedding_request *r, const float *embs) {
const float *render_embs = embs;
float *truncated = NULL;
if (r->model->renormalize_truncated && r->dims < full_dim) {
- truncated = xmalloc(
- (size_t)r->n_inputs * r->dims * sizeof(*truncated));
+ truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated));
for (int i = 0; i < r->n_inputs; i++) {
float *dst = truncated + (size_t)i * r->dims;
- memcpy(dst, embs + (size_t)i * full_dim,
- (size_t)r->dims * sizeof(*dst));
+ memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst));
if (embed_l2_normalize(dst, r->dims) != 0) {
free(truncated);
return -1;
@@ -1519,25 +1606,23 @@ static int render_embedding_response(embedding_request *r, const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
for (int i = 0; i < r->n_inputs; i++) {
- if (i) sbuf_putc(&b, ',');
- append_embedding_value(&b, i, render_embs + (size_t)i * render_stride,
- r->dims, r->encoding, api);
+ if (i)
+ sbuf_putc(&b, ',');
+ append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding,
+ api);
}
sbuf_printf(&b,
- "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
+ "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
set_response_from_buf(r->j, &b);
free(truncated);
return 0;
}
-static int embedding_request_compatible(const embedding_request *a,
- const embedding_request *b) {
- return a->ready && b->ready &&
- a->model == b->model &&
- a->dims == b->dims &&
+static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
!strcmp(a->encoding, b->encoding);
}
@@ -1549,21 +1634,27 @@ typedef struct {
static int embedding_batch_item_cmp(const void *a, const void *b) {
const embedding_batch_item *ia = a;
const embedding_batch_item *ib = b;
- if (ia->input.n_tokens < ib->input.n_tokens) return -1;
- if (ia->input.n_tokens > ib->input.n_tokens) return 1;
+ if (ia->input.n_tokens < ib->input.n_tokens)
+ return -1;
+ if (ia->input.n_tokens > ib->input.n_tokens)
+ return 1;
return ia->output_index - ib->output_index;
}
static int embedding_request_fits_group(const embedding_request *r,
- int group_inputs, int group_tokens,
- int max_batch, int max_batch_tokens) {
- if (group_inputs == 0) return 1;
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_inputs == 0)
+ return 1;
return r->n_inputs <= max_batch - group_inputs &&
r->total_tokens <= max_batch_tokens - group_tokens;
}
static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
- if (n_reqs <= 0) return;
+ if (n_reqs <= 0)
+ return;
loaded_model *m = reqs[0]->model;
int dim = m->info->dim;
int total_inputs = 0;
@@ -1583,9 +1674,9 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
}
qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp);
- int max_batch = reqs[0]->j->srv->batch_size > 0
- ? reqs[0]->j->srv->batch_size : total_inputs;
- if (max_batch > total_inputs) max_batch = total_inputs;
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs;
+ if (max_batch > total_inputs)
+ max_batch = total_inputs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
@@ -1597,8 +1688,7 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
int tokens = 0;
while (start + cur < total_inputs && cur < max_batch) {
int n_tokens = items[start + cur].input.n_tokens;
- if (!inference_batch_accepts_input(m, cur, tokens, n_tokens,
- max_batch_tokens))
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens))
break;
inputs[cur] = items[start + cur].input;
tokens += n_tokens;
@@ -1612,8 +1702,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
break;
}
for (int i = 0; i < cur; i++)
- memcpy(embs + (size_t)items[start + i].output_index * dim,
- batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float));
+ memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim,
+ (size_t)dim * sizeof(float));
start += cur;
}
@@ -1624,10 +1714,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
} else {
uint64_t t0 = nstime();
- if (render_embedding_response(
- reqs[i], embs + (size_t)pos * dim) != 0)
- job_set_error(reqs[i]->j, 500, "embedding normalization failed",
- "server_error");
+ if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
+ job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
reqs[i]->j->encode_ns += nstime() - t0;
}
pos += reqs[i]->n_inputs;
@@ -1639,41 +1727,40 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
free(items);
}
-static void render_contextual_response(contextual_request *r,
- const float *embs) {
+static void render_contextual_response(contextual_request *r, const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
int span_offset = 0;
for (int di = 0; di < r->n_docs; di++) {
- if (di) sbuf_putc(&b, ',');
+ if (di)
+ sbuf_putc(&b, ',');
sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di);
for (int ci = 0; ci < r->docs[di].n_spans; ci++) {
- if (ci) sbuf_putc(&b, ',');
+ if (ci)
+ sbuf_putc(&b, ',');
const float *emb = embs + (size_t)span_offset * r->model->info->dim;
- append_embedding_value(&b, ci, emb, r->dims, r->encoding,
- r->model->info->api);
+ append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api);
span_offset++;
}
sbuf_puts(&b, "]}");
}
- sbuf_printf(&b,
- "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
+ sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
set_response_from_buf(r->j, &b);
}
-static int contextual_request_compatible(const contextual_request *a,
- const contextual_request *b) {
- return a->ready && b->ready &&
- a->model == b->model &&
- a->dims == b->dims &&
+static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
!strcmp(a->encoding, b->encoding);
}
static int contextual_request_fits_group(const contextual_request *r,
- int group_docs, int group_tokens,
- int max_batch, int max_batch_tokens) {
- if (group_docs == 0) return 1;
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_docs == 0)
+ return 1;
return r->n_docs <= max_batch - group_docs &&
r->total_tokens <= max_batch_tokens - group_tokens;
}
@@ -1687,14 +1774,16 @@ typedef struct {
static int contextual_batch_item_cmp(const void *a, const void *b) {
const contextual_batch_item *ia = a;
const contextual_batch_item *ib = b;
- if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1;
- if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1;
+ if (ia->input.input.n_tokens < ib->input.input.n_tokens)
+ return -1;
+ if (ia->input.input.n_tokens > ib->input.input.n_tokens)
+ return 1;
return ia->order - ib->order;
}
-static void execute_contextual_request_list(contextual_request **reqs,
- int n_reqs) {
- if (n_reqs <= 0) return;
+static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
loaded_model *m = reqs[0]->model;
int dim = m->info->dim;
int total_docs = 0;
@@ -1703,8 +1792,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
if (total_docs > INT_MAX - reqs[i]->n_docs ||
total_chunks > INT_MAX - reqs[i]->total_chunks) {
for (int k = 0; k < n_reqs; k++)
- job_set_error(reqs[k]->j, 500, "contextual batch is too large",
- "server_error");
+ job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error");
return;
}
total_docs += reqs[i]->n_docs;
@@ -1712,13 +1800,11 @@ static void execute_contextual_request_list(contextual_request **reqs,
}
if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
for (int i = 0; i < n_reqs; i++)
- job_set_error(reqs[i]->j, 500, "contextual batch is too large",
- "server_error");
+ job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error");
return;
}
- contextual_batch_item *items =
- xmalloc((size_t)total_docs * sizeof(*items));
+ contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items));
float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
int pos = 0;
int span_offset = 0;
@@ -1737,12 +1823,11 @@ static void execute_contextual_request_list(contextual_request **reqs,
}
qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
- int max_batch = reqs[0]->j->srv->batch_size > 0
- ? reqs[0]->j->srv->batch_size : total_docs;
- if (max_batch > total_docs) max_batch = total_docs;
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs;
+ if (max_batch > total_docs)
+ max_batch = total_docs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_context_input_t *inputs =
- xmalloc((size_t)max_batch * sizeof(*inputs));
+ embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc(sizeof(*batch_embs));
size_t batch_emb_cap = 0;
@@ -1754,8 +1839,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
int chunks = 0;
while (start + cur < total_docs && cur < max_batch) {
const embed_context_input_t *input = &items[start + cur].input;
- if (!inference_batch_accepts_input(m, cur, tokens,
- input->input.n_tokens,
+ if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
max_batch_tokens))
break;
inputs[cur] = *input;
@@ -1779,8 +1863,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
for (int i = 0; i < cur; i++) {
int n_spans = inputs[i].n_spans;
memcpy(embs + (size_t)items[start + i].output_span_index * dim,
- batch_embs + (size_t)batch_span * dim,
- (size_t)n_spans * dim * sizeof(*embs));
+ batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs));
batch_span += n_spans;
}
start += cur;
@@ -1790,12 +1873,10 @@ static void execute_contextual_request_list(contextual_request **reqs,
for (int i = 0; i < n_reqs; i++) {
reqs[i]->j->infer_ns += infer_ns;
if (failed) {
- job_set_error(reqs[i]->j, 500, "contextual embedding failed",
- "server_error");
+ job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error");
} else {
uint64_t t0 = nstime();
- render_contextual_response(reqs[i],
- embs + (size_t)span_offset * dim);
+ render_contextual_response(reqs[i], embs + (size_t)span_offset * dim);
reqs[i]->j->encode_ns += nstime() - t0;
}
span_offset += reqs[i]->total_chunks;
@@ -1807,9 +1888,13 @@ static void execute_contextual_request_list(contextual_request **reqs,
free(items);
}
-static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
- loaded_model **out_model, int *out_dims,
- const char **out_encoding, http_server *s) {
+static int validate_common(cJSON *root,
+ cJSON *detail,
+ model_kind expected_kind,
+ loaded_model **out_model,
+ int *out_dims,
+ const char **out_encoding,
+ http_server *s) {
cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
const char *model_id = NULL;
if (!model_item) {
@@ -1832,8 +1917,7 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
}
}
model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN;
- embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api
- : EMBED_API_PERPLEXITY;
+ embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY;
const char *enc = encoding_from_root(root, detail, api);
int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128;
int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560;
@@ -1841,11 +1925,12 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
*out_encoding = enc;
*out_dims = dims;
return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN &&
- !s->models[model_slot_for_id(model_id)].info ? 503 : 0;
+ !s->models[model_slot_for_id(model_id)].info
+ ? 503
+ : 0;
}
-static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
- embedding_request *out) {
+static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -1854,8 +1939,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD,
- &m, &dims, &encoding, s);
+ int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
@@ -1866,8 +1950,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (cJSON_IsString(input)) {
if (!input->valuestring || input->valuestring[0] == '\0')
- ve_add(detail, "[\"body\",\"input\"]", "input must not be empty",
- "value_error.empty");
+ ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty");
n_inputs = 1;
} else if (cJSON_IsArray(input)) {
n_inputs = cJSON_GetArraySize(input);
@@ -1890,13 +1973,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
i++;
}
} else {
- ve_add(detail, "[\"body\",\"input\"]",
- "input must be a string or an array of strings", "type_error");
+ ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings",
+ "type_error");
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -1916,8 +1998,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
out->inputs[0].n_tokens = out->tokens[0].n_tokens;
out->total_tokens = out->tokens[0].n_tokens;
if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]",
- "input exceeds 32768 token limit",
+ ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
"value_error.context_length");
}
} else {
@@ -1926,15 +2007,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, loc, "input item must be a string",
- "type_error.string");
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
idx++;
continue;
}
- if (tokenize_one(m, j, item->valuestring,
- &out->tokens[idx]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_one(m, j, item->valuestring, &out->tokens[idx]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
out->inputs[idx].ids = out->tokens[idx].ids;
out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
@@ -1962,8 +2040,8 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
cJSON_Delete(detail);
}
-static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
- contextual_request *out) {
+static void
+prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -1972,8 +2050,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL,
- &m, &dims, &encoding, s);
+ int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
@@ -1984,8 +2061,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
if (!input) {
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (!cJSON_IsArray(input)) {
- ve_add(detail, "[\"body\",\"input\"]",
- "input must be an array of document chunk arrays", "type_error.array");
+ ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays",
+ "type_error.array");
} else {
n_docs = cJSON_GetArraySize(input);
if (n_docs < 1)
@@ -2000,8 +2077,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
if (!cJSON_IsArray(doc_arr)) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
- ve_add(detail, loc, "document must be an array of strings",
- "type_error.array");
+ ve_add(detail, loc, "document must be an array of strings", "type_error.array");
di++;
continue;
}
@@ -2036,8 +2112,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -2051,8 +2126,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(doc_arr, input) {
contextual_doc *doc = &out->docs[di];
int n_chunks = cJSON_GetArraySize(doc_arr);
- token_buf *chunk_tokens =
- xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
+ token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
doc->n_spans = n_chunks;
int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
@@ -2063,10 +2137,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(chunk, doc_arr) {
char loc[80];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
- if (tokenize_one(m, j, chunk->valuestring,
- &chunk_tokens[ci]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
doc_valid = 0;
} else {
doc_tokens += chunk_tokens[ci].n_tokens;
@@ -2103,8 +2175,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
di++;
}
if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) {
- ve_add(detail, "[\"body\",\"input\"]",
- "request exceeds 120000 token limit",
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
"value_error.context_length");
} else {
out->total_tokens = (int)request_tokens;
@@ -2122,30 +2193,28 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_Delete(detail);
}
-static int validate_rerank_model(cJSON *root, cJSON *detail, http_server *s,
- loaded_model **out_model) {
+static int
+validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) {
cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model");
if (!item) {
ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
return 0;
}
if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, "[\"body\",\"model\"]", "model must be a string",
- "type_error.string");
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
return 0;
}
model_slot slot = model_slot_for_id(item->valuestring);
if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) {
- ve_add(detail, "[\"body\",\"model\"]",
- "value is not a valid enum member for this endpoint", "enum");
+ ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
+ "enum");
return 0;
}
*out_model = s->models[slot].info ? &s->models[slot] : NULL;
return *out_model ? 0 : 503;
}
-static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
- rerank_request *out) {
+static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -2157,28 +2226,24 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
if (!query) {
ve_add(detail, "[\"body\",\"query\"]", "field required", "missing");
} else if (!cJSON_IsString(query)) {
- ve_add(detail, "[\"body\",\"query\"]", "query must be a string",
- "type_error.string");
+ ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string");
} else if (!query->valuestring || !query->valuestring[0]) {
- ve_add(detail, "[\"body\",\"query\"]", "query must not be empty",
- "value_error.empty");
+ ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty");
}
int n_documents = 0;
if (!documents) {
ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing");
} else if (!cJSON_IsArray(documents)) {
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must be an array of strings", "type_error.array");
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings",
+ "type_error.array");
} else {
n_documents = cJSON_GetArraySize(documents);
if (n_documents < 1)
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must contain at least 1 item",
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
"value_error.list.min_items");
else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS)
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must contain at most 1000 items",
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
"value_error.list.max_items");
cJSON *document;
int i = 0;
@@ -2186,51 +2251,43 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
char loc[72];
snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++);
if (!cJSON_IsString(document))
- ve_add(detail, loc, "document must be a string",
- "type_error.string");
+ ve_add(detail, loc, "document must be a string", "type_error.string");
else if (!document->valuestring || !document->valuestring[0])
- ve_add(detail, loc, "document must not be empty",
- "value_error.empty");
+ ve_add(detail, loc, "document must not be empty", "value_error.empty");
}
}
cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n");
cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k");
if (top_n && top_k) {
- ve_add(detail, "[\"body\",\"top_n\"]",
- "top_n and top_k are aliases; provide only one",
+ ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one",
"value_error.conflict");
}
cJSON *top = top_n ? top_n : top_k;
out->top_n = n_documents;
if (top) {
if (!cjson_is_integer(top)) {
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]"
- : "[\"body\",\"top_k\"]",
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
"top_n must be an integer", "type_error.integer");
} else {
out->top_n = top->valueint;
if (out->top_n < 1 || out->top_n > n_documents)
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]"
- : "[\"body\",\"top_k\"]",
- "top_n must be between 1 and the number of documents",
- "value_error.range");
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be between 1 and the number of documents", "value_error.range");
}
}
- cJSON *return_documents =
- cJSON_GetObjectItemCaseSensitive(root, "return_documents");
+ cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents");
if (return_documents) {
if (!cJSON_IsBool(return_documents))
- ve_add(detail, "[\"body\",\"return_documents\"]",
- "return_documents must be a boolean", "type_error.bool");
+ ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean",
+ "type_error.bool");
else
out->return_documents = cJSON_IsTrue(return_documents);
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -2238,15 +2295,13 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
if (cJSON_GetArraySize(detail) == 0 && out->model) {
out->n_documents = n_documents;
out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
- if (tokenize_late_text(out->model, j, query->valuestring, 1,
- &out->query) != 0) {
+ if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) {
ve_add(detail, "[\"body\",\"query\"]", "tokenization failed",
"value_error.tokenization");
} else {
out->query_tokens = out->query.n_tokens;
if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"query\"]",
- "query exceeds 32768 token limit",
+ ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
"value_error.context_length");
}
@@ -2255,10 +2310,9 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(document, documents) {
char loc[72];
snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i);
- if (tokenize_late_text(out->model, j, document->valuestring, 0,
- &out->documents[i]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) !=
+ 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
int n = out->documents[i].n_tokens;
if (n > EMBED_API_MAX_ITEM_TOKENS)
@@ -2271,10 +2325,8 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
}
i++;
}
- if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS -
- out->document_tokens) {
- ve_add(detail, "[\"body\",\"documents\"]",
- "request exceeds 120000 token limit",
+ if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) {
+ ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
"value_error.context_length");
}
}
@@ -2297,8 +2349,10 @@ typedef struct {
static int rerank_result_cmp(const void *a, const void *b) {
const rerank_result *ra = a;
const rerank_result *rb = b;
- if (ra->score > rb->score) return -1;
- if (ra->score < rb->score) return 1;
+ if (ra->score > rb->score)
+ return -1;
+ if (ra->score < rb->score)
+ return 1;
return ra->index - rb->index;
}
@@ -2317,26 +2371,24 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) {
float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all));
int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
- int rc = embed_late_model_encode_tokens(
- m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens,
- 1, query);
+ int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
+ r->query.n_tokens, 1, query);
int pos = 0;
offsets[0] = 0;
for (int i = 0; rc == 0 && i < r->n_documents; i++) {
late_tokens *doc = &r->documents[i];
- rc = embed_late_model_encode_tokens(
- m->cpu_late_model, m->cpu_late_ws, doc->ids, doc->n_tokens, 1, all);
+ rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids,
+ doc->n_tokens, 1, all);
for (int k = 0; rc == 0 && k < doc->n_keep; k++) {
- memcpy(docs + (size_t)pos * dim,
- all + (size_t)doc->keep[k] * dim,
+ memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim,
(size_t)dim * sizeof(*docs));
pos++;
}
offsets[i + 1] = pos;
}
if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets,
- r->n_documents, dim, scores);
+ rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
+ scores);
free(offsets);
free(all);
free(docs);
@@ -2347,25 +2399,22 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) {
#ifdef USE_MLX
static int execute_rerank_mlx(rerank_request *r, float *scores) {
loaded_model *m = r->model;
- embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device(
- m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
- embed_mlx_late_vectors_t **docs =
- xcalloc((size_t)r->n_documents, sizeof(*docs));
- const embed_mlx_late_vectors_t **doc_refs =
- xcalloc((size_t)r->n_documents, sizeof(*doc_refs));
+ embed_mlx_late_vectors_t *query =
+ embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
+ embed_mlx_late_vectors_t **docs = xcalloc((size_t)r->n_documents, sizeof(*docs));
+ const embed_mlx_late_vectors_t **doc_refs = xcalloc((size_t)r->n_documents, sizeof(*doc_refs));
int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
int rc = query ? 0 : -1;
offsets[0] = 0;
for (int i = 0; rc == 0 && i < r->n_documents; i++) {
late_tokens *doc = &r->documents[i];
- embed_mlx_late_vectors_t *all = embed_mlx_late_encode_tokens_device(
- m->mlx_late_ctx, doc->ids, doc->n_tokens, 1);
+ embed_mlx_late_vectors_t *all =
+ embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, doc->ids, doc->n_tokens, 1);
if (!all) {
rc = -1;
break;
}
- docs[i] = embed_mlx_late_vectors_select(
- m->mlx_late_ctx, all, doc->keep, doc->n_keep);
+ docs[i] = embed_mlx_late_vectors_select(m->mlx_late_ctx, all, doc->keep, doc->n_keep);
embed_mlx_late_vectors_free(all);
if (!docs[i]) {
rc = -1;
@@ -2374,14 +2423,13 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) {
doc_refs[i] = docs[i];
offsets[i + 1] = offsets[i] + doc->n_keep;
}
- embed_mlx_late_vectors_t *packed = rc == 0
- ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs,
- r->n_documents)
- : NULL;
- if (!packed) rc = -1;
+ embed_mlx_late_vectors_t *packed =
+ rc == 0 ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, r->n_documents) : NULL;
+ if (!packed)
+ rc = -1;
if (rc == 0)
- rc = embed_mlx_late_maxsim_batch_device(
- m->mlx_late_ctx, query, packed, offsets, r->n_documents, scores);
+ rc = embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets,
+ r->n_documents, scores);
embed_mlx_late_vectors_free(packed);
for (int i = 0; i < r->n_documents; i++)
embed_mlx_late_vectors_free(docs[i]);
@@ -2396,29 +2444,24 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) {
static void execute_rerank_request(rerank_request *r) {
if (!r || !r->model) {
if (r && r->j)
- job_set_error(r->j, 500, "late-interaction model is unavailable",
- "server_error");
+ job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
return;
}
float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
uint64_t t0 = nstime();
#ifdef USE_MLX
- int rc = r->model->mlx_late_ctx
- ? execute_rerank_mlx(r, scores)
- : execute_rerank_cpu(r, scores);
+ int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
#else
int rc = execute_rerank_cpu(r, scores);
#endif
r->j->infer_ns += nstime() - t0;
if (rc != 0) {
free(scores);
- job_set_error(r->j, 500, "late-interaction reranking failed",
- "server_error");
+ job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
return;
}
- rerank_result *ranked =
- xmalloc((size_t)r->n_documents * sizeof(*ranked));
+ rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked));
for (int i = 0; i < r->n_documents; i++) {
ranked[i].index = i;
ranked[i].score = scores[i];
@@ -2427,14 +2470,13 @@ static void execute_rerank_request(rerank_request *r) {
t0 = nstime();
sbuf b = {0};
- sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[",
- r->model->info->id);
- cJSON *documents =
- cJSON_GetObjectItemCaseSensitive(r->root, "documents");
+ sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id);
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents");
for (int i = 0; i < r->top_n; i++) {
- if (i) sbuf_putc(&b, ',');
- sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g",
- ranked[i].index, (double)ranked[i].score);
+ if (i)
+ sbuf_putc(&b, ',');
+ sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index,
+ (double)ranked[i].score);
if (r->return_documents) {
cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index);
sbuf_puts(&b, ",\"document\":");
@@ -2443,10 +2485,9 @@ static void execute_rerank_request(rerank_request *r) {
sbuf_putc(&b, '}');
}
sbuf_printf(&b,
- "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->query_tokens, r->document_tokens,
- r->query_tokens + r->document_tokens);
+ "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens);
set_response_from_buf(r->j, &b);
r->j->encode_ns += nstime() - t0;
free(ranked);
@@ -2469,7 +2510,8 @@ static int parse_job_root(job *j, cJSON **out_root) {
static void process_unknown_job(job *j) {
cJSON *root = NULL;
- if (!j->started_ns) j->started_ns = nstime();
+ if (!j->started_ns)
+ j->started_ns = nstime();
uint64_t t0 = nstime();
int rc = parse_job_root(j, &root);
j->parse_ns += nstime() - t0;
@@ -2482,8 +2524,7 @@ static void process_unknown_job(job *j) {
static void process_job_group(http_server *s, job **jobs, int n_jobs) {
embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
- rerank_request *rerank_reqs =
- xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
+ rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
int *kind = xcalloc((size_t)n_jobs, sizeof(*kind));
int *done = xcalloc((size_t)n_jobs, sizeof(*done));
embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
@@ -2491,7 +2532,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int i = 0; i < n_jobs; i++) {
cJSON *root = NULL;
- if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime();
+ if (!jobs[i]->started_ns)
+ jobs[i]->started_ns = nstime();
if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
kind[i] = 1;
} else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
@@ -2526,8 +2568,7 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
continue;
}
- if ((kind[i] == 1 && !std_reqs[i].ready) ||
- (kind[i] == 2 && !ctx_reqs[i].ready) ||
+ if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) ||
(kind[i] == 3 && !rerank_reqs[i].ready)) {
job_set_timing_header(jobs[i]);
enqueue_done(jobs[i]);
@@ -2548,8 +2589,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
- embedding_request_fits_group(&std_reqs[k], group_inputs,
- group_tokens, s->batch_size, s->max_batch_tokens)) {
+ embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
std_group[group_n++] = &std_reqs[k];
group_inputs += std_reqs[k].n_inputs;
group_tokens += std_reqs[k].total_tokens;
@@ -2570,8 +2611,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
- contextual_request_fits_group(&ctx_reqs[k], group_docs,
- group_tokens, s->batch_size, s->max_batch_tokens)) {
+ contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
ctx_group[group_n++] = &ctx_reqs[k];
group_docs += ctx_reqs[k].n_docs;
group_tokens += ctx_reqs[k].total_tokens;
@@ -2601,26 +2642,19 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
free(std_reqs);
}
-static int configure_loaded_model(loaded_model *m,
- const embed_config_t *config,
- int token_dim)
-{
- if (!m || !m->info || !config ||
- config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim ||
- config->attention_mode != m->info->attention_mode ||
+static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size != m->info->dim ||
+ token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode ||
config->pooling_mode != m->info->pooling_mode ||
config->normalize_embeddings != m->info->normalize_embeddings) {
if (m && m->info && config)
- server_log("embed-server: model %s has incompatible config",
- m->info->id);
+ server_log("embed-server: model %s has incompatible config", m->info->id);
return -1;
}
m->append_terminal_token = config->append_terminal_token;
/* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
m->renormalize_truncated =
- config->normalize_embeddings &&
- config->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
return 0;
}
@@ -2631,28 +2665,27 @@ static void *worker_main(void *arg) {
int rc = 0;
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
- if (!m->info) continue;
+ if (!m->info)
+ continue;
embed_mlx_options_t mlx_opts = {
.quantize_bits = s->mlx_quantize_bits,
.quantize_group_size = s->mlx_quantize_group_size,
};
if (m->info->kind == MODEL_KIND_LATE)
- m->mlx_late_ctx =
- embed_mlx_late_load_with_options(m->path, &mlx_opts);
+ m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts);
else
m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts);
if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) ||
(!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load MLX model on worker: %s",
- m->path);
+ server_log("embed-server: failed to load MLX model on worker: %s", m->path);
rc = 1;
break;
}
const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_config(m->mlx_late_ctx)
- : embed_mlx_config(m->mlx_ctx);
- int token_dim = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
+ ? embed_mlx_late_config(m->mlx_late_ctx)
+ : embed_mlx_config(m->mlx_ctx);
+ int token_dim =
+ m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
if (configure_loaded_model(m, config, token_dim) != 0 ||
(m->info->kind == MODEL_KIND_LATE &&
(m->late_mask_id >= config->vocab_size ||
@@ -2665,7 +2698,8 @@ static void *worker_main(void *arg) {
pthread_mutex_lock(&s->mu);
s->worker_init_rc = rc;
s->worker_ready = 1;
- if (rc) s->stopping = 1;
+ if (rc)
+ s->stopping = 1;
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
@@ -2685,20 +2719,19 @@ static void *worker_main(void *arg) {
} else
#endif
#ifdef USE_CUDA
- if (s->use_cuda) {
+ if (s->use_cuda) {
int rc = 0;
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
- if (!m->info) continue;
+ if (!m->info)
+ continue;
m->cuda_ctx = embed_cuda_load(m->path);
if (!m->cuda_ctx) {
- server_log("embed-server: failed to load CUDA model on worker: %s",
- m->path);
+ server_log("embed-server: failed to load CUDA model on worker: %s", m->path);
rc = 1;
break;
}
- if (configure_loaded_model(
- m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
+ if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
rc = 1;
break;
}
@@ -2706,7 +2739,8 @@ static void *worker_main(void *arg) {
pthread_mutex_lock(&s->mu);
s->worker_init_rc = rc;
s->worker_ready = 1;
- if (rc) s->stopping = 1;
+ if (rc)
+ s->stopping = 1;
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
@@ -2730,9 +2764,9 @@ static void *worker_main(void *arg) {
for (;;) {
job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS];
- int n_jobs = collect_job_batch(
- s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
- if (n_jobs == 0) break;
+ int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
+ if (n_jobs == 0)
+ break;
process_job_group(s, jobs, n_jobs);
}
#ifdef USE_MLX
@@ -2792,8 +2826,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
m->tok_ws = qwen_tokenizer_workspace_new();
if (!m->tok_ws) {
- server_log("embed-server: failed to allocate tokenizer workspace: %s",
- path);
+ server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
return -1;
}
/* The contextual chunk separator is the tokenizer's <|endoftext|>. The
@@ -2802,8 +2835,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
* token (e.g. small test models) takes precedence so the id stays
* inside that model's embedding table. */
int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>");
- m->context_separator_id = sep_id >= 0
- ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
/* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
* suffix - the same token as the separator, not the model's chat
* eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
@@ -2812,16 +2844,13 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
int id = qwen_tokenizer_token_id(m->tok, "[MASK]");
m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
id = qwen_tokenizer_token_id(m->tok, "[Q]");
- m->late_query_prefix_id =
- id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
+ m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
id = qwen_tokenizer_token_id(m->tok, "[D]");
- m->late_document_prefix_id =
- id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
+ m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) /
- sizeof(m->late_skip_ids[0]));
+ for (const char *p = punct; *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) /
+ sizeof(m->late_skip_ids[0]));
p++) {
char text[2] = {*p, '\0'};
int n_ids = 0;
@@ -2854,8 +2883,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
if (!m->cpu_late_ws) {
- server_log("embed-server: failed to allocate late workspace: %s",
- path);
+ server_log("embed-server: failed to allocate late workspace: %s", path);
return -1;
}
config = embed_late_model_config(m->cpu_late_model);
@@ -2877,8 +2905,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
return -1;
}
if (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
+ (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
m->late_document_prefix_id >= config->vocab_size)) {
server_log("embed-server: late special-token ids exceed model vocab");
return -1;
@@ -2890,18 +2917,27 @@ static void free_models(http_server *s) {
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
free(m->path);
- if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws);
- if (m->tok) qwen_tokenizer_free(m->tok);
- if (m->cpu_ws) embed_workspace_free(m->cpu_ws);
- if (m->cpu_model) embed_model_free(m->cpu_model);
- if (m->cpu_late_ws) embed_late_workspace_free(m->cpu_late_ws);
- if (m->cpu_late_model) embed_late_model_free(m->cpu_late_model);
+ if (m->tok_ws)
+ qwen_tokenizer_workspace_free(m->tok_ws);
+ if (m->tok)
+ qwen_tokenizer_free(m->tok);
+ if (m->cpu_ws)
+ embed_workspace_free(m->cpu_ws);
+ if (m->cpu_model)
+ embed_model_free(m->cpu_model);
+ if (m->cpu_late_ws)
+ embed_late_workspace_free(m->cpu_late_ws);
+ if (m->cpu_late_model)
+ embed_late_model_free(m->cpu_late_model);
#ifdef USE_MLX
- if (m->mlx_ctx) embed_mlx_free(m->mlx_ctx);
- if (m->mlx_late_ctx) embed_mlx_late_free(m->mlx_late_ctx);
+ if (m->mlx_ctx)
+ embed_mlx_free(m->mlx_ctx);
+ if (m->mlx_late_ctx)
+ embed_mlx_late_free(m->mlx_late_ctx);
#endif
#ifdef USE_CUDA
- if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx);
+ if (m->cuda_ctx)
+ embed_cuda_free(m->cuda_ctx);
#endif
}
}
@@ -2910,14 +2946,12 @@ static uint64_t physical_memory_nbytes(void) {
#ifdef __APPLE__
uint64_t bytes = 0;
size_t len = sizeof(bytes);
- if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 &&
- len == sizeof(bytes) && bytes > 0)
+ if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0)
return bytes;
#endif
long pages = sysconf(_SC_PHYS_PAGES);
long page_size = sysconf(_SC_PAGESIZE);
- if (pages <= 0 || page_size <= 0 ||
- (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
+ if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
return 0;
return (uint64_t)pages * (uint64_t)page_size;
}
@@ -2928,13 +2962,11 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
const embed_server_model_spec_t *spec = &cfg->models[i];
model_slot slot = model_slot_for_id(spec->id);
if (slot == MODEL_UNKNOWN) {
- fprintf(stderr, "embed-server: unknown model id: %s\n",
- spec->id ? spec->id : "<null>");
+ fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : "<null>");
return -1;
}
if (!spec->path || !spec->path[0]) {
- fprintf(stderr, "embed-server: model %s has an empty path\n",
- spec->id);
+ fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id);
return -1;
}
if (seen[slot]) {
@@ -2942,8 +2974,7 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
return -1;
}
if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) {
- fprintf(stderr,
- "embed-server: model %s has no CUDA late-interaction backend\n",
+ fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n",
spec->id);
return -1;
}
@@ -2953,17 +2984,16 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
}
static int mlx_memory_preflight(const embed_server_config_t *cfg) {
- if (!cfg->use_mlx) return 0;
+ if (!cfg->use_mlx)
+ return 0;
uint64_t payload = 0;
for (int i = 0; i < cfg->n_models; i++) {
- multi_safetensors_t *ms =
- multi_safetensors_open(cfg->models[i].path);
+ multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path);
size_t model_payload = 0;
if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
multi_safetensors_close(ms);
- fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n",
- cfg->models[i].path);
+ fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path);
return -1;
}
multi_safetensors_close(ms);
@@ -2985,22 +3015,21 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) {
"skipping MLX memory preflight");
return 0;
}
- double utilization = cfg->memory_utilization > 0.0
- ? cfg->memory_utilization
- : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
+ double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization
+ : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
uint64_t budget = (uint64_t)((double)physical * utilization);
const double gib = 1024.0 * 1024.0 * 1024.0;
server_log("embed-server: MLX memory preflight: %.1f GiB tensors, "
"%.1f GiB estimated resident, %.1f GiB budget "
"(%.2f of physical memory)",
- (double)payload / gib, (double)estimated / gib,
- (double)budget / gib, utilization);
+ (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization);
if (cfg->mlx_quantize_bits) {
server_log("embed-server: MLX %d-bit quantization enabled; preflight "
"uses source payload as a conservative peak estimate",
cfg->mlx_quantize_bits);
}
- if (estimated <= budget) return 0;
+ if (estimated <= budget)
+ return 0;
server_log("embed-server: refusing MLX model set above the host-memory "
"budget");
server_log("embed-server: use BF16 artifacts, load fewer models, or raise "
@@ -3020,24 +3049,23 @@ int embed_run_server(const embed_server_config_t *cfg) {
memset(&s, 0, sizeof(s));
s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
s.port = cfg->port > 0 ? cfg->port : 8000;
- s.batch_size = cfg->batch_size > 0
- ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
- s.max_batch_tokens = cfg->max_batch_tokens > 0
- ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
- s.batch_wait_us = cfg->batch_wait_us >= 0
- ? cfg->batch_wait_us
- : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
- : EMBED_SERVER_BATCH_WAIT_US);
+ s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
+ s.max_batch_tokens =
+ cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
+ s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us
+ : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
+ : EMBED_SERVER_BATCH_WAIT_US);
s.use_mlx = cfg->use_mlx;
s.use_cuda = cfg->use_cuda;
s.mlx_quantize_bits = cfg->mlx_quantize_bits;
- s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0
- ? cfg->mlx_quantize_group_size : 64;
+ s.mlx_quantize_group_size =
+ cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);
else {
const char *env_key = getenv("EMBED_API_KEY");
- if (env_key && env_key[0]) s.api_key = xstrdup(env_key);
+ if (env_key && env_key[0])
+ s.api_key = xstrdup(env_key);
}
pthread_mutex_init(&s.mu, NULL);
@@ -3124,7 +3152,8 @@ int embed_run_server(const embed_server_config_t *cfg) {
aeMain(s.loop);
server_log("embed-server: shutdown requested");
- if (s.listen_fd >= 0) close(s.listen_fd);
+ if (s.listen_fd >= 0)
+ close(s.listen_fd);
client *c = s.clients;
while (c) {
client *next = c->next;
@@ -3155,59 +3184,58 @@ int embed_run_server(const embed_server_config_t *cfg) {
* ==================================================================== */
#ifndef EMBED_SERVER_TEST
-static void print_usage(const char *prog)
-{
+static void print_usage(const char *prog) {
fprintf(stderr,
- "Usage: %s --model ID=DIR [options]\n"
- "\n"
- "Options:\n"
- " --model ID=DIR Model to serve (repeatable)\n"
+ "Usage: %s --model ID=DIR [options]\n"
+ "\n"
+ "Options:\n"
+ " --model ID=DIR Model to serve (repeatable)\n"
#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
+ " --mlx Use Apple MLX GPU backend\n"
#endif
#ifdef USE_CUDA
- " --cuda Use NVIDIA CUDA GPU backend\n"
+ " --cuda Use NVIDIA CUDA GPU backend\n"
#endif
- " --host HOST Bind host (default: 127.0.0.1)\n"
- " --port N Bind port (default: 8000)\n"
- " --api-key K Require Authorization: Bearer K\n"
- " -b, --batch-size N Max texts or documents per inference batch\n"
- " (default: 32)\n"
- " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
- " --batch-wait-us N First-arrival micro-batch deadline in us\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 32)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N First-arrival micro-batch deadline in us\n"
#ifdef USE_CUDA
- " (default: CUDA 1000; CPU 0)\n"
+ " (default: CUDA 1000; CPU 0)\n"
#else
- " (default: 0)\n"
+ " (default: 0)\n"
#endif
#ifdef USE_MLX
- " --memory-utilization F Fraction of physical memory the MLX model-set\n"
- " preflight may plan for (default: 0.90; values\n"
- " above 1.0 overcommit)\n"
- " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
- " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
+ " --memory-utilization F Fraction of physical memory the MLX model-set\n"
+ " preflight may plan for (default: 0.90; values\n"
+ " above 1.0 overcommit)\n"
+ " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
+ " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
#endif
#ifdef USE_CUDA
- " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
- " (default: f32, exact)\n"
- " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
- " f32); bf16 halves weight memory\n"
+ " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
+ " (default: f32, exact)\n"
+ " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
+ " f32); bf16 halves weight memory\n"
#endif
- " -t, --threads N CPU threads (default: all cores)\n"
- " -v, --verbose Verbose (-vv for debug)\n"
- " -V, --version Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h, --help Show this help\n"
- "\n"
- "Examples:\n"
- " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " -t, --threads N CPU threads (default: all cores)\n"
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -V, --version Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
#ifdef USE_MLX
- " %s --mlx --mlx-quant-bits 8 \\\n"
- " --model pplx-embed-v1-4b=./model-4b-bf16\n",
- prog, prog, prog);
+ " %s --mlx --mlx-quant-bits 8 \\\n"
+ " --model pplx-embed-v1-4b=./model-4b-bf16\n",
+ prog, prog, prog);
#else
- ,
- prog, prog);
+ ,
+ prog, prog);
#endif
}
@@ -3217,8 +3245,7 @@ typedef struct {
int cap;
} model_specs_t;
-static int append_model_spec(model_specs_t *specs, const char *arg)
-{
+static int append_model_spec(model_specs_t *specs, const char *arg) {
const char *eq = strchr(arg, '=');
if (!eq || eq == arg || !eq[1]) {
fprintf(stderr, "--model expects MODEL_ID=DIR\n");
@@ -3227,7 +3254,8 @@ static int append_model_spec(model_specs_t *specs, const char *arg)
if (specs->n == specs->cap) {
int new_cap = specs->cap ? specs->cap * 2 : 4;
void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p) return -1;
+ if (!p)
+ return -1;
specs->v = p;
specs->cap = new_cap;
}
@@ -3246,9 +3274,9 @@ static int append_model_spec(model_specs_t *specs, const char *arg)
return 0;
}
-static void free_model_specs(model_specs_t *specs)
-{
- if (!specs) return;
+static void free_model_specs(model_specs_t *specs) {
+ if (!specs)
+ return;
for (int i = 0; i < specs->n; i++) {
free((char *)specs->v[i].id);
free((char *)specs->v[i].path);
@@ -3256,8 +3284,7 @@ static void free_model_specs(model_specs_t *specs)
free(specs->v);
}
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
int n_threads = 0;
int verbose = 0;
int batch_size = 0;
@@ -3287,37 +3314,34 @@ int main(int argc, char *argv[])
embed_print_version(prog);
free_model_specs(&model_specs);
return 0;
- }
- else if (!strcmp(f, "--build-info")) {
+ } else if (!strcmp(f, "--build-info")) {
embed_print_build_info(prog);
free_model_specs(&model_specs);
return 0;
- }
- else if (!strcmp(f, "--model")) {
+ } else if (!strcmp(f, "--model")) {
if (append_model_spec(&model_specs, argv[++arg]) != 0) {
free_model_specs(&model_specs);
return 1;
}
}
#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
- else if (!strcmp(f, "--mlx-quant-bits")) {
+ else if (!strcmp(f, "--mlx")) {
+ use_mlx = 1;
+ } else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg]);
if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
+ } else if (!strcmp(f, "--mlx-quant-group-size")) {
mlx_quantize_group_size = atoi(argv[++arg]);
if (mlx_quantize_group_size <= 0) {
fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--memory-utilization")) {
+ } else if (!strcmp(f, "--memory-utilization")) {
memory_utilization = atof(argv[++arg]);
if (memory_utilization <= 0.0) {
fprintf(stderr, "--memory-utilization must be > 0\n");
@@ -3327,47 +3351,47 @@ int main(int argc, char *argv[])
}
#endif
#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
- else if (!strcmp(f, "--cuda-gemm-mode")) {
+ else if (!strcmp(f, "--cuda")) {
+ use_cuda = 1;
+ } else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
+ } else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg];
}
#endif
- else if (!strcmp(f, "--host")) { host = argv[++arg]; }
- else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
- else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ else if (!strcmp(f, "--host")) {
+ host = argv[++arg];
+ } else if (!strcmp(f, "--port")) {
+ port = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--api-key")) {
+ api_key = argv[++arg];
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "--max-batch-tokens")) {
+ } else if (!strcmp(f, "--max-batch-tokens")) {
max_batch_tokens = atoi(argv[++arg]);
if (max_batch_tokens <= 0) {
fprintf(stderr, "--max-batch-tokens must be > 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--batch-wait-us")) {
+ } else if (!strcmp(f, "--batch-wait-us")) {
batch_wait_us = atoi(argv[++arg]);
if (batch_wait_us < 0) {
fprintf(stderr, "--batch-wait-us must be >= 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
n_threads = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(prog);
free_model_specs(&model_specs);
return 0;
- }
- else {
+ } else {
fprintf(stderr, "unknown option: %s\n", f);
print_usage(prog);
free_model_specs(&model_specs);
@@ -3431,18 +3455,22 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (use_mlx) {
- if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ if (verbose >= 1)
+ fprintf(stderr, "Using MLX GPU backend\n");
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ if (use_cuda) {
+ if (verbose >= 1)
+ fprintf(stderr, "Using CUDA GPU backend\n");
} else
#endif
{
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ if (n_threads <= 0)
+ n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ if (verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
}
embed_server_config_t scfg = {