diff options
Diffstat (limited to 'src/server.c')
| -rw-r--r-- | src/server.c | 1042 |
1 files changed, 0 insertions, 1042 deletions
diff --git a/src/server.c b/src/server.c deleted file mode 100644 index 808c6f8..0000000 --- a/src/server.c +++ /dev/null @@ -1,1042 +0,0 @@ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include "build.h" -#include "server.h" -#include "server_internal.h" -#include "server_util.h" -#include "sbuf.h" -#include "embed.h" -#include "safetensors.h" -#include "kernels.h" -#include "tokenizer_bpe.h" -#include "tokenizer_wordpiece.h" - -#ifdef USE_MLX -#include "mlx.h" -#endif -#ifdef USE_CUDA -#include "cuda.h" -#endif - -#include "deps/ae/ae.h" -#include "deps/ae/anet.h" - -#include <arpa/inet.h> -#include <ctype.h> -#include <errno.h> -#include <fcntl.h> -#include <limits.h> -#include <netinet/in.h> -#include <pthread.h> -#include <signal.h> -#include <stdbool.h> -#include <stdint.h> -#include <stdarg.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <strings.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <sys/types.h> -#include <time.h> -#include <unistd.h> - -#ifdef __APPLE__ -#include <sys/sysctl.h> -#endif - -#include <cjson/cJSON.h> - -static volatile sig_atomic_t g_stop_requested = 0; -static volatile sig_atomic_t g_signal_wfd = -1; - -static void stop_signal_handler(int sig) { - (void)sig; - if (g_stop_requested) - _exit(130); - int saved = errno; - g_stop_requested = 1; - int fd = (int)g_signal_wfd; - if (fd >= 0) { - const unsigned char byte = 1; - if (write(fd, &byte, 1) < 0) { - /* best-effort stop wakeup; a failed self-pipe write is ignored */ - } - } - errno = saved; -} - -static bool auth_ok(http_server *s, const char *auth) { - if (!s->api_key || !s->api_key[0]) - return true; - if (!auth) - return false; - const char prefix[] = "Bearer "; - return !strncmp(auth, prefix, sizeof(prefix) - 1) && - !strcmp(auth + sizeof(prefix) - 1, s->api_key); -} - -void dispatch_request(client *c) { - aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - - if (!strcmp(c->req.method, "OPTIONS")) { - append_http_response(c, 204, NULL, "", 0); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - if (!route_is_inference(c->req.method, c->req.path)) { - size_t len; - char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); - append_http_response(c, 404, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - if (!auth_ok(c->srv, c->req.auth)) { - size_t len; - char *body = - json_error_body("missing or invalid bearer token", "authentication_error", &len); - append_http_response(c, 401, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - job *j = xcalloc(1, sizeof(*j)); - j->srv = c->srv; - j->c = c; - snprintf(j->method, sizeof(j->method), "%s", c->req.method); - snprintf(j->path, sizeof(j->path), "%s", c->req.path); - j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); - j->body_len = c->req.body_len; - j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; - j->created_ns = nstime(); - client_incref(c); - enqueue_job(j); -} - -static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; - (void)mask; - http_server *s = clientData; - char cip[64]; - int cport = 0; - for (;;) { - int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); - if (cfd == ANET_ERR) { - if (errno == EAGAIN || errno == EWOULDBLOCK) - return; - if (errno == EINTR) - continue; - return; - } - set_nonblock(cfd); - client *c = xcalloc(1, sizeof(*c)); - c->srv = s; - c->fd = cfd; - c->refcount = 1; - c->last_active_ms = mstime(); - c->linked = 1; - c->next = s->clients; - if (s->clients) - s->clients->prev = c; - s->clients = c; - s->n_clients++; - if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) - close_client(c); - } -} - -static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { - (void)loop; - (void)id; - http_server *s = clientData; - uint64_t now = mstime(); - client *c = s->clients; - while (c) { - client *next = c->next; - /* refcount > 1 means a request from this client is queued or being - * processed by the worker; long batch queues (e.g. 4B long-document - * batches) must not be reaped as idle. */ - if (!c->cancelled && c->refcount <= 1 && - now - c->last_active_ms > EMBED_HTTP_CLIENT_TIMEOUT_MS) - close_client(c); - c = next; - } - return EMBED_HTTP_SWEEP_MS; -} - -static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)fd; - (void)mask; - http_server *s = clientData; - unsigned char tmp[128]; - while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { - } - aeStop(loop); -} - -static void *worker_main(void *arg) { - http_server *s = arg; -#ifdef USE_MLX - if (s->use_mlx) { - int rc = 0; - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - embed_mlx_options_t mlx_opts = { - .quantize_bits = s->mlx_quantize_bits, - .quantize_group_size = s->mlx_quantize_group_size, - }; - if (m->info->kind == MODEL_KIND_LATE) - m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts); - else - m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts); - if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) || - (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load MLX model on worker: %s", m->path); - rc = 1; - break; - } - const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_config(m->mlx_late_ctx) - : embed_mlx_config(m->mlx_ctx); - int token_dim = - m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; - if (configure_loaded_model(m, config, token_dim) != 0 || - (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || - m->late_document_prefix_id >= config->vocab_size))) { - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) - s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - embed_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - if (m->mlx_late_ctx) { - embed_mlx_late_free(m->mlx_late_ctx); - m->mlx_late_ctx = NULL; - } - } - return NULL; - } - } else -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - int rc = 0; - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->info->kind == MODEL_KIND_LATE) - m->cuda_late_ctx = embed_cuda_late_load(m->path); - else - m->cuda_ctx = embed_cuda_load(m->path); - if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) || - (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load CUDA model on worker: %s", m->path); - rc = 1; - break; - } - const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_cuda_late_config(m->cuda_late_ctx) - : embed_cuda_config(m->cuda_ctx); - int token_dim = - m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0; - if (configure_loaded_model(m, config, token_dim) != 0 || - (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || - m->late_document_prefix_id >= config->vocab_size))) { - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) - s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - embed_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - if (m->cuda_late_ctx) { - embed_cuda_late_free(m->cuda_late_ctx); - m->cuda_late_ctx = NULL; - } - } - return NULL; - } - } else -#endif - { - pthread_mutex_lock(&s->mu); - s->worker_ready = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - } - - for (;;) { - job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); - if (n_jobs == 0) - break; - process_job_group(s, jobs, n_jobs); - } -#ifdef USE_MLX - if (s->use_mlx) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - embed_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - if (m->mlx_late_ctx) { - embed_mlx_late_free(m->mlx_late_ctx); - m->mlx_late_ctx = NULL; - } - } - } -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - embed_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - if (m->cuda_late_ctx) { - embed_cuda_late_free(m->cuda_late_ctx); - m->cuda_late_ctx = NULL; - } - } - } -#endif - return NULL; -} - -static int listen_on(const char *host, int port) { - char portbuf[32]; - snprintf(portbuf, sizeof(portbuf), "%d", port); - char err[ANET_ERR_LEN]; - int fd = anetTcpServer(err, port, (char *)host, EMBED_HTTP_BACKLOG); - (void)portbuf; - if (fd == ANET_ERR) { - server_log("embed-server: listen failed on %s:%d: %s", host, port, err); - return -1; - } - set_nonblock(fd); - return fd; -} - -static uint64_t physical_memory_nbytes(void) { -#ifdef __APPLE__ - uint64_t bytes = 0; - size_t len = sizeof(bytes); - if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) - return bytes; -#endif - long pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGESIZE); - if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) - return 0; - return (uint64_t)pages * (uint64_t)page_size; -} - -static int validate_model_specs(const embed_server_config_t *cfg) { - for (int i = 0; i < cfg->n_models; i++) { - const embed_server_model_spec_t *spec = &cfg->models[i]; - if (!spec->id || !spec->id[0]) { - fprintf(stderr, "embed-server: model label must not be empty\n"); - return -1; - } - if (!spec->path || !spec->path[0]) { - fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); - return -1; - } - if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) { - fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id); - return -1; - } - if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) { - fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id); - return -1; - } - if (spec->min_dim < 0) { - fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id); - return -1; - } - for (int j = 0; j < i; j++) { - if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) { - fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id); - return -1; - } - } - } - return 0; -} - -static int mlx_memory_preflight(const embed_server_config_t *cfg) { - if (!cfg->use_mlx) - return 0; - - uint64_t payload = 0; - for (int i = 0; i < cfg->n_models; i++) { - multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path); - size_t model_payload = 0; - if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { - multi_safetensors_close(ms); - fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path); - return -1; - } - multi_safetensors_close(ms); - if ((uint64_t)model_payload > UINT64_MAX - payload) { - fprintf(stderr, "embed-server: MLX model payload size overflow\n"); - return -1; - } - payload += (uint64_t)model_payload; - } - - if (payload > UINT64_MAX / EMBED_MLX_RESIDENT_MULTIPLIER) { - fprintf(stderr, "embed-server: MLX resident-memory estimate overflow\n"); - return -1; - } - uint64_t estimated = payload * EMBED_MLX_RESIDENT_MULTIPLIER; - uint64_t physical = physical_memory_nbytes(); - if (physical == 0) { - server_log("embed-server: warning: could not determine physical memory; " - "skipping MLX memory preflight"); - return 0; - } - double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization - : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; - uint64_t budget = (uint64_t)((double)physical * utilization); - const double gib = 1024.0 * 1024.0 * 1024.0; - server_log("embed-server: MLX memory preflight: %.1f GiB tensors, " - "%.1f GiB estimated resident, %.1f GiB budget " - "(%.2f of physical memory)", - (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); - if (cfg->mlx_quantize_bits) { - server_log("embed-server: MLX %d-bit quantization enabled; preflight " - "uses source payload as a conservative peak estimate", - cfg->mlx_quantize_bits); - } - if (estimated <= budget) - return 0; - server_log("embed-server: refusing MLX model set above the host-memory " - "budget"); - server_log("embed-server: use BF16 artifacts, load fewer models, or raise " - "--memory-utilization (values above 1.0 overcommit)"); - return -1; -} - -int embed_run_server(const embed_server_config_t *cfg) { - if (!cfg || !cfg->models || cfg->n_models <= 0) { - fprintf(stderr, "embed-server: at least one model label and path is required\n"); - return 1; - } - if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) - return 1; - - http_server s; - memset(&s, 0, sizeof(s)); - s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; - s.port = cfg->port > 0 ? cfg->port : 8000; - s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; - s.max_batch_tokens = - cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; - s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us - : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US - : EMBED_SERVER_BATCH_WAIT_US); - s.use_mlx = cfg->use_mlx; - s.use_cuda = cfg->use_cuda; - s.mlx_quantize_bits = cfg->mlx_quantize_bits; - s.mlx_quantize_group_size = - cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; - if (cfg->api_key && cfg->api_key[0]) - s.api_key = xstrdup(cfg->api_key); - else { - const char *env_key = getenv("EMBED_API_KEY"); - if (env_key && env_key[0]) - s.api_key = xstrdup(env_key); - } - - pthread_mutex_init(&s.mu, NULL); - pthread_cond_init(&s.cv, NULL); - - for (int i = 0; i < cfg->n_models; i++) { - if (load_one_model(&s, &cfg->models[i]) != 0) { - free_models(&s); - return 1; - } - } - - if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { - perror("pipe"); - free_models(&s); - return 1; - } - set_nonblock(s.completion_pipe[0]); - set_nonblock(s.completion_pipe[1]); - set_nonblock(s.signal_pipe[0]); - set_nonblock(s.signal_pipe[1]); - - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = stop_signal_handler; - sigemptyset(&sa.sa_mask); - sigaction(SIGINT, &sa, NULL); - sigaction(SIGTERM, &sa, NULL); - signal(SIGPIPE, SIG_IGN); - g_signal_wfd = s.signal_pipe[1]; - - s.loop = aeCreateEventLoop(EMBED_HTTP_SETSIZE); - if (!s.loop) { - fprintf(stderr, "embed-server: failed to create event loop\n"); - free_models(&s); - return 1; - } - - if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { - fprintf(stderr, "embed-server: failed to start worker\n"); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - pthread_mutex_lock(&s.mu); - while (!s.worker_ready) - pthread_cond_wait(&s.cv, &s.mu); - int worker_init_rc = s.worker_init_rc; - pthread_mutex_unlock(&s.mu); - if (worker_init_rc) { - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 1; - } - - s.listen_fd = listen_on(s.host, s.port); - if (s.listen_fd < 0) { - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - - aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); - aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); - aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); - aeCreateTimeEvent(s.loop, EMBED_HTTP_SWEEP_MS, timeout_cb, &s, NULL); - - server_log("embed-server: listening on http://%s:%d", s.host, s.port); - aeMain(s.loop); - - server_log("embed-server: shutdown requested"); - if (s.listen_fd >= 0) - close(s.listen_fd); - client *c = s.clients; - while (c) { - client *next = c->next; - close_client(c); - c = next; - } - - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - - aeDeleteEventLoop(s.loop); - close(s.completion_pipe[0]); - close(s.completion_pipe[1]); - close(s.signal_pipe[0]); - close(s.signal_pipe[1]); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 0; -} - -/* ==================================================================== - * embed-server entry point - * ==================================================================== */ -#ifndef EMBED_SERVER_TEST - -static void print_usage(const char *prog) { - fprintf(stderr, - "Usage: %s --model LABEL=DIR [options]\n" - "\n" - "Options:\n" - " --model LABEL=DIR[:key=val]\n" - " Standard embedding model (repeatable)\n" - " --contextual-model LABEL=DIR[:key=val]\n" - " Contextual embedding model (repeatable)\n" - " --late-model LABEL=DIR[:key=val]\n" - " Late-interaction rerank model (repeatable)\n" - " keys: api=openai|perplexity, min_dim=N,\n" - " query=TEXT (last key; \\n escapes newline)\n" -#ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" -#endif -#ifdef USE_CUDA - " --cuda Use NVIDIA CUDA GPU backend\n" -#endif - " --host HOST Bind host (default: 127.0.0.1)\n" - " --port N Bind port (default: 8000)\n" - " --api-key K Require Authorization: Bearer K\n" - " -b, --batch-size N Max texts or documents per inference batch\n" - " (default: 32)\n" - " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N First-arrival micro-batch deadline in us\n" -#ifdef USE_CUDA - " (default: CUDA 1000; CPU 0)\n" -#else - " (default: 0)\n" -#endif -#ifdef USE_MLX - " --memory-utilization F Fraction of physical memory the MLX model-set\n" - " preflight may plan for (default: 0.90; values\n" - " above 1.0 overcommit)\n" - " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" - " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" -#endif -#ifdef USE_CUDA - " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" - " (default: f32, exact)\n" - " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" - " f32); bf16 halves weight memory\n" -#endif - " -t, --threads N CPU threads (default: available cores, cgroup-aware)\n" - " -v, --verbose Verbose (-vv for debug)\n" - " -V, --version Print version and exit\n" - " --build-info Print build details and exit\n" - " -h, --help Show this help\n" - "\n" - "Examples:\n" - " %s --model docs=./model --port 8000\n" - " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n" -#ifdef USE_MLX - " %s --mlx --mlx-quant-bits 8 \\\n" - " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n", - prog, prog, prog, prog); -#else - , - prog, prog, prog); -#endif -} - -typedef struct { - embed_server_model_spec_t *v; - int n; - int cap; -} model_specs_t; - -static char *dup_range(const char *p, size_t n) { - char *s = (char *)malloc(n + 1); - if (!s) - return NULL; - memcpy(s, p, n); - s[n] = '\0'; - return s; -} - -static char *unescape_model_option(const char *s) { - size_t n = strlen(s); - char *out = (char *)malloc(n + 1); - if (!out) - return NULL; - size_t w = 0; - for (size_t r = 0; r < n; r++) { - if (s[r] == '\\' && r + 1 < n) { - r++; - if (s[r] == 'n') - out[w++] = '\n'; - else if (s[r] == 't') - out[w++] = '\t'; - else if (s[r] == 'r') - out[w++] = '\r'; - else - out[w++] = s[r]; - } else { - out[w++] = s[r]; - } - } - out[w] = '\0'; - return out; -} - -static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) { - const char *eq = strchr(opt, '='); - if (!eq || eq == opt || !eq[1]) { - fprintf(stderr, "%s option expects key=value: %s\n", flag, opt); - return -1; - } - size_t key_len = (size_t)(eq - opt); - const char *val = eq + 1; - if (key_len == 3 && strncmp(opt, "api", key_len) == 0) { - if (!strcmp(val, "openai")) { - spec->api = EMBED_SERVER_API_OPENAI; - } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) { - spec->api = EMBED_SERVER_API_PERPLEXITY; - } else if (!strcmp(val, "default")) { - spec->api = EMBED_SERVER_API_DEFAULT; - } else { - fprintf(stderr, "%s api must be openai or perplexity\n", flag); - return -1; - } - return 0; - } - if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) { - char *end = NULL; - long v = strtol(val, &end, 10); - if (!end || *end || v <= 0 || v > INT_MAX) { - fprintf(stderr, "%s min_dim must be a positive integer\n", flag); - return -1; - } - spec->min_dim = (int)v; - return 0; - } - fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt); - return -1; -} - -static int append_model_spec(model_specs_t *specs, - const char *arg, - embed_server_model_kind_t kind, - const char *flag) { - const char *eq = strchr(arg, '='); - if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "%s expects LABEL=DIR\n", flag); - return -1; - } - if (specs->n == specs->cap) { - int new_cap = specs->cap ? specs->cap * 2 : 4; - void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) - return -1; - specs->v = p; - specs->cap = new_cap; - } - size_t id_len = (size_t)(eq - arg); - const char *path_start = eq + 1; - const char *opts = strchr(path_start, ':'); - size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start); - if (path_len == 0) { - fprintf(stderr, "%s expects a non-empty DIR\n", flag); - return -1; - } - embed_server_model_spec_t spec; - memset(&spec, 0, sizeof(spec)); - spec.id = dup_range(arg, id_len); - spec.path = dup_range(path_start, path_len); - spec.kind = kind; - if (!spec.id || !spec.path) { - free((char *)spec.id); - free((char *)spec.path); - return -1; - } - const char *p = opts ? opts + 1 : NULL; - while (p && *p) { - if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) { - const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15); - spec.query_instruct = unescape_model_option(val); - if (!spec.query_instruct) - goto fail; - break; - } - const char *next = strchr(p, ':'); - size_t opt_len = next ? (size_t)(next - p) : strlen(p); - char *opt = dup_range(p, opt_len); - if (!opt) - goto fail; - int rc = parse_model_option(&spec, opt, flag); - free(opt); - if (rc != 0) - goto fail; - p = next ? next + 1 : NULL; - } - specs->v[specs->n] = spec; - specs->n++; - return 0; - -fail: - free((char *)spec.id); - free((char *)spec.path); - free((char *)spec.query_instruct); - return -1; -} - -static void free_model_specs(model_specs_t *specs) { - if (!specs) - return; - for (int i = 0; i < specs->n; i++) { - free((char *)specs->v[i].id); - free((char *)specs->v[i].path); - free((char *)specs->v[i].query_instruct); - } - free(specs->v); -} - -int main(int argc, char *argv[]) { - int n_threads = 0; - int verbose = 0; - int batch_size = 0; - int max_batch_tokens = 0; - int batch_wait_us = -1; -#ifdef USE_MLX - int use_mlx = 0; - int mlx_quantize_bits = 0; - int mlx_quantize_group_size = 64; - double memory_utilization = 0.0; -#endif -#ifdef USE_CUDA - int use_cuda = 0; - const char *cuda_fast_gemm = NULL; - const char *cuda_weights = NULL; -#endif - const char *host = "127.0.0.1"; - const char *api_key = NULL; - int port = 8000; - model_specs_t model_specs = {0}; - - const char *prog = embed_prog_name(argv[0]); - int arg = 1; - while (arg < argc && argv[arg][0] == '-') { - const char *f = argv[arg]; - if (!strcmp(f, "-V") || !strcmp(f, "--version")) { - embed_print_version(prog); - free_model_specs(&model_specs); - return 0; - } else if (!strcmp(f, "--build-info")) { - embed_print_build_info(prog); - free_model_specs(&model_specs); - return 0; - } else if (!strcmp(f, "--model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) != - 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--contextual-model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) != - 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--late-model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) { - free_model_specs(&model_specs); - return 1; - } - } -#ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { - use_mlx = 1; - } else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--memory-utilization")) { - memory_utilization = atof(argv[++arg]); - if (memory_utilization <= 0.0) { - fprintf(stderr, "--memory-utilization must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } -#endif -#ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { - use_cuda = 1; - } else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg]; - } else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg]; - } -#endif - else if (!strcmp(f, "--host")) { - host = argv[++arg]; - } else if (!strcmp(f, "--port")) { - port = atoi(argv[++arg]); - } else if (!strcmp(f, "--api-key")) { - api_key = argv[++arg]; - } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { - batch_size = atoi(argv[++arg]); - } else if (!strcmp(f, "--max-batch-tokens")) { - max_batch_tokens = atoi(argv[++arg]); - if (max_batch_tokens <= 0) { - fprintf(stderr, "--max-batch-tokens must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--batch-wait-us")) { - batch_wait_us = atoi(argv[++arg]); - if (batch_wait_us < 0) { - fprintf(stderr, "--batch-wait-us must be >= 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { - n_threads = atoi(argv[++arg]); - } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { - verbose++; - } else if (!strcmp(f, "-vv")) { - verbose = 2; - } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(prog); - free_model_specs(&model_specs); - return 0; - } else { - fprintf(stderr, "unknown option: %s\n", f); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } - arg++; - } - - embed_verbose = verbose; - - if (arg < argc) { - fprintf(stderr, "unexpected argument: %s\n", argv[arg]); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } - if (model_specs.n == 0) { - fprintf(stderr, "at least one --model LABEL=DIR is required\n"); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } -#ifdef USE_MLX - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); - free_model_specs(&model_specs); - return 1; - } -#endif -#ifdef USE_CUDA - if (cuda_fast_gemm) { - if (!use_cuda) { - fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); - free_model_specs(&model_specs); - return 1; - } - } - if (cuda_weights) { - if (!use_cuda) { - fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (!strcmp(cuda_weights, "bf16")) { - embed_cuda_set_weights_bf16(1); - } else if (!strcmp(cuda_weights, "f32")) { - embed_cuda_set_weights_bf16(0); - } else { - fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); - free_model_specs(&model_specs); - return 1; - } - } -#endif - -#ifdef USE_MLX - if (use_mlx) { - if (verbose >= 1) - fprintf(stderr, "Using MLX GPU backend\n"); - } else -#endif -#ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) - fprintf(stderr, "Using CUDA GPU backend\n"); - } else -#endif - { - if (n_threads <= 0) - n_threads = embed_get_num_cpus(); - embed_set_threads(n_threads); - if (verbose >= 1) - fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } - - embed_server_config_t scfg = { - .models = model_specs.v, - .n_models = model_specs.n, - .host = host, - .port = port, - .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE, - .max_batch_tokens = max_batch_tokens, - .batch_wait_us = batch_wait_us, -#ifdef USE_MLX - .use_mlx = use_mlx, - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .memory_utilization = memory_utilization, -#endif -#ifdef USE_CUDA - .use_cuda = use_cuda, -#endif - .api_key = api_key, - }; - int rc = embed_run_server(&scfg); - free_model_specs(&model_specs); - return rc; -} - -#endif /* EMBED_SERVER_TEST */ |
