diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 05:19:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 05:19:05 +0200 |
| commit | d717f607807a931d3352af8778f17779a3b6f71a (patch) | |
| tree | 36e92522fc106b661fc89106aa85b18fb699ca64 | |
| parent | 6214651cc06845417e6546a6df23a3da1a821327 (diff) | |
| download | ffwd-d717f607807a931d3352af8778f17779a3b6f71a.tar.gz ffwd-d717f607807a931d3352af8778f17779a3b6f71a.zip | |
server: version flags, drop cost block, tidy batch-wait note
| -rw-r--r-- | embed_server.c | 65 | ||||
| -rw-r--r-- | tools/openapi.yaml | 7 |
2 files changed, 35 insertions, 37 deletions
diff --git a/embed_server.c b/embed_server.c index 97a99a5..8c7743c 100644 --- a/embed_server.c +++ b/embed_server.c @@ -2,6 +2,7 @@ #define _GNU_SOURCE #endif +#include "embed_build.h" #include "embed_server.h" #include "embed.h" #include "qwen_safetensors.h" @@ -70,6 +71,10 @@ * was no faster and inflated long-document tail latency. */ #define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 #define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +/* Microseconds the worker waits for more requests before dispatching a batch. + * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU + * busy; MLX and CPU dispatch immediately, where waiting only adds latency. + * Override with --batch-wait-us. */ #define EMBED_SERVER_BATCH_WAIT_US 0 #define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 #define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 @@ -239,7 +244,6 @@ typedef struct { int dim; int min_dim; int token_dim; - double price_per_mtok; embed_attention_mode_t attention_mode; embed_pooling_mode_t pooling_mode; int normalize_embeddings; @@ -247,21 +251,21 @@ typedef struct { } model_info; static const model_info k_models[] = { - {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, 0.004, + {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, 0.030, + {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, 0.0, + {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, 0.0, + {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, 0.0, + {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, 0.008, + {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, 0.050, + {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, 0.0, + {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, }; @@ -1519,22 +1523,10 @@ static int render_embedding_response(embedding_request *r, const float *embs) { append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, api); } - /* Perplexity reports a per-request cost block; OpenAI/DashScope (Qwen3) - * usage is just the token counts, so keep the Qwen3 envelope OpenAI-shaped. */ - if (api == EMBED_API_OPENAI) { - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); - } else { - double cost = ((double)r->total_tokens / 1000000.0) * - r->model->info->price_per_mtok; - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d,\"cost\":{\"input_cost\":%.10g," - "\"total_cost\":%.10g,\"currency\":\"USD\"}}}", - r->model->info->id, r->total_tokens, r->total_tokens, cost, cost); - } + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); set_response_from_buf(r->j, &b); free(truncated); @@ -3203,6 +3195,8 @@ static void print_usage(const char *prog) #endif " -t, --threads N CPU threads (default: all cores)\n" " -v, --verbose Verbose (-vv for debug)\n" + " -V, --version Print version and exit\n" + " --build-info Print build details and exit\n" " -h, --help Show this help\n" "\n" "Examples:\n" @@ -3285,10 +3279,21 @@ int main(int argc, char *argv[]) int port = 8000; model_specs_t model_specs = {0}; + const char *prog = embed_prog_name(argv[0]); int arg = 1; while (arg < argc && argv[arg][0] == '-') { const char *f = argv[arg]; - if (!strcmp(f, "--model")) { + if (!strcmp(f, "-V") || !strcmp(f, "--version")) { + embed_print_version(prog); + free_model_specs(&model_specs); + return 0; + } + else if (!strcmp(f, "--build-info")) { + embed_print_build_info(prog); + free_model_specs(&model_specs); + return 0; + } + else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg]) != 0) { free_model_specs(&model_specs); return 1; @@ -3358,13 +3363,13 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(argv[0]); + print_usage(prog); free_model_specs(&model_specs); return 0; } else { fprintf(stderr, "unknown option: %s\n", f); - print_usage(argv[0]); + print_usage(prog); free_model_specs(&model_specs); return 1; } @@ -3376,13 +3381,13 @@ int main(int argc, char *argv[]) if (arg < argc) { fprintf(stderr, "unexpected argument: %s\n", argv[arg]); - print_usage(argv[0]); + print_usage(prog); free_model_specs(&model_specs); return 1; } if (model_specs.n == 0) { fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); - print_usage(argv[0]); + print_usage(prog); free_model_specs(&model_specs); return 1; } diff --git a/tools/openapi.yaml b/tools/openapi.yaml index 8a4b96f..3381dca 100644 --- a/tools/openapi.yaml +++ b/tools/openapi.yaml @@ -227,13 +227,6 @@ components: properties: prompt_tokens: { type: integer } total_tokens: { type: integer } - cost: - type: object - description: pplx-embed only; Qwen3-Embedding omits it. - properties: - input_cost: { type: number } - total_cost: { type: number } - currency: { type: string, example: USD } ValidationDetail: type: object properties: |
