diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-30 04:49:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:59 +0200 |
| commit | 5bab6ce54b51d019c480e6f88a446d9c8a127602 (patch) | |
| tree | 680b4efd1d97463cef456c74443d7ba935a61bf7 /main.c | |
| parent | ab50d908ba2eae14929a07341b1b716cde681c1c (diff) | |
| download | ffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.tar.gz ffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.zip | |
server: add embeddings micro-batching
Diffstat (limited to 'main.c')
| -rw-r--r-- | main.c | 16 |
1 files changed, 14 insertions, 2 deletions
@@ -40,7 +40,9 @@ static void print_usage(const char *prog) " --port N Server bind port (default: 8000)\n" " --cors Enable CORS headers in --serve mode\n" " --api-key K Require Authorization: Bearer K in --serve mode\n" - " -b <n> Max texts per engine batch (default: all; stdin/server: 1)\n" + " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n" + " --batch-wait-us N\n" + " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n" " -t <n> CPU threads (default: all cores)\n" " -e Print raw embeddings (with multiple texts)\n" " -v Verbose (-vv for debug)\n" @@ -466,6 +468,7 @@ int main(int argc, char *argv[]) int stdin_mode = 0; int serve_mode = 0; int batch_size = 0; + int batch_wait_us = -1; const char *backend = "cpu"; const char *host = "127.0.0.1"; const char *api_key = NULL; @@ -481,6 +484,14 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); } + else if (!strcmp(f, "--batch-wait-us")) { + batch_wait_us = atoi(argv[++arg_start]); + if (batch_wait_us < 0) { + fprintf(stderr, "--batch-wait-us must be >= 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1; } @@ -557,7 +568,8 @@ int main(int argc, char *argv[]) .n_models = model_specs.n, .host = host, .port = port, - .batch_size = batch_size > 0 ? batch_size : 1, + .batch_size = batch_size > 0 ? batch_size : 8, + .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, .enable_cors = cors, .api_key = api_key, |
