aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-30 04:49:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:59 +0200
commit5bab6ce54b51d019c480e6f88a446d9c8a127602 (patch)
tree680b4efd1d97463cef456c74443d7ba935a61bf7 /main.c
parentab50d908ba2eae14929a07341b1b716cde681c1c (diff)
downloadffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.tar.gz
ffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.zip
server: add embeddings micro-batching
Diffstat (limited to 'main.c')
-rw-r--r--main.c16
1 files changed, 14 insertions, 2 deletions
diff --git a/main.c b/main.c
index 1eba511..6c9f25f 100644
--- a/main.c
+++ b/main.c
@@ -40,7 +40,9 @@ static void print_usage(const char *prog)
" --port N Server bind port (default: 8000)\n"
" --cors Enable CORS headers in --serve mode\n"
" --api-key K Require Authorization: Bearer K in --serve mode\n"
- " -b <n> Max texts per engine batch (default: all; stdin/server: 1)\n"
+ " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n"
+ " --batch-wait-us N\n"
+ " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n"
" -t <n> CPU threads (default: all cores)\n"
" -e Print raw embeddings (with multiple texts)\n"
" -v Verbose (-vv for debug)\n"
@@ -466,6 +468,7 @@ int main(int argc, char *argv[])
int stdin_mode = 0;
int serve_mode = 0;
int batch_size = 0;
+ int batch_wait_us = -1;
const char *backend = "cpu";
const char *host = "127.0.0.1";
const char *api_key = NULL;
@@ -481,6 +484,14 @@ int main(int argc, char *argv[])
else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
}
+ else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg_start]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
else if (!strcmp(f, "--model")) {
if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1;
}
@@ -557,7 +568,8 @@ int main(int argc, char *argv[])
.n_models = model_specs.n,
.host = host,
.port = port,
- .batch_size = batch_size > 0 ? batch_size : 1,
+ .batch_size = batch_size > 0 ? batch_size : 8,
+ .batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
.enable_cors = cors,
.api_key = api_key,