aboutsummaryrefslogtreecommitdiff
path: root/tools/server/server.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-24 01:51:53 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-24 01:51:53 +0200
commitcb85a499f3d5178a752c85063bf9ff277ca3a71e (patch)
tree78f68836e954d08dd6409e6d56cf44a7fb3c35e7 /tools/server/server.c
parent716d2051de27123bb298dbe2536c23537ec9512d (diff)
downloadffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.tar.gz
ffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.zip
server: make tokenize/render overlap adaptive, drop env switches
Diffstat (limited to 'tools/server/server.c')
-rw-r--r--tools/server/server.c19
1 files changed, 5 insertions, 14 deletions
diff --git a/tools/server/server.c b/tools/server/server.c
index 63438b9..7a71a1f 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -66,17 +66,6 @@ static bool auth_ok(http_server *s, const char *auth) {
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
}
-/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the
- * tokenizer thread so parsing and tokenization overlap the GPU instead of
- * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0
- * keeps tokenization on the worker for an A/B comparison. */
-static int tokenize_off_worker_default(void) {
- const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER");
- if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
- return 0;
- return 1;
-}
-
void dispatch_request(client *c) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -117,7 +106,10 @@ void dispatch_request(client *c) {
j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
j->created_ns = nstime();
client_incref(c);
- if (c->srv->tokenize_off_worker)
+ /* Tokenize off the worker when the pipeline is already busy so it overlaps
+ * in-flight work; tokenize inline on the worker when idle to avoid a
+ * queue hand-off that single-stream requests gain nothing from. */
+ if (server_has_backlog(c->srv))
enqueue_raw_job(j);
else
enqueue_job(j);
@@ -232,7 +224,7 @@ static void *worker_main(void *arg) {
/* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the
* worker's job queue. Waits for the worker to finish model init first, because
* a GPU build finalizes the model config (dims, pooling) on the worker and
- * request validation reads it. Idle when tokenize-off-worker is disabled. */
+ * request validation reads it. Idle while no request is routed through it. */
static void *tokenizer_main(void *arg) {
http_server *s = arg;
pthread_mutex_lock(&s->mu);
@@ -354,7 +346,6 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.max_batch_tokens =
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
- s.tokenize_off_worker = tokenize_off_worker_default();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);