aboutsummaryrefslogtreecommitdiff
path: root/tools/server/server.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-24 00:53:21 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-24 00:53:21 +0200
commit214d6aa4e641296dba6040ab776d91baefdca82e (patch)
treecd185930075edf792fc014d30ce6df9c87a36418 /tools/server/server.c
parentf182896ac7febe75d2ea81425782fc11d2117278 (diff)
downloadffwd-214d6aa4e641296dba6040ab776d91baefdca82e.tar.gz
ffwd-214d6aa4e641296dba6040ab776d91baefdca82e.zip
server: tokenize requests off the worker thread
Diffstat (limited to 'tools/server/server.c')
-rw-r--r--tools/server/server.c89
1 files changed, 88 insertions, 1 deletions
diff --git a/tools/server/server.c b/tools/server/server.c
index 8762c4f..63438b9 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -66,6 +66,17 @@ static bool auth_ok(http_server *s, const char *auth) {
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
}
+/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the
+ * tokenizer thread so parsing and tokenization overlap the GPU instead of
+ * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0
+ * keeps tokenization on the worker for an A/B comparison. */
+static int tokenize_off_worker_default(void) {
+ const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER");
+ if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
+ return 0;
+ return 1;
+}
+
void dispatch_request(client *c) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -106,7 +117,10 @@ void dispatch_request(client *c) {
j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
j->created_ns = nstime();
client_incref(c);
- enqueue_job(j);
+ if (c->srv->tokenize_off_worker)
+ enqueue_raw_job(j);
+ else
+ enqueue_job(j);
}
static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
@@ -215,6 +229,30 @@ static void *worker_main(void *arg) {
return NULL;
}
+/* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the
+ * worker's job queue. Waits for the worker to finish model init first, because
+ * a GPU build finalizes the model config (dims, pooling) on the worker and
+ * request validation reads it. Idle when tokenize-off-worker is disabled. */
+static void *tokenizer_main(void *arg) {
+ http_server *s = arg;
+ pthread_mutex_lock(&s->mu);
+ while (!s->worker_ready)
+ pthread_cond_wait(&s->cv, &s->mu);
+ int rc = s->worker_init_rc;
+ pthread_mutex_unlock(&s->mu);
+ if (rc)
+ return NULL;
+
+ for (;;) {
+ job *j = dequeue_raw_job(s);
+ if (!j)
+ break;
+ tokenize_job(s, j);
+ enqueue_job(j);
+ }
+ return NULL;
+}
+
static void *renderer_main(void *arg) {
http_server *s = arg;
for (;;) {
@@ -316,6 +354,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.max_batch_tokens =
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
+ s.tokenize_off_worker = tokenize_off_worker_default();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);
@@ -327,6 +366,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
pthread_mutex_init(&s.mu, NULL);
pthread_cond_init(&s.cv, NULL);
+ pthread_cond_init(&s.raw_cv, NULL);
pthread_cond_init(&s.render_cv, NULL);
for (int i = 0; i < cfg->n_models; i++) {
@@ -334,6 +374,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -345,6 +386,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -369,6 +411,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -380,6 +423,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -396,6 +440,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -420,6 +465,33 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
+
+ /* Started after worker init so the tokenizer sees a finalized model config
+ * (a GPU build sets dims/pooling on the worker, and request validation reads
+ * them). Joined before the worker at shutdown so it drains raw jobs into the
+ * job queue first. */
+ if (pthread_create(&s.tokenizer, NULL, tokenizer_main, &s) != 0) {
+ fprintf(stderr, "ffwd-server: failed to start tokenizer\n");
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -428,6 +500,11 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.listen_fd = listen_on(s.host, s.port);
if (s.listen_fd < 0) {
pthread_mutex_lock(&s.mu);
+ s.raw_stopping = 1;
+ pthread_cond_signal(&s.raw_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.tokenizer, NULL);
+ pthread_mutex_lock(&s.mu);
s.stopping = 1;
pthread_cond_signal(&s.cv);
pthread_mutex_unlock(&s.mu);
@@ -441,6 +518,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -464,6 +542,14 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
c = next;
}
+ /* Drain stages front to back: the tokenizer flushes raw jobs into the job
+ * queue, then the worker drains that, then the renderer drains finished
+ * jobs, so nothing is stranded mid-pipeline. */
+ pthread_mutex_lock(&s.mu);
+ s.raw_stopping = 1;
+ pthread_cond_signal(&s.raw_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.tokenizer, NULL);
pthread_mutex_lock(&s.mu);
s.stopping = 1;
pthread_cond_signal(&s.cv);
@@ -483,6 +569,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 0;