aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-24 02:29:52 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-24 02:29:52 +0200
commitdc29ac64a203a954161026ad5f6288df201e11ad (patch)
tree019f84f6a15b53b6134d75ef7f539f986a918fda /tools
parentcb85a499f3d5178a752c85063bf9ff277ca3a71e (diff)
downloadffwd-dc29ac64a203a954161026ad5f6288df201e11ad.tar.gz
ffwd-dc29ac64a203a954161026ad5f6288df201e11ad.zip
server: fix tokenizer data race under concurrency
Diffstat (limited to 'tools')
-rw-r--r--tools/server/schedule.c11
-rw-r--r--tools/server/server.c11
-rw-r--r--tools/server/server_internal.h8
3 files changed, 8 insertions, 22 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index 8fb4231..13e5dc7 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -182,17 +182,6 @@ int worker_has_pending_jobs(http_server *s) {
return pending;
}
-/* True when work is already queued anywhere ahead of the worker (untokenized or
- * tokenized). dispatch routes a new request to the tokenizer thread when this
- * holds, so tokenization overlaps in-flight work; an otherwise idle pipeline
- * tokenizes the request inline on the worker and skips the queue hand-off. */
-int server_has_backlog(http_server *s) {
- pthread_mutex_lock(&s->mu);
- int busy = s->raw_head != NULL || s->job_head != NULL;
- pthread_mutex_unlock(&s->mu);
- return busy;
-}
-
void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
diff --git a/tools/server/server.c b/tools/server/server.c
index 7a71a1f..f18d264 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -106,13 +106,10 @@ void dispatch_request(client *c) {
j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
j->created_ns = nstime();
client_incref(c);
- /* Tokenize off the worker when the pipeline is already busy so it overlaps
- * in-flight work; tokenize inline on the worker when idle to avoid a
- * queue hand-off that single-stream requests gain nothing from. */
- if (server_has_backlog(c->srv))
- enqueue_raw_job(j);
- else
- enqueue_job(j);
+ /* All requests go through the tokenizer thread. Tokenization must run on a
+ * single thread because the model tokenizer is shared mutable state; letting
+ * the worker also tokenize would race it. */
+ enqueue_raw_job(j);
}
static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index b332404..df6b9ea 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -105,9 +105,10 @@ typedef struct {
int worker_init_rc;
job *job_head;
job *job_tail;
- /* Tokenizer stage: dispatch enqueues a raw job here when the pipeline is
- * busy, and a dedicated thread parses + tokenizes it off the worker so
- * tokenization overlaps the GPU instead of serializing in front of it. */
+ /* Tokenizer stage: dispatch enqueues every raw job here, and a single
+ * dedicated thread parses + tokenizes it off the worker so tokenization
+ * overlaps the GPU. One thread, so the shared model tokenizer is never
+ * touched concurrently. */
pthread_t tokenizer;
pthread_cond_t raw_cv;
int raw_stopping;
@@ -330,7 +331,6 @@ void execute_rerank_request(rerank_request *r);
/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
void enqueue_job(job *j);
int worker_has_pending_jobs(http_server *s);
-int server_has_backlog(http_server *s);
void enqueue_raw_job(job *j);
job *dequeue_raw_job(http_server *s);
void tokenize_job(http_server *s, job *j);