diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 02:29:52 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 02:29:52 +0200 |
| commit | dc29ac64a203a954161026ad5f6288df201e11ad (patch) | |
| tree | 019f84f6a15b53b6134d75ef7f539f986a918fda | |
| parent | cb85a499f3d5178a752c85063bf9ff277ca3a71e (diff) | |
| download | ffwd-dc29ac64a203a954161026ad5f6288df201e11ad.tar.gz ffwd-dc29ac64a203a954161026ad5f6288df201e11ad.zip | |
server: fix tokenizer data race under concurrency
| -rw-r--r-- | tools/server/schedule.c | 11 | ||||
| -rw-r--r-- | tools/server/server.c | 11 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 8 |
3 files changed, 8 insertions, 22 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c index 8fb4231..13e5dc7 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -182,17 +182,6 @@ int worker_has_pending_jobs(http_server *s) { return pending; } -/* True when work is already queued anywhere ahead of the worker (untokenized or - * tokenized). dispatch routes a new request to the tokenizer thread when this - * holds, so tokenization overlaps in-flight work; an otherwise idle pipeline - * tokenizes the request inline on the worker and skips the queue hand-off. */ -int server_has_backlog(http_server *s) { - pthread_mutex_lock(&s->mu); - int busy = s->raw_head != NULL || s->job_head != NULL; - pthread_mutex_unlock(&s->mu); - return busy; -} - void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); diff --git a/tools/server/server.c b/tools/server/server.c index 7a71a1f..f18d264 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -106,13 +106,10 @@ void dispatch_request(client *c) { j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; j->created_ns = nstime(); client_incref(c); - /* Tokenize off the worker when the pipeline is already busy so it overlaps - * in-flight work; tokenize inline on the worker when idle to avoid a - * queue hand-off that single-stream requests gain nothing from. */ - if (server_has_backlog(c->srv)) - enqueue_raw_job(j); - else - enqueue_job(j); + /* All requests go through the tokenizer thread. Tokenization must run on a + * single thread because the model tokenizer is shared mutable state; letting + * the worker also tokenize would race it. */ + enqueue_raw_job(j); } static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index b332404..df6b9ea 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -105,9 +105,10 @@ typedef struct { int worker_init_rc; job *job_head; job *job_tail; - /* Tokenizer stage: dispatch enqueues a raw job here when the pipeline is - * busy, and a dedicated thread parses + tokenizes it off the worker so - * tokenization overlaps the GPU instead of serializing in front of it. */ + /* Tokenizer stage: dispatch enqueues every raw job here, and a single + * dedicated thread parses + tokenizes it off the worker so tokenization + * overlaps the GPU. One thread, so the shared model tokenizer is never + * touched concurrently. */ pthread_t tokenizer; pthread_cond_t raw_cv; int raw_stopping; @@ -330,7 +331,6 @@ void execute_rerank_request(rerank_request *r); /* ---- server_schedule.c: job queue, micro-batching, completion ---- */ void enqueue_job(job *j); int worker_has_pending_jobs(http_server *s); -int server_has_backlog(http_server *s); void enqueue_raw_job(job *j); job *dequeue_raw_job(http_server *s); void tokenize_job(http_server *s, job *j); |
