From 76758db35873a8f236e6ea0df544e71391edc4b1 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 24 Jun 2026 03:10:47 +0200 Subject: server: bound merged batches by token budget --- tools/server/server_internal.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tools/server/server_internal.h') diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index df6b9ea..315acdd 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -103,6 +103,10 @@ typedef struct { int stopping; int worker_ready; int worker_init_rc; + /* Worker-only hint: current collected window still has unprocessed jobs, so + * response rendering can overlap that local work even if the shared queue + * is empty. */ + int worker_local_backlog; job *job_head; job *job_tail; /* Tokenizer stage: dispatch enqueues every raw job here, and a single -- cgit v1.3.1