aboutsummaryrefslogtreecommitdiff
path: root/tools/server/schedule.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-04 03:19:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-04 03:19:20 +0200
commit6a3dbc6edce2daae2dfb5a217e9e1318804000f4 (patch)
treef08d8b43b0ce39f620410eeb0772ba0078192597 /tools/server/schedule.c
parented23a1571fe6911eba9d4727bef30c82df7be06d (diff)
downloadffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.tar.gz
ffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.zip
server: bound the collect window by the batch token budget
Diffstat (limited to 'tools/server/schedule.c')
-rw-r--r--tools/server/schedule.c27
1 files changed, 27 insertions, 0 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index c71cad3..68b32e9 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -131,6 +131,23 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t
return pthread_cond_timedwait(cv, mu, &abs);
}
+/* Tokenized size of a queued job, for bounding the collect window. Jobs reach
+ * the queue through the tokenizer thread, so prep is already filled; a parse
+ * or validation failure leaves prep NULL and counts as 0. */
+static int64_t job_prep_tokens(const job *j) {
+ if (!j->prep)
+ return 0;
+ if (j->prep_kind == 1)
+ return ((const embedding_request *)j->prep)->total_tokens;
+ if (j->prep_kind == 2)
+ return ((const contextual_request *)j->prep)->total_tokens;
+ if (j->prep_kind == 3) {
+ const rerank_request *r = j->prep;
+ return (int64_t)r->query_tokens + r->document_tokens;
+ }
+ return 0;
+}
+
int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
if (max_jobs <= 0)
return 0;
@@ -159,6 +176,7 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
uint64_t deadline_ns =
first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
+ int64_t window_tokens = job_prep_tokens(first);
while (n < max_jobs) {
while (!s->job_head && !s->stopping && wait_ns > 0) {
int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
@@ -169,6 +187,15 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
break;
if (wait_ns > 0 && s->job_head->created_ns > deadline_ns)
break;
+ /* Stop the window at one token budget so the worker returns to the
+ * queue after about one execution group. Requests that arrive while a
+ * group runs then join the very next window instead of waiting for
+ * everything collected here to finish. */
+ int64_t next_tokens = job_prep_tokens(s->job_head);
+ if (s->max_batch_tokens > 0 && window_tokens > 0 && next_tokens > 0 &&
+ window_tokens + next_tokens > s->max_batch_tokens)
+ break;
+ window_tokens += next_tokens;
jobs[n++] = pop_job_locked(s);
if (wait_ns == 0 && !s->job_head)
break;