diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-04 03:19:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-04 03:19:20 +0200 |
| commit | 6a3dbc6edce2daae2dfb5a217e9e1318804000f4 (patch) | |
| tree | f08d8b43b0ce39f620410eeb0772ba0078192597 /tools/server/schedule.c | |
| parent | ed23a1571fe6911eba9d4727bef30c82df7be06d (diff) | |
| download | ffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.tar.gz ffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.zip | |
server: bound the collect window by the batch token budget
Diffstat (limited to 'tools/server/schedule.c')
| -rw-r--r-- | tools/server/schedule.c | 27 |
1 files changed, 27 insertions, 0 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c index c71cad3..68b32e9 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -131,6 +131,23 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t return pthread_cond_timedwait(cv, mu, &abs); } +/* Tokenized size of a queued job, for bounding the collect window. Jobs reach + * the queue through the tokenizer thread, so prep is already filled; a parse + * or validation failure leaves prep NULL and counts as 0. */ +static int64_t job_prep_tokens(const job *j) { + if (!j->prep) + return 0; + if (j->prep_kind == 1) + return ((const embedding_request *)j->prep)->total_tokens; + if (j->prep_kind == 2) + return ((const contextual_request *)j->prep)->total_tokens; + if (j->prep_kind == 3) { + const rerank_request *r = j->prep; + return (int64_t)r->query_tokens + r->document_tokens; + } + return 0; +} + int collect_job_batch(http_server *s, job **jobs, int max_jobs) { if (max_jobs <= 0) return 0; @@ -159,6 +176,7 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) { uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns; + int64_t window_tokens = job_prep_tokens(first); while (n < max_jobs) { while (!s->job_head && !s->stopping && wait_ns > 0) { int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); @@ -169,6 +187,15 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) { break; if (wait_ns > 0 && s->job_head->created_ns > deadline_ns) break; + /* Stop the window at one token budget so the worker returns to the + * queue after about one execution group. Requests that arrive while a + * group runs then join the very next window instead of waiting for + * everything collected here to finish. */ + int64_t next_tokens = job_prep_tokens(s->job_head); + if (s->max_batch_tokens > 0 && window_tokens > 0 && next_tokens > 0 && + window_tokens + next_tokens > s->max_batch_tokens) + break; + window_tokens += next_tokens; jobs[n++] = pop_job_locked(s); if (wait_ns == 0 && !s->job_head) break; |
