aboutsummaryrefslogtreecommitdiff
path: root/tools/server
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-04 03:19:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-04 03:19:20 +0200
commit6a3dbc6edce2daae2dfb5a217e9e1318804000f4 (patch)
treef08d8b43b0ce39f620410eeb0772ba0078192597 /tools/server
parented23a1571fe6911eba9d4727bef30c82df7be06d (diff)
downloadffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.tar.gz
ffwd-6a3dbc6edce2daae2dfb5a217e9e1318804000f4.zip
server: bound the collect window by the batch token budget
Diffstat (limited to 'tools/server')
-rw-r--r--tools/server/schedule.c27
-rw-r--r--tools/server/test_schedule.c36
2 files changed, 63 insertions, 0 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index c71cad3..68b32e9 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -131,6 +131,23 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t
return pthread_cond_timedwait(cv, mu, &abs);
}
+/* Tokenized size of a queued job, for bounding the collect window. Jobs reach
+ * the queue through the tokenizer thread, so prep is already filled; a parse
+ * or validation failure leaves prep NULL and counts as 0. */
+static int64_t job_prep_tokens(const job *j) {
+ if (!j->prep)
+ return 0;
+ if (j->prep_kind == 1)
+ return ((const embedding_request *)j->prep)->total_tokens;
+ if (j->prep_kind == 2)
+ return ((const contextual_request *)j->prep)->total_tokens;
+ if (j->prep_kind == 3) {
+ const rerank_request *r = j->prep;
+ return (int64_t)r->query_tokens + r->document_tokens;
+ }
+ return 0;
+}
+
int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
if (max_jobs <= 0)
return 0;
@@ -159,6 +176,7 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
uint64_t deadline_ns =
first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
+ int64_t window_tokens = job_prep_tokens(first);
while (n < max_jobs) {
while (!s->job_head && !s->stopping && wait_ns > 0) {
int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
@@ -169,6 +187,15 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
break;
if (wait_ns > 0 && s->job_head->created_ns > deadline_ns)
break;
+ /* Stop the window at one token budget so the worker returns to the
+ * queue after about one execution group. Requests that arrive while a
+ * group runs then join the very next window instead of waiting for
+ * everything collected here to finish. */
+ int64_t next_tokens = job_prep_tokens(s->job_head);
+ if (s->max_batch_tokens > 0 && window_tokens > 0 && next_tokens > 0 &&
+ window_tokens + next_tokens > s->max_batch_tokens)
+ break;
+ window_tokens += next_tokens;
jobs[n++] = pop_job_locked(s);
if (wait_ns == 0 && !s->job_head)
break;
diff --git a/tools/server/test_schedule.c b/tools/server/test_schedule.c
index 1f799f1..26976ee 100644
--- a/tools/server/test_schedule.c
+++ b/tools/server/test_schedule.c
@@ -66,8 +66,44 @@ static void test_collect_job_batch_zero_wait(void) {
pthread_mutex_destroy(&s.mu);
}
+static void test_collect_job_batch_token_budget(void) {
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.batch_size = 32;
+ s.max_batch_tokens = 1000;
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ /* 600 + 300 fit the 1000-token window; the 200-token job must wait for
+ * the next window. */
+ embedding_request r1 = {.total_tokens = 600};
+ embedding_request r2 = {.total_tokens = 300};
+ embedding_request r3 = {.total_tokens = 200};
+ job a = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r1};
+ job b = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r2};
+ job c = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r3};
+ snprintf(a.path, sizeof(a.path), "%s", "/v1/embeddings");
+ snprintf(b.path, sizeof(b.path), "%s", "/v1/embeddings");
+ snprintf(c.path, sizeof(c.path), "%s", "/v1/embeddings");
+ enqueue_job(&a);
+ enqueue_job(&b);
+ enqueue_job(&c);
+
+ job *batch[3] = {0};
+ TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2);
+ TEST_ASSERT(batch[0] == &a);
+ TEST_ASSERT(batch[1] == &b);
+ TEST_ASSERT(s.job_head == &c);
+
+ s.job_head = NULL;
+ s.job_tail = NULL;
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+}
+
int main(void) {
test_collect_job_batch_deadline();
test_collect_job_batch_zero_wait();
+ test_collect_job_batch_token_budget();
return TEST_REPORT("schedule");
}