aboutsummaryrefslogtreecommitdiff
path: root/tools/server
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-24 03:10:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-24 03:10:47 +0200
commit76758db35873a8f236e6ea0df544e71391edc4b1 (patch)
tree352fdbefca581ef7966c8e8c8da0cd24d10b1cd6 /tools/server
parentdc29ac64a203a954161026ad5f6288df201e11ad (diff)
downloadffwd-76758db35873a8f236e6ea0df544e71391edc4b1.tar.gz
ffwd-76758db35873a8f236e6ea0df544e71391edc4b1.zip
server: bound merged batches by token budget
Diffstat (limited to 'tools/server')
-rw-r--r--tools/server/schedule.c48
-rw-r--r--tools/server/server_internal.h4
2 files changed, 46 insertions, 6 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index 13e5dc7..9fb7db6 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -172,10 +172,12 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
return n;
}
-/* True when more jobs are already waiting for the worker. The render placement
+/* True when more work is already waiting for the worker. The render placement
* decision (handlers.c) uses this so it only hands a finished batch to the
* renderer thread when the worker has another batch to run. */
int worker_has_pending_jobs(http_server *s) {
+ if (s->worker_local_backlog)
+ return 1;
pthread_mutex_lock(&s->mu);
int pending = s->job_head != NULL;
pthread_mutex_unlock(&s->mu);
@@ -282,6 +284,20 @@ void finish_job(job *j) {
enqueue_done(j);
}
+static int merge_token_budget_accepts(int64_t used_tokens, int next_tokens, int budget) {
+ if (budget <= 0 || used_tokens <= 0 || next_tokens <= 0)
+ return 1;
+ return used_tokens <= (int64_t)budget - next_tokens;
+}
+
+static int has_unfinished_jobs(const int *done, int n_jobs) {
+ for (int i = 0; i < n_jobs; i++) {
+ if (!done[i])
+ return 1;
+ }
+ return 0;
+}
+
static int parse_job_root(job *j, cJSON **out_root) {
cJSON *detail = cJSON_CreateArray();
cJSON *root = parse_json_body(j, detail);
@@ -407,14 +423,26 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
* execute_embedding_request_list length-sorts the inputs and packs
* them into token-budget batches, so mixed client batch shapes fill
* the GPU batch instead of each request running on a partly-empty
- * launch of its own. */
+ * launch of its own. The merged execution unit is also capped by
+ * the configured token budget so one arrival window cannot become
+ * an unbounded host/output buffer before any response can finish. */
int group_n = 1;
+ int64_t group_tokens = std_reqs[i].total_tokens;
std_group[0] = &std_reqs[i];
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
- embedding_request_compatible(&std_reqs[i], &std_reqs[k]))
+ embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
+ merge_token_budget_accepts(group_tokens, std_reqs[k].total_tokens,
+ s->max_batch_tokens)) {
std_group[group_n++] = &std_reqs[k];
+ group_tokens += std_reqs[k].total_tokens;
+ }
}
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(std_group[k] - std_reqs);
+ done[idx] = 1;
+ }
+ s->worker_local_backlog = has_unfinished_jobs(done, n_jobs);
execute_embedding_request_list(std_group, group_n);
for (int k = 0; k < group_n; k++) {
int idx = (int)(std_group[k] - std_reqs);
@@ -422,21 +450,29 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
enqueue_render_job(jobs[idx]);
else
finish_job(jobs[idx]);
- done[idx] = 1;
}
+ s->worker_local_backlog = 0;
} else {
int group_n = 1;
+ int64_t group_tokens = ctx_reqs[i].total_tokens;
ctx_group[0] = &ctx_reqs[i];
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
- contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]))
+ contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
+ merge_token_budget_accepts(group_tokens, ctx_reqs[k].total_tokens,
+ s->max_batch_tokens)) {
ctx_group[group_n++] = &ctx_reqs[k];
+ group_tokens += ctx_reqs[k].total_tokens;
+ }
+ }
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(ctx_group[k] - ctx_reqs);
+ done[idx] = 1;
}
execute_contextual_request_list(ctx_group, group_n);
for (int k = 0; k < group_n; k++) {
int idx = (int)(ctx_group[k] - ctx_reqs);
finish_job(jobs[idx]);
- done[idx] = 1;
}
}
}
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index df6b9ea..315acdd 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -103,6 +103,10 @@ typedef struct {
int stopping;
int worker_ready;
int worker_init_rc;
+ /* Worker-only hint: current collected window still has unprocessed jobs, so
+ * response rendering can overlap that local work even if the shared queue
+ * is empty. */
+ int worker_local_backlog;
job *job_head;
job *job_tail;
/* Tokenizer stage: dispatch enqueues every raw job here, and a single