diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 01:51:53 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 01:51:53 +0200 |
| commit | cb85a499f3d5178a752c85063bf9ff277ca3a71e (patch) | |
| tree | 78f68836e954d08dd6409e6d56cf44a7fb3c35e7 | |
| parent | 716d2051de27123bb298dbe2536c23537ec9512d (diff) | |
| download | ffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.tar.gz ffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.zip | |
server: make tokenize/render overlap adaptive, drop env switches
| -rw-r--r-- | tools/server/handlers.c | 53 | ||||
| -rw-r--r-- | tools/server/schedule.c | 88 | ||||
| -rw-r--r-- | tools/server/server.c | 19 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 36 |
4 files changed, 50 insertions, 146 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 4ca3157..fed4671 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -27,33 +27,11 @@ typedef struct { int output_index; } embedding_batch_item; -/* Render placement (auto by default). JSON serialization is deferred to the - * renderer thread only when another batch is already queued for the worker, so - * the worker can start the next GPU launch while results serialize; when the - * queue is empty (single-stream), the worker renders inline and skips the - * thread handoff, which otherwise costs ~1-2% with nothing to overlap. - * FFWD_SERVER_ASYNC_RENDER=0 forces inline, =1 forces always-deferred (A/B). */ -static int async_render_mode(void) { - static int init = 0, mode = -1; /* -1 auto, 0 inline, 1 deferred */ - if (!init) { - const char *e = getenv("FFWD_SERVER_ASYNC_RENDER"); - if (e) { - if (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")) - mode = 0; - else if (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "true")) - mode = 1; - } - init = 1; - } - return mode; -} - -static int render_should_defer(http_server *s) { - int mode = async_render_mode(); - if (mode >= 0) - return mode; - return worker_has_pending_jobs(s); -} +/* Hand JSON serialization to the renderer thread only when another batch is + * already queued, so the worker can start that GPU launch while these results + * serialize. With nothing else queued the worker renders inline, which avoids a + * thread hand-off that buys no overlap and adds latency to the reply. */ +static int render_should_defer(http_server *s) { return worker_has_pending_jobs(s); } static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; @@ -65,17 +43,6 @@ static int embedding_batch_item_cmp(const void *a, const void *b) { return ia->output_index - ib->output_index; } -int embedding_request_fits_group(const embedding_request *r, - int group_inputs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_inputs == 0) - return 1; - return r->n_inputs <= max_batch - group_inputs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { if (n_reqs <= 0) return; @@ -159,16 +126,6 @@ int contextual_request_compatible(const contextual_request *a, const contextual_ !strcmp(a->encoding, b->encoding); } -int contextual_request_fits_group(const contextual_request *r, - int group_docs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_docs == 0) - return 1; - return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; -} - typedef struct { ffwd_context_input_t input; int output_span_index; diff --git a/tools/server/schedule.c b/tools/server/schedule.c index 34db038..8fb4231 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -182,6 +182,17 @@ int worker_has_pending_jobs(http_server *s) { return pending; } +/* True when work is already queued anywhere ahead of the worker (untokenized or + * tokenized). dispatch routes a new request to the tokenizer thread when this + * holds, so tokenization overlaps in-flight work; an otherwise idle pipeline + * tokenizes the request inline on the worker and skips the queue hand-off. */ +int server_has_backlog(http_server *s) { + pthread_mutex_lock(&s->mu); + int busy = s->raw_head != NULL || s->job_head != NULL; + pthread_mutex_unlock(&s->mu); + return busy; +} + void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); @@ -309,11 +320,12 @@ static void process_unknown_job(job *j) { cJSON_Delete(root); } -/* Parse and tokenize one job into a heap request attached to j->prep. Runs on - * the tokenizer thread when tokenize-off-worker is on, or inline on the worker - * otherwise; the worker moves *prep into its request array. An unknown endpoint - * leaves prep_kind 0 (process_unknown_job handles it); a parse failure leaves - * prep NULL with the 422 already set, so the worker finishes it as not-ready. */ +/* Parse and tokenize one job into a heap request attached to j->prep, which the + * worker then moves into its request array. Runs on the tokenizer thread, or on + * the worker itself for a request that reached the job queue untokenized. An + * unknown endpoint leaves prep_kind 0 (process_unknown_job handles it); a parse + * failure leaves prep NULL with the 422 already set, so the worker finishes it + * as not-ready. The tokenized flag guards against tokenizing twice. */ void tokenize_job(http_server *s, job *j) { j->tokenized = 1; if (!j->started_ns) @@ -349,25 +361,7 @@ void tokenize_job(http_server *s, job *j) { } } -/* Cross-request batch merging (default on). When enabled, every compatible - * ready request in the arrival window is flattened into one pool and re-packed - * into token-budget batches by execute_*_request_list, so mixed client batch - * shapes fill the GPU batch instead of each whole request running on its own - * partly-empty launch. FFWD_SERVER_MERGE_BATCHES=0 restores whole-request - * grouping for an A/B comparison. Read once on the worker thread, so no race. */ -static int server_merge_batches_enabled(void) { - static int init = 0, enabled = 1; - if (!init) { - const char *e = getenv("FFWD_SERVER_MERGE_BATCHES"); - if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))) - enabled = 0; - init = 1; - } - return enabled; -} - void process_job_group(http_server *s, job **jobs, int n_jobs) { - int merge_batches = server_merge_batches_enabled(); embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); @@ -376,11 +370,11 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); - /* Tokenize on the worker only for jobs the tokenizer stage did not already - * prepare (tokenize-off-worker disabled), then move each prepared request - * out of its job into the contiguous array the grouping below indexes. The - * move transfers the heap pointers; freeing the wrapper leaves the array - * slot owning them, freed once at the end. */ + /* Tokenize any job that reached the queue untokenized (dispatched inline + * past the tokenizer thread), then move each prepared request out of its job + * into the contiguous array the grouping below indexes. The move transfers + * the heap pointers; freeing the wrapper leaves the array slot owning them, + * freed once at the end. */ for (int i = 0; i < n_jobs; i++) { if (!jobs[i]->tokenized) tokenize_job(s, jobs[i]); @@ -420,24 +414,17 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { finish_job(jobs[i]); done[i] = 1; } else if (kind[i] == 1) { + /* Pool every compatible ready request in this window into one list. + * execute_embedding_request_list length-sorts the inputs and packs + * them into token-budget batches, so mixed client batch shapes fill + * the GPU batch instead of each request running on a partly-empty + * launch of its own. */ int group_n = 1; - int group_inputs = std_reqs[i].n_inputs; - int group_tokens = std_reqs[i].total_tokens; std_group[0] = &std_reqs[i]; for (int k = i + 1; k < n_jobs; k++) { - if (done[k] || kind[k] != 1 || !std_reqs[k].ready || - !embedding_request_compatible(&std_reqs[i], &std_reqs[k])) - continue; - /* Merge mode pools every compatible request and lets the executor - * pack token-budget batches; the off path keeps only requests - * that wholly fit one batch (the pre-merge grouping). */ - if (!merge_batches && - !embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens, - s->batch_size, s->max_batch_tokens)) - continue; - std_group[group_n++] = &std_reqs[k]; - group_inputs += std_reqs[k].n_inputs; - group_tokens += std_reqs[k].total_tokens; + if (!done[k] && kind[k] == 1 && std_reqs[k].ready && + embedding_request_compatible(&std_reqs[i], &std_reqs[k])) + std_group[group_n++] = &std_reqs[k]; } execute_embedding_request_list(std_group, group_n); for (int k = 0; k < group_n; k++) { @@ -450,20 +437,11 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { } } else { int group_n = 1; - int group_docs = ctx_reqs[i].n_docs; - int group_tokens = ctx_reqs[i].total_tokens; ctx_group[0] = &ctx_reqs[i]; for (int k = i + 1; k < n_jobs; k++) { - if (done[k] || kind[k] != 2 || !ctx_reqs[k].ready || - !contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k])) - continue; - if (!merge_batches && - !contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens, - s->batch_size, s->max_batch_tokens)) - continue; - ctx_group[group_n++] = &ctx_reqs[k]; - group_docs += ctx_reqs[k].n_docs; - group_tokens += ctx_reqs[k].total_tokens; + if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && + contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k])) + ctx_group[group_n++] = &ctx_reqs[k]; } execute_contextual_request_list(ctx_group, group_n); for (int k = 0; k < group_n; k++) { diff --git a/tools/server/server.c b/tools/server/server.c index 63438b9..7a71a1f 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -66,17 +66,6 @@ static bool auth_ok(http_server *s, const char *auth) { !strcmp(auth + sizeof(prefix) - 1, s->api_key); } -/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the - * tokenizer thread so parsing and tokenization overlap the GPU instead of - * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0 - * keeps tokenization on the worker for an A/B comparison. */ -static int tokenize_off_worker_default(void) { - const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER"); - if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))) - return 0; - return 1; -} - void dispatch_request(client *c) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); @@ -117,7 +106,10 @@ void dispatch_request(client *c) { j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; j->created_ns = nstime(); client_incref(c); - if (c->srv->tokenize_off_worker) + /* Tokenize off the worker when the pipeline is already busy so it overlaps + * in-flight work; tokenize inline on the worker when idle to avoid a + * queue hand-off that single-stream requests gain nothing from. */ + if (server_has_backlog(c->srv)) enqueue_raw_job(j); else enqueue_job(j); @@ -232,7 +224,7 @@ static void *worker_main(void *arg) { /* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the * worker's job queue. Waits for the worker to finish model init first, because * a GPU build finalizes the model config (dims, pooling) on the worker and - * request validation reads it. Idle when tokenize-off-worker is disabled. */ + * request validation reads it. Idle while no request is routed through it. */ static void *tokenizer_main(void *arg) { http_server *s = arg; pthread_mutex_lock(&s->mu); @@ -354,7 +346,6 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { s.max_batch_tokens = cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); - s.tokenize_off_worker = tokenize_off_worker_default(); s.backend_opts = backend_opts; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index c0932d5..b332404 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -51,18 +51,6 @@ #define FFWD_MLX_MEMORY_BUDGET_PERCENT 90 #define FFWD_MLX_RESIDENT_MULTIPLIER 2 -/* Runtime diagnostic switches. All are optional: the default is always the - * tuned release path, so nothing has to be set to get full performance. Each is - * read once at startup and exists only to isolate one pipeline stage for an A/B - * comparison. - * FFWD_SERVER_TOKENIZE_OFF_WORKER default on - 0 tokenizes on the worker - * instead of a separate thread - * FFWD_SERVER_MERGE_BATCHES default on - 0 groups only requests that - * wholly fit one batch - * FFWD_SERVER_ASYNC_RENDER default auto - 0 always inline, 1 always - * defer; auto overlaps on load - * Backend kernel switches (FFWD_CUDA_*) are separate and live in libffwd. */ - typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; typedef enum { @@ -117,10 +105,9 @@ typedef struct { int worker_init_rc; job *job_head; job *job_tail; - /* Tokenizer stage: when tokenize_off_worker is set, dispatch enqueues raw - * jobs here and a dedicated thread parses + tokenizes them off the worker so + /* Tokenizer stage: dispatch enqueues a raw job here when the pipeline is + * busy, and a dedicated thread parses + tokenizes it off the worker so * tokenization overlaps the GPU instead of serializing in front of it. */ - int tokenize_off_worker; pthread_t tokenizer; pthread_cond_t raw_cv; int raw_stopping; @@ -179,10 +166,10 @@ struct job { char *extra_headers; char *response; size_t response_len; - /* Filled by the tokenizer stage (tokenize_job): prep points at a heap - * request of type prep_kind (1 embedding, 2 contextual, 3 rerank), which the - * worker moves into its request array. tokenized guards re-tokenizing when - * the worker runs tokenize_job inline (tokenize-off-worker disabled). */ + /* Filled by tokenize_job (on the tokenizer thread, or on the worker for a + * request dispatched inline): prep points at a heap request of type + * prep_kind (1 embedding, 2 contextual, 3 rerank), which the worker moves + * into its request array. tokenized guards against tokenizing twice. */ int tokenized; int prep_kind; void *prep; @@ -333,19 +320,9 @@ void free_models(http_server *s); /* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out); int embedding_request_compatible(const embedding_request *a, const embedding_request *b); -int embedding_request_fits_group(const embedding_request *r, - int group_inputs, - int group_tokens, - int max_batch, - int max_batch_tokens); void execute_embedding_request_list(embedding_request **reqs, int n_reqs); void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out); int contextual_request_compatible(const contextual_request *a, const contextual_request *b); -int contextual_request_fits_group(const contextual_request *r, - int group_docs, - int group_tokens, - int max_batch, - int max_batch_tokens); void execute_contextual_request_list(contextual_request **reqs, int n_reqs); void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out); void execute_rerank_request(rerank_request *r); @@ -353,6 +330,7 @@ void execute_rerank_request(rerank_request *r); /* ---- server_schedule.c: job queue, micro-batching, completion ---- */ void enqueue_job(job *j); int worker_has_pending_jobs(http_server *s); +int server_has_backlog(http_server *s); void enqueue_raw_job(job *j); job *dequeue_raw_job(http_server *s); void tokenize_job(http_server *s, job *j); |
