aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--tools/server/handlers.c53
-rw-r--r--tools/server/schedule.c88
-rw-r--r--tools/server/server.c19
-rw-r--r--tools/server/server_internal.h36
4 files changed, 50 insertions, 146 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 4ca3157..fed4671 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -27,33 +27,11 @@ typedef struct {
int output_index;
} embedding_batch_item;
-/* Render placement (auto by default). JSON serialization is deferred to the
- * renderer thread only when another batch is already queued for the worker, so
- * the worker can start the next GPU launch while results serialize; when the
- * queue is empty (single-stream), the worker renders inline and skips the
- * thread handoff, which otherwise costs ~1-2% with nothing to overlap.
- * FFWD_SERVER_ASYNC_RENDER=0 forces inline, =1 forces always-deferred (A/B). */
-static int async_render_mode(void) {
- static int init = 0, mode = -1; /* -1 auto, 0 inline, 1 deferred */
- if (!init) {
- const char *e = getenv("FFWD_SERVER_ASYNC_RENDER");
- if (e) {
- if (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))
- mode = 0;
- else if (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "true"))
- mode = 1;
- }
- init = 1;
- }
- return mode;
-}
-
-static int render_should_defer(http_server *s) {
- int mode = async_render_mode();
- if (mode >= 0)
- return mode;
- return worker_has_pending_jobs(s);
-}
+/* Hand JSON serialization to the renderer thread only when another batch is
+ * already queued, so the worker can start that GPU launch while these results
+ * serialize. With nothing else queued the worker renders inline, which avoids a
+ * thread hand-off that buys no overlap and adds latency to the reply. */
+static int render_should_defer(http_server *s) { return worker_has_pending_jobs(s); }
static int embedding_batch_item_cmp(const void *a, const void *b) {
const embedding_batch_item *ia = a;
@@ -65,17 +43,6 @@ static int embedding_batch_item_cmp(const void *a, const void *b) {
return ia->output_index - ib->output_index;
}
-int embedding_request_fits_group(const embedding_request *r,
- int group_inputs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens) {
- if (group_inputs == 0)
- return 1;
- return r->n_inputs <= max_batch - group_inputs &&
- r->total_tokens <= max_batch_tokens - group_tokens;
-}
-
void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
if (n_reqs <= 0)
return;
@@ -159,16 +126,6 @@ int contextual_request_compatible(const contextual_request *a, const contextual_
!strcmp(a->encoding, b->encoding);
}
-int contextual_request_fits_group(const contextual_request *r,
- int group_docs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens) {
- if (group_docs == 0)
- return 1;
- return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens;
-}
-
typedef struct {
ffwd_context_input_t input;
int output_span_index;
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index 34db038..8fb4231 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -182,6 +182,17 @@ int worker_has_pending_jobs(http_server *s) {
return pending;
}
+/* True when work is already queued anywhere ahead of the worker (untokenized or
+ * tokenized). dispatch routes a new request to the tokenizer thread when this
+ * holds, so tokenization overlaps in-flight work; an otherwise idle pipeline
+ * tokenizes the request inline on the worker and skips the queue hand-off. */
+int server_has_backlog(http_server *s) {
+ pthread_mutex_lock(&s->mu);
+ int busy = s->raw_head != NULL || s->job_head != NULL;
+ pthread_mutex_unlock(&s->mu);
+ return busy;
+}
+
void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
@@ -309,11 +320,12 @@ static void process_unknown_job(job *j) {
cJSON_Delete(root);
}
-/* Parse and tokenize one job into a heap request attached to j->prep. Runs on
- * the tokenizer thread when tokenize-off-worker is on, or inline on the worker
- * otherwise; the worker moves *prep into its request array. An unknown endpoint
- * leaves prep_kind 0 (process_unknown_job handles it); a parse failure leaves
- * prep NULL with the 422 already set, so the worker finishes it as not-ready. */
+/* Parse and tokenize one job into a heap request attached to j->prep, which the
+ * worker then moves into its request array. Runs on the tokenizer thread, or on
+ * the worker itself for a request that reached the job queue untokenized. An
+ * unknown endpoint leaves prep_kind 0 (process_unknown_job handles it); a parse
+ * failure leaves prep NULL with the 422 already set, so the worker finishes it
+ * as not-ready. The tokenized flag guards against tokenizing twice. */
void tokenize_job(http_server *s, job *j) {
j->tokenized = 1;
if (!j->started_ns)
@@ -349,25 +361,7 @@ void tokenize_job(http_server *s, job *j) {
}
}
-/* Cross-request batch merging (default on). When enabled, every compatible
- * ready request in the arrival window is flattened into one pool and re-packed
- * into token-budget batches by execute_*_request_list, so mixed client batch
- * shapes fill the GPU batch instead of each whole request running on its own
- * partly-empty launch. FFWD_SERVER_MERGE_BATCHES=0 restores whole-request
- * grouping for an A/B comparison. Read once on the worker thread, so no race. */
-static int server_merge_batches_enabled(void) {
- static int init = 0, enabled = 1;
- if (!init) {
- const char *e = getenv("FFWD_SERVER_MERGE_BATCHES");
- if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
- enabled = 0;
- init = 1;
- }
- return enabled;
-}
-
void process_job_group(http_server *s, job **jobs, int n_jobs) {
- int merge_batches = server_merge_batches_enabled();
embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
@@ -376,11 +370,11 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group));
- /* Tokenize on the worker only for jobs the tokenizer stage did not already
- * prepare (tokenize-off-worker disabled), then move each prepared request
- * out of its job into the contiguous array the grouping below indexes. The
- * move transfers the heap pointers; freeing the wrapper leaves the array
- * slot owning them, freed once at the end. */
+ /* Tokenize any job that reached the queue untokenized (dispatched inline
+ * past the tokenizer thread), then move each prepared request out of its job
+ * into the contiguous array the grouping below indexes. The move transfers
+ * the heap pointers; freeing the wrapper leaves the array slot owning them,
+ * freed once at the end. */
for (int i = 0; i < n_jobs; i++) {
if (!jobs[i]->tokenized)
tokenize_job(s, jobs[i]);
@@ -420,24 +414,17 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
finish_job(jobs[i]);
done[i] = 1;
} else if (kind[i] == 1) {
+ /* Pool every compatible ready request in this window into one list.
+ * execute_embedding_request_list length-sorts the inputs and packs
+ * them into token-budget batches, so mixed client batch shapes fill
+ * the GPU batch instead of each request running on a partly-empty
+ * launch of its own. */
int group_n = 1;
- int group_inputs = std_reqs[i].n_inputs;
- int group_tokens = std_reqs[i].total_tokens;
std_group[0] = &std_reqs[i];
for (int k = i + 1; k < n_jobs; k++) {
- if (done[k] || kind[k] != 1 || !std_reqs[k].ready ||
- !embedding_request_compatible(&std_reqs[i], &std_reqs[k]))
- continue;
- /* Merge mode pools every compatible request and lets the executor
- * pack token-budget batches; the off path keeps only requests
- * that wholly fit one batch (the pre-merge grouping). */
- if (!merge_batches &&
- !embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens,
- s->batch_size, s->max_batch_tokens))
- continue;
- std_group[group_n++] = &std_reqs[k];
- group_inputs += std_reqs[k].n_inputs;
- group_tokens += std_reqs[k].total_tokens;
+ if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
+ embedding_request_compatible(&std_reqs[i], &std_reqs[k]))
+ std_group[group_n++] = &std_reqs[k];
}
execute_embedding_request_list(std_group, group_n);
for (int k = 0; k < group_n; k++) {
@@ -450,20 +437,11 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
}
} else {
int group_n = 1;
- int group_docs = ctx_reqs[i].n_docs;
- int group_tokens = ctx_reqs[i].total_tokens;
ctx_group[0] = &ctx_reqs[i];
for (int k = i + 1; k < n_jobs; k++) {
- if (done[k] || kind[k] != 2 || !ctx_reqs[k].ready ||
- !contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]))
- continue;
- if (!merge_batches &&
- !contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens,
- s->batch_size, s->max_batch_tokens))
- continue;
- ctx_group[group_n++] = &ctx_reqs[k];
- group_docs += ctx_reqs[k].n_docs;
- group_tokens += ctx_reqs[k].total_tokens;
+ if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
+ contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]))
+ ctx_group[group_n++] = &ctx_reqs[k];
}
execute_contextual_request_list(ctx_group, group_n);
for (int k = 0; k < group_n; k++) {
diff --git a/tools/server/server.c b/tools/server/server.c
index 63438b9..7a71a1f 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -66,17 +66,6 @@ static bool auth_ok(http_server *s, const char *auth) {
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
}
-/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the
- * tokenizer thread so parsing and tokenization overlap the GPU instead of
- * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0
- * keeps tokenization on the worker for an A/B comparison. */
-static int tokenize_off_worker_default(void) {
- const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER");
- if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
- return 0;
- return 1;
-}
-
void dispatch_request(client *c) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -117,7 +106,10 @@ void dispatch_request(client *c) {
j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
j->created_ns = nstime();
client_incref(c);
- if (c->srv->tokenize_off_worker)
+ /* Tokenize off the worker when the pipeline is already busy so it overlaps
+ * in-flight work; tokenize inline on the worker when idle to avoid a
+ * queue hand-off that single-stream requests gain nothing from. */
+ if (server_has_backlog(c->srv))
enqueue_raw_job(j);
else
enqueue_job(j);
@@ -232,7 +224,7 @@ static void *worker_main(void *arg) {
/* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the
* worker's job queue. Waits for the worker to finish model init first, because
* a GPU build finalizes the model config (dims, pooling) on the worker and
- * request validation reads it. Idle when tokenize-off-worker is disabled. */
+ * request validation reads it. Idle while no request is routed through it. */
static void *tokenizer_main(void *arg) {
http_server *s = arg;
pthread_mutex_lock(&s->mu);
@@ -354,7 +346,6 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.max_batch_tokens =
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
- s.tokenize_off_worker = tokenize_off_worker_default();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index c0932d5..b332404 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -51,18 +51,6 @@
#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90
#define FFWD_MLX_RESIDENT_MULTIPLIER 2
-/* Runtime diagnostic switches. All are optional: the default is always the
- * tuned release path, so nothing has to be set to get full performance. Each is
- * read once at startup and exists only to isolate one pipeline stage for an A/B
- * comparison.
- * FFWD_SERVER_TOKENIZE_OFF_WORKER default on - 0 tokenizes on the worker
- * instead of a separate thread
- * FFWD_SERVER_MERGE_BATCHES default on - 0 groups only requests that
- * wholly fit one batch
- * FFWD_SERVER_ASYNC_RENDER default auto - 0 always inline, 1 always
- * defer; auto overlaps on load
- * Backend kernel switches (FFWD_CUDA_*) are separate and live in libffwd. */
-
typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
typedef enum {
@@ -117,10 +105,9 @@ typedef struct {
int worker_init_rc;
job *job_head;
job *job_tail;
- /* Tokenizer stage: when tokenize_off_worker is set, dispatch enqueues raw
- * jobs here and a dedicated thread parses + tokenizes them off the worker so
+ /* Tokenizer stage: dispatch enqueues a raw job here when the pipeline is
+ * busy, and a dedicated thread parses + tokenizes it off the worker so
* tokenization overlaps the GPU instead of serializing in front of it. */
- int tokenize_off_worker;
pthread_t tokenizer;
pthread_cond_t raw_cv;
int raw_stopping;
@@ -179,10 +166,10 @@ struct job {
char *extra_headers;
char *response;
size_t response_len;
- /* Filled by the tokenizer stage (tokenize_job): prep points at a heap
- * request of type prep_kind (1 embedding, 2 contextual, 3 rerank), which the
- * worker moves into its request array. tokenized guards re-tokenizing when
- * the worker runs tokenize_job inline (tokenize-off-worker disabled). */
+ /* Filled by tokenize_job (on the tokenizer thread, or on the worker for a
+ * request dispatched inline): prep points at a heap request of type
+ * prep_kind (1 embedding, 2 contextual, 3 rerank), which the worker moves
+ * into its request array. tokenized guards against tokenizing twice. */
int tokenized;
int prep_kind;
void *prep;
@@ -333,19 +320,9 @@ void free_models(http_server *s);
/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out);
int embedding_request_compatible(const embedding_request *a, const embedding_request *b);
-int embedding_request_fits_group(const embedding_request *r,
- int group_inputs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens);
void execute_embedding_request_list(embedding_request **reqs, int n_reqs);
void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out);
int contextual_request_compatible(const contextual_request *a, const contextual_request *b);
-int contextual_request_fits_group(const contextual_request *r,
- int group_docs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens);
void execute_contextual_request_list(contextual_request **reqs, int n_reqs);
void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out);
void execute_rerank_request(rerank_request *r);
@@ -353,6 +330,7 @@ void execute_rerank_request(rerank_request *r);
/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
void enqueue_job(job *j);
int worker_has_pending_jobs(http_server *s);
+int server_has_backlog(http_server *s);
void enqueue_raw_job(job *j);
job *dequeue_raw_job(http_server *s);
void tokenize_job(http_server *s, job *j);