aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--tools/server/schedule.c124
-rw-r--r--tools/server/server.c89
-rw-r--r--tools/server/server_internal.h19
3 files changed, 209 insertions, 23 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index fc63c91..34db038 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -65,6 +65,34 @@ job *dequeue_render_job(http_server *s) {
return j;
}
+void enqueue_raw_job(job *j) {
+ http_server *s = j->srv;
+ pthread_mutex_lock(&s->mu);
+ if (s->raw_tail)
+ s->raw_tail->next = j;
+ else
+ s->raw_head = j;
+ s->raw_tail = j;
+ pthread_cond_signal(&s->raw_cv);
+ pthread_mutex_unlock(&s->mu);
+}
+
+job *dequeue_raw_job(http_server *s) {
+ pthread_mutex_lock(&s->mu);
+ while (!s->raw_head && !s->raw_stopping)
+ pthread_cond_wait(&s->raw_cv, &s->mu);
+
+ job *j = s->raw_head;
+ if (j) {
+ s->raw_head = j->next;
+ if (!s->raw_head)
+ s->raw_tail = NULL;
+ j->next = NULL;
+ }
+ pthread_mutex_unlock(&s->mu);
+ return j;
+}
+
static job *pop_job_locked(http_server *s) {
job *j = s->job_head;
if (j) {
@@ -179,9 +207,26 @@ static job *pop_done(http_server *s) {
return j;
}
+/* Free a tokenizer-stage request still attached to a job. In the normal flow
+ * the worker moves *prep out and clears it before the job is freed; this only
+ * fires if a tokenized job is discarded without being processed. */
+static void job_prep_free(job *j) {
+ if (!j->prep)
+ return;
+ if (j->prep_kind == 1)
+ embedding_request_free(j->prep);
+ else if (j->prep_kind == 2)
+ contextual_request_free(j->prep);
+ else if (j->prep_kind == 3)
+ rerank_request_free(j->prep);
+ free(j->prep);
+ j->prep = NULL;
+}
+
static void job_free(job *j) {
if (!j)
return;
+ job_prep_free(j);
job_render_free(j);
free(j->body);
free(j->auth);
@@ -264,6 +309,46 @@ static void process_unknown_job(job *j) {
cJSON_Delete(root);
}
+/* Parse and tokenize one job into a heap request attached to j->prep. Runs on
+ * the tokenizer thread when tokenize-off-worker is on, or inline on the worker
+ * otherwise; the worker moves *prep into its request array. An unknown endpoint
+ * leaves prep_kind 0 (process_unknown_job handles it); a parse failure leaves
+ * prep NULL with the 422 already set, so the worker finishes it as not-ready. */
+void tokenize_job(http_server *s, job *j) {
+ j->tokenized = 1;
+ if (!j->started_ns)
+ j->started_ns = nstime();
+ if (!strcmp(j->path, "/v1/embeddings"))
+ j->prep_kind = 1;
+ else if (!strcmp(j->path, "/v1/contextualizedembeddings"))
+ j->prep_kind = 2;
+ else if (!strcmp(j->path, "/v1/rerank"))
+ j->prep_kind = 3;
+ else
+ return;
+
+ cJSON *root = NULL;
+ uint64_t t0 = nstime();
+ int parse_rc = parse_job_root(j, &root);
+ j->parse_ns += nstime() - t0;
+ if (parse_rc != 0)
+ return;
+
+ if (j->prep_kind == 1) {
+ embedding_request *r = xcalloc(1, sizeof(*r));
+ prepare_embedding_request(j, root, s, r);
+ j->prep = r;
+ } else if (j->prep_kind == 2) {
+ contextual_request *r = xcalloc(1, sizeof(*r));
+ prepare_contextual_request(j, root, s, r);
+ j->prep = r;
+ } else {
+ rerank_request *r = xcalloc(1, sizeof(*r));
+ prepare_rerank_request(j, root, s, r);
+ j->prep = r;
+ }
+}
+
/* Cross-request batch merging (default on). When enabled, every compatible
* ready request in the arrival window is flattened into one pool and re-packed
* into token-budget batches by execute_*_request_list, so mixed client batch
@@ -291,30 +376,25 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group));
+ /* Tokenize on the worker only for jobs the tokenizer stage did not already
+ * prepare (tokenize-off-worker disabled), then move each prepared request
+ * out of its job into the contiguous array the grouping below indexes. The
+ * move transfers the heap pointers; freeing the wrapper leaves the array
+ * slot owning them, freed once at the end. */
for (int i = 0; i < n_jobs; i++) {
- cJSON *root = NULL;
- if (!jobs[i]->started_ns)
- jobs[i]->started_ns = nstime();
- if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
- kind[i] = 1;
- } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
- kind[i] = 2;
- } else if (!strcmp(jobs[i]->path, "/v1/rerank")) {
- kind[i] = 3;
- } else {
- continue;
+ if (!jobs[i]->tokenized)
+ tokenize_job(s, jobs[i]);
+ kind[i] = jobs[i]->prep_kind;
+ if (jobs[i]->prep) {
+ if (kind[i] == 1)
+ std_reqs[i] = *(embedding_request *)jobs[i]->prep;
+ else if (kind[i] == 2)
+ ctx_reqs[i] = *(contextual_request *)jobs[i]->prep;
+ else if (kind[i] == 3)
+ rerank_reqs[i] = *(rerank_request *)jobs[i]->prep;
+ free(jobs[i]->prep);
+ jobs[i]->prep = NULL;
}
- uint64_t t0 = nstime();
- int parse_rc = parse_job_root(jobs[i], &root);
- jobs[i]->parse_ns += nstime() - t0;
- if (parse_rc != 0)
- continue;
- if (kind[i] == 1)
- prepare_embedding_request(jobs[i], root, s, &std_reqs[i]);
- else if (kind[i] == 2)
- prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]);
- else
- prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]);
}
for (int i = 0; i < n_jobs; i++) {
diff --git a/tools/server/server.c b/tools/server/server.c
index 8762c4f..63438b9 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -66,6 +66,17 @@ static bool auth_ok(http_server *s, const char *auth) {
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
}
+/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the
+ * tokenizer thread so parsing and tokenization overlap the GPU instead of
+ * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0
+ * keeps tokenization on the worker for an A/B comparison. */
+static int tokenize_off_worker_default(void) {
+ const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER");
+ if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
+ return 0;
+ return 1;
+}
+
void dispatch_request(client *c) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -106,7 +117,10 @@ void dispatch_request(client *c) {
j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
j->created_ns = nstime();
client_incref(c);
- enqueue_job(j);
+ if (c->srv->tokenize_off_worker)
+ enqueue_raw_job(j);
+ else
+ enqueue_job(j);
}
static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
@@ -215,6 +229,30 @@ static void *worker_main(void *arg) {
return NULL;
}
+/* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the
+ * worker's job queue. Waits for the worker to finish model init first, because
+ * a GPU build finalizes the model config (dims, pooling) on the worker and
+ * request validation reads it. Idle when tokenize-off-worker is disabled. */
+static void *tokenizer_main(void *arg) {
+ http_server *s = arg;
+ pthread_mutex_lock(&s->mu);
+ while (!s->worker_ready)
+ pthread_cond_wait(&s->cv, &s->mu);
+ int rc = s->worker_init_rc;
+ pthread_mutex_unlock(&s->mu);
+ if (rc)
+ return NULL;
+
+ for (;;) {
+ job *j = dequeue_raw_job(s);
+ if (!j)
+ break;
+ tokenize_job(s, j);
+ enqueue_job(j);
+ }
+ return NULL;
+}
+
static void *renderer_main(void *arg) {
http_server *s = arg;
for (;;) {
@@ -316,6 +354,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.max_batch_tokens =
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
+ s.tokenize_off_worker = tokenize_off_worker_default();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);
@@ -327,6 +366,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
pthread_mutex_init(&s.mu, NULL);
pthread_cond_init(&s.cv, NULL);
+ pthread_cond_init(&s.raw_cv, NULL);
pthread_cond_init(&s.render_cv, NULL);
for (int i = 0; i < cfg->n_models; i++) {
@@ -334,6 +374,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -345,6 +386,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -369,6 +411,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -380,6 +423,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -396,6 +440,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -420,6 +465,33 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
+
+ /* Started after worker init so the tokenizer sees a finalized model config
+ * (a GPU build sets dims/pooling on the worker, and request validation reads
+ * them). Joined before the worker at shutdown so it drains raw jobs into the
+ * job queue first. */
+ if (pthread_create(&s.tokenizer, NULL, tokenizer_main, &s) != 0) {
+ fprintf(stderr, "ffwd-server: failed to start tokenizer\n");
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -428,6 +500,11 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.listen_fd = listen_on(s.host, s.port);
if (s.listen_fd < 0) {
pthread_mutex_lock(&s.mu);
+ s.raw_stopping = 1;
+ pthread_cond_signal(&s.raw_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.tokenizer, NULL);
+ pthread_mutex_lock(&s.mu);
s.stopping = 1;
pthread_cond_signal(&s.cv);
pthread_mutex_unlock(&s.mu);
@@ -441,6 +518,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -464,6 +542,14 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
c = next;
}
+ /* Drain stages front to back: the tokenizer flushes raw jobs into the job
+ * queue, then the worker drains that, then the renderer drains finished
+ * jobs, so nothing is stranded mid-pipeline. */
+ pthread_mutex_lock(&s.mu);
+ s.raw_stopping = 1;
+ pthread_cond_signal(&s.raw_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.tokenizer, NULL);
pthread_mutex_lock(&s.mu);
s.stopping = 1;
pthread_cond_signal(&s.cv);
@@ -483,6 +569,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
free_models(&s);
free(s.api_key);
pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.raw_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 0;
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index c79862f..3ec4e8d 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -105,6 +105,15 @@ typedef struct {
int worker_init_rc;
job *job_head;
job *job_tail;
+ /* Tokenizer stage: when tokenize_off_worker is set, dispatch enqueues raw
+ * jobs here and a dedicated thread parses + tokenizes them off the worker so
+ * tokenization overlaps the GPU instead of serializing in front of it. */
+ int tokenize_off_worker;
+ pthread_t tokenizer;
+ pthread_cond_t raw_cv;
+ int raw_stopping;
+ job *raw_head;
+ job *raw_tail;
pthread_t renderer;
pthread_cond_t render_cv;
int render_stopping;
@@ -158,6 +167,13 @@ struct job {
char *extra_headers;
char *response;
size_t response_len;
+ /* Filled by the tokenizer stage (tokenize_job): prep points at a heap
+ * request of type prep_kind (1 embedding, 2 contextual, 3 rerank), which the
+ * worker moves into its request array. tokenized guards re-tokenizing when
+ * the worker runs tokenize_job inline (tokenize-off-worker disabled). */
+ int tokenized;
+ int prep_kind;
+ void *prep;
uint64_t created_ns;
uint64_t started_ns;
uint64_t parse_ns;
@@ -325,6 +341,9 @@ void execute_rerank_request(rerank_request *r);
/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
void enqueue_job(job *j);
int worker_has_pending_jobs(http_server *s);
+void enqueue_raw_job(job *j);
+job *dequeue_raw_job(http_server *s);
+void tokenize_job(http_server *s, job *j);
void enqueue_render_job(job *j);
job *dequeue_render_job(http_server *s);
void finish_job(job *j);