diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-13 01:25:21 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:10 +0200 |
| commit | ce352727a6f56ae19038a95b4f8f533c487fac4b (patch) | |
| tree | a76e0da7123efc7b4144d6913d03410b8deaebec | |
| parent | a716f7fa3bbedf1563db15e590cfc1b8419f5b5b (diff) | |
| download | ffwd-ce352727a6f56ae19038a95b4f8f533c487fac4b.tar.gz ffwd-ce352727a6f56ae19038a95b4f8f533c487fac4b.zip | |
server: make microbatch admission deterministic
| -rw-r--r-- | embed_server.c | 98 | ||||
| -rw-r--r-- | tests/test_server.c | 63 |
2 files changed, 131 insertions, 30 deletions
diff --git a/embed_server.c b/embed_server.c index 4a9eeb8..793d764 100644 --- a/embed_server.c +++ b/embed_server.c @@ -65,7 +65,7 @@ * was no faster and inflated long-document tail latency. */ #define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 #define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 -#define EMBED_SERVER_BATCH_WAIT_US 1000 +#define EMBED_SERVER_BATCH_WAIT_US 0 #define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 #define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 #define EMBED_MLX_RESIDENT_MULTIPLIER 2 @@ -692,29 +692,78 @@ static void enqueue_done(job *j) { (void)write(s->completion_pipe[1], &byte, 1); } -static job *dequeue_job(http_server *s) { - pthread_mutex_lock(&s->mu); - while (!s->job_head && !s->stopping) - pthread_cond_wait(&s->cv, &s->mu); +static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { s->job_head = j->next; if (!s->job_head) s->job_tail = NULL; j->next = NULL; } - pthread_mutex_unlock(&s->mu); return j; } -static int drain_ready_jobs(http_server *s, job **jobs, int max_jobs) { - int n = 0; +static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, + uint64_t deadline_ns) { + uint64_t now_ns = nstime(); + if (now_ns >= deadline_ns) return ETIMEDOUT; + + uint64_t remaining_ns = deadline_ns - now_ns; + struct timespec abs; + if (clock_gettime(CLOCK_REALTIME, &abs) != 0) { + struct timeval tv; + gettimeofday(&tv, NULL); + abs.tv_sec = tv.tv_sec; + abs.tv_nsec = (long)tv.tv_usec * 1000L; + } + abs.tv_sec += (time_t)(remaining_ns / 1000000000u); + abs.tv_nsec += (long)(remaining_ns % 1000000000u); + if (abs.tv_nsec >= 1000000000L) { + abs.tv_sec++; + abs.tv_nsec -= 1000000000L; + } + return pthread_cond_timedwait(cv, mu, &abs); +} + +static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { + if (max_jobs <= 0) return 0; + pthread_mutex_lock(&s->mu); - while (n < max_jobs && s->job_head) { - job *j = s->job_head; - s->job_head = j->next; - if (!s->job_head) s->job_tail = NULL; - j->next = NULL; - jobs[n++] = j; + while (!s->job_head && !s->stopping) + pthread_cond_wait(&s->cv, &s->mu); + + job *first = pop_job_locked(s); + if (!first) { + pthread_mutex_unlock(&s->mu); + return 0; + } + + int n = 1; + jobs[0] = first; + int batchable = s->batch_size > 1 && + (!strcmp(first->path, "/v1/embeddings") || + !strcmp(first->path, "/v1/contextualizedembeddings")); + if (!batchable) { + pthread_mutex_unlock(&s->mu); + return n; + } + + /* Exact item and token budgets are applied after worker-side tokenization; + * this queue stage only defines which arrival window may be grouped. */ + uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; + uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns + ? UINT64_MAX : first->created_ns + wait_ns; + while (n < max_jobs) { + while (!s->job_head && !s->stopping && wait_ns > 0) { + int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); + if (rc == ETIMEDOUT) break; + } + if (!s->job_head) + break; + if (wait_ns > 0 && s->job_head->created_ns > deadline_ns) + break; + jobs[n++] = pop_job_locked(s); + if (wait_ns == 0 && !s->job_head) + break; } pthread_mutex_unlock(&s->mu); return n; @@ -2078,21 +2127,10 @@ static void *worker_main(void *arg) { } for (;;) { - job *j = dequeue_job(s); - if (!j) break; job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = 1; - jobs[0] = j; - - if ((!strcmp(j->path, "/v1/embeddings") || - !strcmp(j->path, "/v1/contextualizedembeddings")) && - s->batch_size > 1) { - if (s->batch_wait_us > 0) - usleep((useconds_t)s->batch_wait_us); - n_jobs += drain_ready_jobs(s, jobs + 1, - EMBED_SERVER_MICROBATCH_MAX_JOBS - 1); - } - + int n_jobs = collect_job_batch( + s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); + if (n_jobs == 0) break; process_job_group(s, jobs, n_jobs); } #ifdef USE_MLX @@ -2467,8 +2505,8 @@ static void print_usage(const char *prog) " -b, --batch-size N Max texts or documents per inference batch\n" " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N Micro-batch wait in microseconds\n" - " (default: 1000; 0 drains only queued work)\n" + " --batch-wait-us N First-arrival micro-batch deadline in us\n" + " (default: 0; drain queued work immediately)\n" " --memory-utilization F Fraction of physical memory the MLX model-set\n" " preflight may plan for (default: 0.90; values\n" " above 1.0 overcommit)\n" diff --git a/tests/test_server.c b/tests/test_server.c index 6f1e48e..2ed53a8 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -82,6 +82,67 @@ static void test_encode_embedding_binary(void) free(want); } +static void test_collect_job_batch_deadline(void) +{ + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + s.batch_wait_us = 1000; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + uint64_t first_ns = nstime() - 5000000u; + job first = {.srv = &s, .created_ns = first_ns}; + job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u}; + job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", + "/v1/embeddings"); + snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", + "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&before_deadline); + enqueue_job(&after_deadline); + + job *batch[3] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &before_deadline); + TEST_ASSERT(s.job_head == &after_deadline); + TEST_ASSERT(s.job_tail == &after_deadline); + + s.job_head = NULL; + s.job_tail = NULL; + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + +static void test_collect_job_batch_zero_wait(void) +{ + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + job first = {.srv = &s, .created_ns = nstime()}; + job second = {.srv = &s, .created_ns = nstime()}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&second); + + job *batch[2] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &second); + TEST_ASSERT(s.job_head == NULL); + TEST_ASSERT(s.job_tail == NULL); + + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + /* ==================================================================== * HTTP request-path tests against an in-process server * ==================================================================== */ @@ -613,6 +674,8 @@ int main(void) test_quantize_int8_tanh(); test_encode_embedding_int8(); test_encode_embedding_binary(); + test_collect_job_batch_deadline(); + test_collect_job_batch_zero_wait(); test_http_server(); if (test_failures) { fprintf(stderr, "server tests: %d failure(s)\n", test_failures); |
