diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-02 08:50:46 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 0a4664e88871ee03cf638b6075c08d90d24e429b (patch) | |
| tree | bc422feba515fc95c9ca5cd8ee72257c73a3022a | |
| parent | 2b919b3381076bd5b235e93396ed56bf5ea8f08d (diff) | |
| download | ffwd-0a4664e88871ee03cf638b6075c08d90d24e429b.tar.gz ffwd-0a4664e88871ee03cf638b6075c08d90d24e429b.zip | |
server: bound mlx batches by padded token rows
| -rw-r--r-- | pplx_server.c | 27 |
1 files changed, 25 insertions, 2 deletions
diff --git a/pplx_server.c b/pplx_server.c index e981321..2bee40c 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -1005,6 +1005,26 @@ static int model_embed_spans_batch(loaded_model *m, out); } +static int model_uses_dense_batches(const loaded_model *m) { +#ifdef USE_MLX + return m->mlx_ctx != NULL; +#else + (void)m; + return 0; +#endif +} + +/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX + * pads each dense row to the longest input in the chunk. */ +static int inference_batch_accepts_input(const loaded_model *m, int batch, + int packed_tokens, int next_tokens, + int max_batch_tokens) { + if (batch == 0) return 1; + if (model_uses_dense_batches(m)) + return next_tokens <= max_batch_tokens / (batch + 1); + return next_tokens <= max_batch_tokens - packed_tokens; +} + static void append_embedding_object(sbuf *b, int index, const char *embedding) { sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); sbuf_puts(b, embedding); @@ -1125,7 +1145,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) int tokens = 0; while (start + cur < total_inputs && cur < max_batch) { int n_tokens = items[start + cur].input.n_tokens; - if (cur > 0 && n_tokens > max_batch_tokens - tokens) + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, + max_batch_tokens)) break; inputs[cur] = items[start + cur].input; tokens += n_tokens; @@ -1277,7 +1298,9 @@ static void execute_contextual_request_list(contextual_request **reqs, int chunks = 0; while (start + cur < total_docs && cur < max_batch) { const pplx_context_input_t *input = &items[start + cur].input; - if (cur > 0 && input->input.n_tokens > max_batch_tokens - tokens) + if (!inference_batch_accepts_input(m, cur, tokens, + input->input.n_tokens, + max_batch_tokens)) break; inputs[cur] = *input; tokens += input->input.n_tokens; |
