aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-02 08:50:46 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit0a4664e88871ee03cf638b6075c08d90d24e429b (patch)
treebc422feba515fc95c9ca5cd8ee72257c73a3022a
parent2b919b3381076bd5b235e93396ed56bf5ea8f08d (diff)
downloadffwd-0a4664e88871ee03cf638b6075c08d90d24e429b.tar.gz
ffwd-0a4664e88871ee03cf638b6075c08d90d24e429b.zip
server: bound mlx batches by padded token rows
-rw-r--r--pplx_server.c27
1 files changed, 25 insertions, 2 deletions
diff --git a/pplx_server.c b/pplx_server.c
index e981321..2bee40c 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -1005,6 +1005,26 @@ static int model_embed_spans_batch(loaded_model *m,
out);
}
+static int model_uses_dense_batches(const loaded_model *m) {
+#ifdef USE_MLX
+ return m->mlx_ctx != NULL;
+#else
+ (void)m;
+ return 0;
+#endif
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+static int inference_batch_accepts_input(const loaded_model *m, int batch,
+ int packed_tokens, int next_tokens,
+ int max_batch_tokens) {
+ if (batch == 0) return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
static void append_embedding_object(sbuf *b, int index, const char *embedding) {
sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index);
sbuf_puts(b, embedding);
@@ -1125,7 +1145,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
int tokens = 0;
while (start + cur < total_inputs && cur < max_batch) {
int n_tokens = items[start + cur].input.n_tokens;
- if (cur > 0 && n_tokens > max_batch_tokens - tokens)
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens,
+ max_batch_tokens))
break;
inputs[cur] = items[start + cur].input;
tokens += n_tokens;
@@ -1277,7 +1298,9 @@ static void execute_contextual_request_list(contextual_request **reqs,
int chunks = 0;
while (start + cur < total_docs && cur < max_batch) {
const pplx_context_input_t *input = &items[start + cur].input;
- if (cur > 0 && input->input.n_tokens > max_batch_tokens - tokens)
+ if (!inference_batch_accepts_input(m, cur, tokens,
+ input->input.n_tokens,
+ max_batch_tokens))
break;
inputs[cur] = *input;
tokens += input->input.n_tokens;