aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pplx_server.c27
1 files changed, 25 insertions, 2 deletions
diff --git a/pplx_server.c b/pplx_server.c
index e981321..2bee40c 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -1005,6 +1005,26 @@ static int model_embed_spans_batch(loaded_model *m,
out);
}
+static int model_uses_dense_batches(const loaded_model *m) {
+#ifdef USE_MLX
+ return m->mlx_ctx != NULL;
+#else
+ (void)m;
+ return 0;
+#endif
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+static int inference_batch_accepts_input(const loaded_model *m, int batch,
+ int packed_tokens, int next_tokens,
+ int max_batch_tokens) {
+ if (batch == 0) return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
static void append_embedding_object(sbuf *b, int index, const char *embedding) {
sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index);
sbuf_puts(b, embedding);
@@ -1125,7 +1145,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
int tokens = 0;
while (start + cur < total_inputs && cur < max_batch) {
int n_tokens = items[start + cur].input.n_tokens;
- if (cur > 0 && n_tokens > max_batch_tokens - tokens)
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens,
+ max_batch_tokens))
break;
inputs[cur] = items[start + cur].input;
tokens += n_tokens;
@@ -1277,7 +1298,9 @@ static void execute_contextual_request_list(contextual_request **reqs,
int chunks = 0;
while (start + cur < total_docs && cur < max_batch) {
const pplx_context_input_t *input = &items[start + cur].input;
- if (cur > 0 && input->input.n_tokens > max_batch_tokens - tokens)
+ if (!inference_batch_accepts_input(m, cur, tokens,
+ input->input.n_tokens,
+ max_batch_tokens))
break;
inputs[cur] = *input;
tokens += input->input.n_tokens;