aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-04 19:11:19 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commita89fc2041de88b2b6fb326b3031a602333068052 (patch)
treed5339d90c8d3537cfd6f8b9f1b17a399518b12c0
parent75160e955a036b8bdd7b07597e971252581618ff (diff)
downloadffwd-a89fc2041de88b2b6fb326b3031a602333068052.tar.gz
ffwd-a89fc2041de88b2b6fb326b3031a602333068052.zip
mlx: optimize ragged embedding batches
-rw-r--r--pplx_embed_mlx.c116
1 files changed, 114 insertions, 2 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 6e5476a..bb948a8 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -1291,8 +1291,9 @@ int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
return rc;
}
-int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
- int batch, float *out_embeddings)
+static int pplx_mlx_embed_batch_dense(pplx_mlx_ctx_t *ctx,
+ const pplx_input_t *inputs, int batch,
+ float *out_embeddings)
{
if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers)
@@ -1445,6 +1446,117 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
return rc;
}
+typedef struct {
+ int index;
+ const pplx_input_t *input;
+} mlx_batch_item_t;
+
+static int mlx_batch_item_cmp(const void *a, const void *b)
+{
+ const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
+ const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
+ int da = ia->input->n_tokens;
+ int db = ib->input->n_tokens;
+ return (da > db) - (da < db);
+}
+
+static int mlx_batch_should_split(const pplx_input_t *inputs, int batch,
+ int *out_total_tokens, int *out_max_seq)
+{
+ int max_seq = 0;
+ int total_tokens = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0;
+ total_tokens += inputs[b].n_tokens;
+ }
+ if (out_total_tokens) *out_total_tokens = total_tokens;
+ if (out_max_seq) *out_max_seq = max_seq;
+ if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0;
+
+ size_t padded;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
+ return 0;
+ if (padded > SIZE_MAX / 5 || (size_t)total_tokens > SIZE_MAX / 11)
+ return 0;
+ return padded * 5 > (size_t)total_tokens * 11;
+}
+
+int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
+
+ if (!mlx_batch_should_split(inputs, batch, NULL, NULL))
+ return pplx_mlx_embed_batch_dense(ctx, inputs, batch, out_embeddings);
+
+ const int hidden = ctx->config.hidden_size;
+ mlx_batch_item_t *items = (mlx_batch_item_t *)malloc(
+ (size_t)batch * sizeof(*items));
+ pplx_input_t *group_inputs = (pplx_input_t *)malloc(
+ (size_t)batch * sizeof(*group_inputs));
+ size_t group_values, group_bytes;
+ if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
+ mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
+ free(items); free(group_inputs);
+ return -1;
+ }
+ float *group_out = (float *)malloc(group_bytes);
+ if (!items || !group_inputs || !group_out) {
+ free(items); free(group_inputs); free(group_out);
+ return -1;
+ }
+ for (int i = 0; i < batch; i++) {
+ items[i].index = i;
+ items[i].input = &inputs[i];
+ }
+ qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
+
+ int rc = 0;
+ for (int start = 0; start < batch && rc == 0; ) {
+ int group_max = 0;
+ int group_tokens = 0;
+ int end = start;
+ while (end < batch) {
+ int next_tokens = items[end].input->n_tokens;
+ int next_count = end - start + 1;
+ int next_max = next_tokens > group_max ? next_tokens : group_max;
+ int next_total = group_tokens + next_tokens;
+ if (end > start) {
+ size_t next_padded;
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max,
+ &next_padded) != 0 ||
+ next_padded > SIZE_MAX / 5 ||
+ (size_t)next_total > SIZE_MAX / 11 ||
+ next_padded * 5 > (size_t)next_total * 11)
+ break;
+ }
+ group_max = next_max;
+ group_tokens = next_total;
+ end++;
+ }
+
+ int group_count = end - start;
+ for (int i = 0; i < group_count; i++)
+ group_inputs[i] = *items[start + i].input;
+ rc = pplx_mlx_embed_batch_dense(ctx, group_inputs, group_count,
+ group_out);
+ if (rc != 0) break;
+ for (int i = 0; i < group_count; i++) {
+ int dst = items[start + i].index;
+ memcpy(out_embeddings + (size_t)dst * hidden,
+ group_out + (size_t)i * hidden,
+ (size_t)hidden * sizeof(float));
+ }
+ start = end;
+ }
+
+ free(items); free(group_inputs); free(group_out);
+ return rc;
+}
+
int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
int n_tokens, float *out_embedding)
{