aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-03 19:18:09 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commite751321fc00e07aeeb9337f5f21b7191b051c08a (patch)
tree2eaa7ad5075e37b31e45134bc40838c9b9f55e0e
parentcb6ae157a09b6794f5d6da27dbc97e0ad791a146 (diff)
downloadffwd-e751321fc00e07aeeb9337f5f21b7191b051c08a.tar.gz
ffwd-e751321fc00e07aeeb9337f5f21b7191b051c08a.zip
mlx: keep late maxsim device resident
-rw-r--r--pplx_embed_mlx.c403
-rw-r--r--pplx_embed_mlx.h27
2 files changed, 405 insertions, 25 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 7fc350c..793bf07 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -9,6 +9,7 @@
#include <string.h>
#include <math.h>
#include <limits.h>
+#include <float.h>
/* ========================================================================
* Per-layer MLX weight arrays
@@ -51,6 +52,13 @@ struct pplx_mlx_late_ctx {
int token_dim;
};
+struct pplx_mlx_late_vectors {
+ pplx_mlx_late_ctx_t *owner;
+ mlx_array vectors; /* [tokens, dim] */
+ int tokens;
+ int dim;
+};
+
/* ========================================================================
* Helpers
* ======================================================================== */
@@ -778,18 +786,18 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x,
return x;
}
-int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors)
+pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device(
+ pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
+ int normalize)
{
- if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || !out_vectors ||
+ if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 ||
ctx->token_dim <= 0 || !linear_ok(ctx->projection))
- return -1;
+ return NULL;
pplx_mlx_ctx_t *base = ctx->base;
if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
!arr_ok(base->embed_tokens) || !arr_ok(base->norm))
- return -1;
+ return NULL;
const pplx_config_t *c = &base->config;
mlx_stream S = base->stream;
@@ -797,31 +805,28 @@ int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
fprintf(stderr, "mlx late: invalid token id %d at position %d\n",
token_ids[i], i);
- return -1;
+ return NULL;
}
}
- size_t id_bytes, out_values, out_bytes;
- if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0 ||
- mlx_mul_size((size_t)n_tokens, (size_t)ctx->token_dim,
- &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
- return -1;
+ size_t id_bytes;
+ if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0)
+ return NULL;
int *ids_data = (int *)malloc(id_bytes);
- if (!ids_data) return -1;
+ if (!ids_data) return NULL;
memcpy(ids_data, token_ids, id_bytes);
int ids_shape[] = {1, n_tokens};
mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
free(ids_data);
- if (!arr_ok(ids)) return -1;
+ if (!arr_ok(ids)) return NULL;
- int rc = -1;
+ pplx_mlx_late_vectors_t *result = NULL;
mlx_array x = mlx_array_new();
mlx_array x_normed = (mlx_array){0};
mlx_array projected = (mlx_array){0};
- mlx_array out_f32 = (mlx_array){0};
+ mlx_array reshaped = (mlx_array){0};
mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
mlx_array_free(ids);
@@ -874,22 +879,370 @@ int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
if (!arr_ok(projected)) goto cleanup;
}
- out_f32 = mlx_array_new();
- mlx_astype(&out_f32, projected, MLX_FLOAT32, S);
- if (!arr_ok(out_f32)) goto cleanup;
+ int vec_shape[] = {n_tokens, ctx->token_dim};
+ reshaped = mlx_array_new();
+ mlx_reshape(&reshaped, projected, vec_shape, 2, S);
+ if (!arr_ok(reshaped)) goto cleanup;
+
+ result = (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*result));
+ if (!result) goto cleanup;
+ result->owner = ctx;
+ result->vectors = reshaped;
+ result->tokens = n_tokens;
+ result->dim = ctx->token_dim;
+ reshaped = (mlx_array){0};
+
+cleanup:
+ mlx_array_free(reshaped);
+ mlx_array_free(projected);
+ mlx_array_free(x_normed);
+ mlx_array_free(x);
+ return result;
+}
+
+void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs)
+{
+ if (!vecs) return;
+ mlx_array_free(vecs->vectors);
+ free(vecs);
+}
+
+int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs)
+{
+ return vecs ? vecs->tokens : 0;
+}
+
+int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs)
+{
+ return vecs ? vecs->dim : 0;
+}
+
+int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs,
+ float *out_vectors)
+{
+ if (!vecs || !out_vectors || !arr_ok(vecs->vectors) ||
+ vecs->tokens <= 0 || vecs->dim <= 0)
+ return -1;
+
+ size_t values, bytes;
+ if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 ||
+ mlx_mul_size(values, sizeof(float), &bytes) != 0)
+ return -1;
+
+ mlx_stream S = vecs->owner->base->stream;
+ mlx_array out_f32 = mlx_array_new();
+ mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S);
+ if (!arr_ok(out_f32)) {
+ mlx_array_free(out_f32);
+ return -1;
+ }
+ int rc = -1;
mlx_array_eval(out_f32);
const float *data = mlx_array_data_float32(out_f32);
if (data) {
- memcpy(out_vectors, data, out_bytes);
+ memcpy(out_vectors, data, bytes);
rc = 0;
}
+ mlx_array_free(out_f32);
+ return rc;
+}
+
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat(
+ pplx_mlx_late_ctx_t *ctx,
+ const pplx_mlx_late_vectors_t *const *items, int count)
+{
+ if (!ctx || !ctx->base || !items || count <= 0) return NULL;
+ int dim = 0;
+ int total = 0;
+ for (int i = 0; i < count; i++) {
+ const pplx_mlx_late_vectors_t *v = items[i];
+ if (!v || v->owner != ctx || !arr_ok(v->vectors) ||
+ v->tokens <= 0 || v->dim <= 0)
+ return NULL;
+ if (i == 0) dim = v->dim;
+ if (v->dim != dim || total > INT_MAX - v->tokens)
+ return NULL;
+ total += v->tokens;
+ }
+
+ mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
+ if (!parts) return NULL;
+ for (int i = 0; i < count; i++)
+ parts[i] = items[i]->vectors;
+
+ mlx_stream S = ctx->base->stream;
+ mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
+ free(parts);
+ if (!vec.ctx) return NULL;
+
+ mlx_array joined = mlx_array_new();
+ mlx_concatenate_axis(&joined, vec, 0, S);
+ mlx_vector_array_free(vec);
+ if (!arr_ok(joined)) {
+ mlx_array_free(joined);
+ return NULL;
+ }
+
+ pplx_mlx_late_vectors_t *out =
+ (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ if (!out) {
+ mlx_array_free(joined);
+ return NULL;
+ }
+ out->owner = ctx;
+ out->vectors = joined;
+ out->tokens = total;
+ out->dim = dim;
+ return out;
+}
+
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs,
+ const int *token_indices, int count)
+{
+ if (!ctx || !ctx->base || !vecs || vecs->owner != ctx ||
+ !arr_ok(vecs->vectors) || !token_indices || count <= 0 ||
+ vecs->tokens <= 0 || vecs->dim <= 0)
+ return NULL;
+
+ for (int i = 0; i < count; i++) {
+ if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens)
+ return NULL;
+ }
+
+ size_t idx_bytes;
+ if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
+ return NULL;
+ int *idx_data = (int *)malloc(idx_bytes);
+ if (!idx_data) return NULL;
+ memcpy(idx_data, token_indices, idx_bytes);
+
+ int idx_shape[] = {count};
+ mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
+ free(idx_data);
+ if (!arr_ok(idx)) return NULL;
+
+ mlx_stream S = ctx->base->stream;
+ mlx_array selected = mlx_array_new();
+ mlx_take_axis(&selected, vecs->vectors, idx, 0, S);
+ mlx_array_free(idx);
+ if (!arr_ok(selected)) {
+ mlx_array_free(selected);
+ return NULL;
+ }
+
+ pplx_mlx_late_vectors_t *out =
+ (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ if (!out) {
+ mlx_array_free(selected);
+ return NULL;
+ }
+ out->owner = ctx;
+ out->vectors = selected;
+ out->tokens = count;
+ out->dim = vecs->dim;
+ return out;
+}
+
+int pplx_mlx_late_maxsim_batch_device(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query,
+ const pplx_mlx_late_vectors_t *docs, const int *doc_offsets,
+ int docs_count, float *scores)
+{
+ if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 ||
+ !scores || query->owner != ctx || docs->owner != ctx ||
+ query->dim <= 0 || query->dim != docs->dim ||
+ query->tokens <= 0 || docs->tokens <= 0 ||
+ !arr_ok(query->vectors) || !arr_ok(docs->vectors) ||
+ doc_offsets[0] != 0)
+ return -1;
+
+ int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
+ if (!lengths) return -1;
+ int max_doc_tokens = 0;
+ for (int i = 0; i < docs_count; i++) {
+ if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
+ doc_offsets[i + 1] > docs->tokens) {
+ free(lengths);
+ return -1;
+ }
+ int n = doc_offsets[i + 1] - doc_offsets[i];
+ lengths[i] = n;
+ if (n > max_doc_tokens) max_doc_tokens = n;
+ }
+ if (max_doc_tokens <= 0) {
+ free(lengths);
+ return -1;
+ }
+
+ mlx_stream S = ctx->base->stream;
+ int dim = query->dim;
+ int rc = -1;
+ mlx_array *doc_rows =
+ (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
+ if (!doc_rows) {
+ free(lengths);
+ return -1;
+ }
+
+ mlx_vector_array doc_vec = (mlx_vector_array){0};
+ mlx_array pad_value = mlx_array_new_float32(0.0f);
+ mlx_array doc_batch = (mlx_array){0};
+ mlx_array query_3d = (mlx_array){0};
+ mlx_array query_batched = (mlx_array){0};
+ mlx_array docs_t = (mlx_array){0};
+ mlx_array sims = (mlx_array){0};
+ mlx_array lengths_arr = (mlx_array){0};
+ mlx_array positions = (mlx_array){0};
+ mlx_array lengths_2d = (mlx_array){0};
+ mlx_array positions_2d = (mlx_array){0};
+ mlx_array valid = (mlx_array){0};
+ mlx_array valid_3d = (mlx_array){0};
+ mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX);
+ mlx_array masked_sims = (mlx_array){0};
+ mlx_array maxes = (mlx_array){0};
+ mlx_array sums = (mlx_array){0};
+ mlx_array sums_f32 = (mlx_array){0};
+
+ for (int i = 0; i < docs_count; i++) {
+ int n = lengths[i];
+ int start[] = {doc_offsets[i], 0};
+ int stop[] = {doc_offsets[i + 1], dim};
+ int strides[] = {1, 1};
+
+ mlx_array doc = mlx_array_new();
+ mlx_array padded = (mlx_array){0};
+
+ mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
+ if (!arr_ok(doc)) goto cleanup;
+
+ if (n < max_doc_tokens) {
+ int axes[] = {0};
+ int low[] = {0};
+ int high[] = {max_doc_tokens - n};
+ padded = mlx_array_new();
+ mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value,
+ "constant", S);
+ mlx_array_free(doc);
+ doc = padded;
+ padded = (mlx_array){0};
+ if (!arr_ok(doc)) goto cleanup;
+ }
+
+ doc_rows[i] = mlx_array_new();
+ mlx_expand_dims(&doc_rows[i], doc, 0, S);
+ mlx_array_free(doc);
+ if (!arr_ok(doc_rows[i])) goto cleanup;
+ }
+
+ doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
+ if (!doc_vec.ctx) goto cleanup;
+
+ doc_batch = mlx_array_new();
+ mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
+ if (!arr_ok(doc_batch)) goto cleanup;
+
+ query_3d = mlx_array_new();
+ mlx_expand_dims(&query_3d, query->vectors, 0, S);
+ if (!arr_ok(query_3d)) goto cleanup;
+
+ int q_shape[] = {docs_count, query->tokens, dim};
+ query_batched = mlx_array_new();
+ mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
+ if (!arr_ok(query_batched)) goto cleanup;
+
+ int perm[] = {0, 2, 1};
+ docs_t = mlx_array_new();
+ mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
+ if (!arr_ok(docs_t)) goto cleanup;
+
+ sims = mlx_array_new();
+ mlx_matmul(&sims, query_batched, docs_t, S);
+ if (!arr_ok(sims)) goto cleanup;
+
+ int len_shape[] = {docs_count};
+ lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
+ if (!arr_ok(lengths_arr)) goto cleanup;
+
+ positions = mlx_array_new();
+ mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
+ if (!arr_ok(positions)) goto cleanup;
+
+ lengths_2d = mlx_array_new();
+ mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
+ if (!arr_ok(lengths_2d)) goto cleanup;
+
+ positions_2d = mlx_array_new();
+ mlx_expand_dims(&positions_2d, positions, 0, S);
+ if (!arr_ok(positions_2d)) goto cleanup;
+
+ valid = mlx_array_new();
+ mlx_less(&valid, positions_2d, lengths_2d, S);
+ if (!arr_ok(valid)) goto cleanup;
+
+ valid_3d = mlx_array_new();
+ mlx_expand_dims(&valid_3d, valid, 1, S);
+ if (!arr_ok(valid_3d)) goto cleanup;
+
+ masked_sims = mlx_array_new();
+ mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
+ if (!arr_ok(masked_sims)) goto cleanup;
+
+ maxes = mlx_array_new();
+ mlx_max_axis(&maxes, masked_sims, 2, false, S);
+ if (!arr_ok(maxes)) goto cleanup;
+
+ sums = mlx_array_new();
+ mlx_sum_axis(&sums, maxes, 1, false, S);
+ if (!arr_ok(sums)) goto cleanup;
+
+ sums_f32 = mlx_array_new();
+ mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
+ if (!arr_ok(sums_f32)) goto cleanup;
+
+ mlx_array_eval(sums_f32);
+ const float *data = mlx_array_data_float32(sums_f32);
+ if (!data) goto cleanup;
+ memcpy(scores, data, (size_t)docs_count * sizeof(float));
+ rc = 0;
cleanup:
- mlx_array_free(out_f32);
- mlx_array_free(projected);
- mlx_array_free(x_normed);
- mlx_array_free(x);
+ mlx_array_free(sums_f32);
+ mlx_array_free(sums);
+ mlx_array_free(maxes);
+ mlx_array_free(masked_sims);
+ mlx_array_free(invalid_value);
+ mlx_array_free(valid_3d);
+ mlx_array_free(valid);
+ mlx_array_free(positions_2d);
+ mlx_array_free(lengths_2d);
+ mlx_array_free(positions);
+ mlx_array_free(lengths_arr);
+ mlx_array_free(sims);
+ mlx_array_free(docs_t);
+ mlx_array_free(query_batched);
+ mlx_array_free(query_3d);
+ mlx_array_free(doc_batch);
+ if (doc_vec.ctx) mlx_vector_array_free(doc_vec);
+ mlx_array_free(pad_value);
+ for (int i = 0; i < docs_count; i++)
+ mlx_array_free(doc_rows[i]);
+ free(doc_rows);
+ free(lengths);
+ return rc;
+}
+
+int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
+ const int *token_ids, int n_tokens,
+ int normalize, float *out_vectors)
+{
+ pplx_mlx_late_vectors_t *vecs =
+ pplx_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens,
+ normalize);
+ if (!vecs) return -1;
+ int rc = pplx_mlx_late_vectors_copy(vecs, out_vectors);
+ pplx_mlx_late_vectors_free(vecs);
return rc;
}
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
index a7cb749..f2ad136 100644
--- a/pplx_embed_mlx.h
+++ b/pplx_embed_mlx.h
@@ -14,6 +14,7 @@
/* Opaque MLX model context */
typedef struct pplx_mlx_ctx pplx_mlx_ctx_t;
typedef struct pplx_mlx_late_ctx pplx_mlx_late_ctx_t;
+typedef struct pplx_mlx_late_vectors pplx_mlx_late_vectors_t;
typedef struct {
int quantize_bits; /* 0 disables; currently 8 */
@@ -65,6 +66,32 @@ int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
int normalize, float *out_vectors);
/*
+ * Device-resident late-interaction token vectors.
+ *
+ * The model context owns the MLX stream; the returned vector handle must not
+ * outlive ctx. Copying is optional: MaxSim can consume these handles directly
+ * and copy back only final candidate scores.
+ */
+pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device(
+ pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
+ int normalize);
+void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs);
+int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs);
+int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs);
+int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs,
+ float *out_vectors);
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat(
+ pplx_mlx_late_ctx_t *ctx,
+ const pplx_mlx_late_vectors_t *const *items, int count);
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs,
+ const int *token_indices, int count);
+int pplx_mlx_late_maxsim_batch_device(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query,
+ const pplx_mlx_late_vectors_t *docs, const int *doc_offsets,
+ int docs_count, float *scores);
+
+/*
* Compute embedding for token_ids[0..n_tokens-1].
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/