aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-03 19:49:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitbd914f876842d5169616fa4eb257c44b55d4bf8b (patch)
treed8d4c74c01ac75d066a40e5d1f948e58a8189912
parent5c2750601749ce78068d29bd568af8809e14a01c (diff)
downloadffwd-bd914f876842d5169616fa4eb257c44b55d4bf8b.tar.gz
ffwd-bd914f876842d5169616fa4eb257c44b55d4bf8b.zip
mlx: cache transposed linear weights
-rw-r--r--pplx_embed_mlx.c50
1 files changed, 46 insertions, 4 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 793bf07..6acdf17 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -17,6 +17,7 @@
typedef struct {
mlx_array w;
+ mlx_array wt;
mlx_array scales;
mlx_array biases;
int quantized;
@@ -175,11 +176,28 @@ static int linear_ok(mlx_linear_t l)
static void free_linear(mlx_linear_t *l)
{
mlx_array_free(l->w);
+ mlx_array_free(l->wt);
mlx_array_free(l->scales);
mlx_array_free(l->biases);
memset(l, 0, sizeof(*l));
}
+static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
+{
+ if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
+ return 0;
+
+ l->wt = mlx_array_new();
+ mlx_transpose(&l->wt, l->w, s);
+ if (!arr_ok(l->wt)) {
+ mlx_array_free(l->wt);
+ l->wt = (mlx_array){0};
+ fprintf(stderr, "mlx: failed to prepare transposed linear weight\n");
+ return -1;
+ }
+ return 0;
+}
+
static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
mlx_stream s)
{
@@ -223,7 +241,9 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
mlx_synchronize(s);
mlx_array_free(l->w);
+ mlx_array_free(l->wt);
l->w = qw;
+ l->wt = (mlx_array){0};
l->scales = scales;
l->biases = biases;
l->quantized = 1;
@@ -245,11 +265,15 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
true, gs, bits, "affine", s);
return res;
}
- mlx_array Wt = mlx_array_new();
- mlx_transpose(&Wt, W->w, s);
mlx_array res = mlx_array_new();
- mlx_matmul(&res, x, Wt, s);
- mlx_array_free(Wt);
+ if (arr_ok(W->wt)) {
+ mlx_matmul(&res, x, W->wt, s);
+ } else {
+ mlx_array Wt = mlx_array_new();
+ mlx_transpose(&Wt, W->w, s);
+ mlx_matmul(&res, x, Wt, s);
+ mlx_array_free(Wt);
+ }
return res;
}
@@ -400,6 +424,17 @@ static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
quantize_linear(&l->down_proj, bits, group_size, s);
}
+static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s)
+{
+ return prepare_linear_transpose(&l->wq, s) ||
+ prepare_linear_transpose(&l->wk, s) ||
+ prepare_linear_transpose(&l->wv, s) ||
+ prepare_linear_transpose(&l->wo, s) ||
+ prepare_linear_transpose(&l->gate_proj, s) ||
+ prepare_linear_transpose(&l->up_proj, s) ||
+ prepare_linear_transpose(&l->down_proj, s);
+}
+
static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
int layer_start, int layer_end,
const pplx_mlx_options_t *opts,
@@ -519,6 +554,11 @@ static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
fprintf(stderr, "mlx: quantized linear weights to %d-bit "
"(group_size=%d)\n",
ctx->quantize_bits, ctx->quantize_group_size);
+ } else {
+ for (int i = layer_start; i < layer_end; i++) {
+ if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0)
+ goto fail;
+ }
}
if (pplx_verbose >= 2) {
@@ -641,6 +681,8 @@ pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options(
late->projection = load_linear_tensor(late->dense_ms, "linear.weight",
shape, 2);
if (!linear_ok(late->projection)) goto fail;
+ if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
+ goto fail;
late->token_dim = shape[0];
if (pplx_verbose >= 1)