From 707002111e3c6366d6822a3bb0c646b96534dd31 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 16 Jun 2026 08:39:03 +0200 Subject: mlx: upcast BERT norm/bias/embedding fields for BF16 models --- embed_mlx.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'embed_mlx.c') diff --git a/embed_mlx.c b/embed_mlx.c index 8b615ee..8d9de39 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -621,8 +621,21 @@ loaded: upcast_f32_inplace(&l->k_norm, ctx->stream); upcast_f32_inplace(&l->input_norm, ctx->stream); upcast_f32_inplace(&l->post_attn_norm, ctx->stream); + /* BERT block biases (no-ops for Qwen, whose arrays are unset); the + * two BERT LayerNorm weights reuse input_norm/post_attn_norm above. */ + upcast_f32_inplace(&l->o_bias, ctx->stream); + upcast_f32_inplace(&l->attn_ln_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_inter_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_out_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream); } upcast_f32_inplace(&ctx->norm, ctx->stream); + /* BERT embedding stage (no-ops for Qwen): learned position and token-type + * embeddings and the embedding LayerNorm all feed the F32 residual. */ + upcast_f32_inplace(&ctx->position_embeddings, ctx->stream); + upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream); + upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream); + upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream); } if (ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { -- cgit v1.3.1