diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-22 00:42:18 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:21 +0200 |
| commit | 6d4f018051c3681588c36e88fed0ab48876f9a80 (patch) | |
| tree | b449a1a649c611b13aa51737fede87a964817064 | |
| parent | f1afb07b1ab10324cf0e6607c6b1a4fa018fe2e1 (diff) | |
| download | ffwd-6d4f018051c3681588c36e88fed0ab48876f9a80.tar.gz ffwd-6d4f018051c3681588c36e88fed0ab48876f9a80.zip | |
kernels: validate real BF16 artifacts
| -rw-r--r-- | pplx_embed_mlx.c | 31 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 52 |
2 files changed, 77 insertions, 6 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 74eb51d..c6991c5 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -44,11 +44,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, fprintf(stderr, "mlx: tensor not found: %s\n", name); return (mlx_array){0}; } - if (t->dtype != DTYPE_F32) { - fprintf(stderr, "mlx: expected F32 for %s\n", name); + mlx_dtype dtype; + if (t->dtype == DTYPE_F32) { + dtype = MLX_FLOAT32; + } else if (t->dtype == DTYPE_BF16) { + dtype = MLX_BFLOAT16; + } else { + fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); return (mlx_array){0}; } - return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32); + return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); } /* y = x @ W^T */ @@ -319,6 +324,16 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, mlx_array_free(lengths); return -1; } + if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) { + mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); + return -1; + } + } float scale = 1.0f / sqrtf((float)head_dim); mlx_array null_arr = (mlx_array){0}; @@ -447,14 +462,18 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, if (has_padding) mlx_array_free(attn_mask); /* 6. Eval and copy [B, hidden] to CPU. */ - mlx_array_eval(emb_n); - const float *data = mlx_array_data_float32(emb_n); + mlx_array emb_f32 = mlx_array_new(); + mlx_astype(&emb_f32, emb_n, MLX_FLOAT32, S); + mlx_array_free(emb_n); + + mlx_array_eval(emb_f32); + const float *data = mlx_array_data_float32(emb_f32); int rc = -1; if (data) { memcpy(out_embeddings, data, (size_t)batch * hidden * sizeof(float)); rc = 0; } - mlx_array_free(emb_n); + mlx_array_free(emb_f32); return rc; } diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index ec2ce9e..c299bc6 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -368,6 +368,50 @@ static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf1 } typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int seq_len; + int in_dim; + int out_dim; +} bf16_linear_rows_task_t; + +static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { + bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) end = t->seq_len; + if (start >= end) return; + + for (int s = start; s < end; s++) { + bf16_matvec_fused(t->y + (size_t)s * t->out_dim, + t->x + (size_t)s * t->in_dim, + t->W_bf16, t->bias, t->in_dim, t->out_dim); + } +} + +static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int seq_len, int in_dim, + int out_dim) { + bf16_linear_rows_task_t task = { + .y = y, + .x = x, + .W_bf16 = W_bf16, + .bias = bias, + .seq_len = seq_len, + .in_dim = in_dim, + .out_dim = out_dim, + }; + + if (tp.n_threads > 1 && seq_len >= 2) + parallel_for(bf16_linear_rows_worker, &task); + else + bf16_linear_rows_worker(0, 1, &task); +} + +typedef struct { float *q; float *k; float *v; @@ -460,6 +504,10 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; } + if (seq_len <= 16) { + bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); + return; + } size_t n = (size_t)out_dim * in_dim; const float *W_f32 = bf16_get_f32_view(W_bf16, n); if (!W_f32) return; @@ -472,6 +520,10 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim); return; } + if (seq_len <= 16) { + bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim); + return; + } size_t n = (size_t)out_dim * in_dim; const float *W_f32 = bf16_get_f32_view(W_bf16, n); if (!W_f32) return; |
