aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-22 00:42:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:21 +0200
commit6d4f018051c3681588c36e88fed0ab48876f9a80 (patch)
treeb449a1a649c611b13aa51737fede87a964817064
parentf1afb07b1ab10324cf0e6607c6b1a4fa018fe2e1 (diff)
downloadffwd-6d4f018051c3681588c36e88fed0ab48876f9a80.tar.gz
ffwd-6d4f018051c3681588c36e88fed0ab48876f9a80.zip
kernels: validate real BF16 artifacts
-rw-r--r--pplx_embed_mlx.c31
-rw-r--r--qwen_asr_kernels.c52
2 files changed, 77 insertions, 6 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 74eb51d..c6991c5 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -44,11 +44,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
fprintf(stderr, "mlx: tensor not found: %s\n", name);
return (mlx_array){0};
}
- if (t->dtype != DTYPE_F32) {
- fprintf(stderr, "mlx: expected F32 for %s\n", name);
+ mlx_dtype dtype;
+ if (t->dtype == DTYPE_F32) {
+ dtype = MLX_FLOAT32;
+ } else if (t->dtype == DTYPE_BF16) {
+ dtype = MLX_BFLOAT16;
+ } else {
+ fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
return (mlx_array){0};
}
- return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32);
+ return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
}
/* y = x @ W^T */
@@ -319,6 +324,16 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
mlx_array_free(lengths);
return -1;
}
+ if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) {
+ mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ return -1;
+ }
+ }
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
@@ -447,14 +462,18 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
if (has_padding) mlx_array_free(attn_mask);
/* 6. Eval and copy [B, hidden] to CPU. */
- mlx_array_eval(emb_n);
- const float *data = mlx_array_data_float32(emb_n);
+ mlx_array emb_f32 = mlx_array_new();
+ mlx_astype(&emb_f32, emb_n, MLX_FLOAT32, S);
+ mlx_array_free(emb_n);
+
+ mlx_array_eval(emb_f32);
+ const float *data = mlx_array_data_float32(emb_f32);
int rc = -1;
if (data) {
memcpy(out_embeddings, data, (size_t)batch * hidden * sizeof(float));
rc = 0;
}
- mlx_array_free(emb_n);
+ mlx_array_free(emb_f32);
return rc;
}
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index ec2ce9e..c299bc6 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -368,6 +368,50 @@ static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf1
}
typedef struct {
+ float *y;
+ const float *x;
+ const uint16_t *W_bf16;
+ const float *bias;
+ int seq_len;
+ int in_dim;
+ int out_dim;
+} bf16_linear_rows_task_t;
+
+static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) {
+ bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg;
+ int chunk = (t->seq_len + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->seq_len) end = t->seq_len;
+ if (start >= end) return;
+
+ for (int s = start; s < end; s++) {
+ bf16_matvec_fused(t->y + (size_t)s * t->out_dim,
+ t->x + (size_t)s * t->in_dim,
+ t->W_bf16, t->bias, t->in_dim, t->out_dim);
+ }
+}
+
+static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int seq_len, int in_dim,
+ int out_dim) {
+ bf16_linear_rows_task_t task = {
+ .y = y,
+ .x = x,
+ .W_bf16 = W_bf16,
+ .bias = bias,
+ .seq_len = seq_len,
+ .in_dim = in_dim,
+ .out_dim = out_dim,
+ };
+
+ if (tp.n_threads > 1 && seq_len >= 2)
+ parallel_for(bf16_linear_rows_worker, &task);
+ else
+ bf16_linear_rows_worker(0, 1, &task);
+}
+
+typedef struct {
float *q;
float *k;
float *v;
@@ -460,6 +504,10 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
return;
}
+ if (seq_len <= 16) {
+ bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
+ return;
+ }
size_t n = (size_t)out_dim * in_dim;
const float *W_f32 = bf16_get_f32_view(W_bf16, n);
if (!W_f32) return;
@@ -472,6 +520,10 @@ void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim);
return;
}
+ if (seq_len <= 16) {
+ bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim);
+ return;
+ }
size_t n = (size_t)out_dim * in_dim;
const float *W_f32 = bf16_get_f32_view(W_bf16, n);
if (!W_f32) return;