aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile128
-rw-r--r--main.c363
-rw-r--r--pplx_embed.c447
-rw-r--r--pplx_embed.h186
-rw-r--r--pplx_embed_mlx.c465
-rw-r--r--pplx_embed_mlx.h35
-rw-r--r--qwen_asr_kernels.c1247
-rw-r--r--qwen_asr_kernels.h173
-rw-r--r--qwen_asr_kernels_avx.c502
-rw-r--r--qwen_asr_kernels_generic.c65
-rw-r--r--qwen_asr_kernels_impl.h65
-rw-r--r--qwen_asr_kernels_neon.c274
-rw-r--r--qwen_asr_safetensors.c394
-rw-r--r--qwen_asr_safetensors.h77
-rw-r--r--qwen_asr_tokenizer.c650
-rw-r--r--qwen_asr_tokenizer.h38
16 files changed, 5109 insertions, 0 deletions
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..ee6b813
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,128 @@
+# pplx_embed - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
+# Makefile
+
+CC = gcc
+CFLAGS_BASE = -Wall -Wextra -O3 -march=native -ffast-math
+LDFLAGS = -lm -lpthread
+
+UNAME_S := $(shell uname -s)
+
+# Source files
+SRCS = pplx_embed.c \
+ qwen_asr_kernels.c \
+ qwen_asr_kernels_generic.c \
+ qwen_asr_kernels_neon.c \
+ qwen_asr_kernels_avx.c \
+ qwen_asr_tokenizer.c \
+ qwen_asr_safetensors.c
+
+MLX_SRCS = pplx_embed_mlx.c
+
+OBJS = $(SRCS:.c=.o)
+MLX_OBJS = $(MLX_SRCS:.c=.o)
+TARGET = pplx_embed
+
+.PHONY: all blas mlx debug clean help
+
+all: help
+
+help:
+ @echo "pplx_embed - Pure C inference for pplx-embed-v1-0.6b"
+ @echo ""
+ @echo "Build targets:"
+ @echo " make blas Build with BLAS acceleration (CPU)"
+ @echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)"
+ @echo " make debug Debug build with AddressSanitizer"
+ @echo " make clean Remove build artifacts"
+ @echo ""
+ @echo "Usage after build:"
+ @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\""
+ @echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)"
+
+# =============================================================================
+# BLAS build (Apple Accelerate on macOS, OpenBLAS on Linux)
+# =============================================================================
+ifeq ($(UNAME_S),Darwin)
+blas: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK
+blas: LDFLAGS += -framework Accelerate
+else
+blas: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -I/usr/include/openblas
+blas: LDFLAGS += -lopenblas
+endif
+blas:
+ $(MAKE) clean
+ $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
+ @echo ""
+ @echo "Built: $(TARGET) (CPU/BLAS)"
+
+# =============================================================================
+# MLX build (Apple Silicon GPU - uses mlx-c pure C API)
+# Requires: brew install mlx mlx-c
+# =============================================================================
+MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null)
+MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null)
+
+mlx: SRCS += pplx_embed_mlx.c
+mlx: OBJS = $(SRCS:.c=.o) pplx_embed_mlx.o
+ifeq ($(UNAME_S),Darwin)
+mlx: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX \
+ -I$(MLXC_PREFIX)/include
+mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \
+ -L$(MLXC_PREFIX)/lib -lmlxc \
+ -L$(MLX_PREFIX)/lib -lmlx \
+ -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib
+endif
+mlx:
+ $(MAKE) clean
+ $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
+ @echo ""
+ @echo "Built: $(TARGET) (CPU/BLAS + MLX GPU, use --mlx flag)"
+
+# =============================================================================
+# Debug build
+# =============================================================================
+debug: CFLAGS = -Wall -Wextra -g -O0 -DDEBUG -fsanitize=address
+debug: LDFLAGS += -fsanitize=address
+debug:
+ $(MAKE) clean
+ $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
+
+# =============================================================================
+# Link
+# =============================================================================
+$(TARGET): $(OBJS) $(EXTRA_OBJS) main.o
+ $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+# =============================================================================
+# Compile rules
+# =============================================================================
+pplx_embed.o: pplx_embed.c pplx_embed.h qwen_asr_kernels.h qwen_asr_safetensors.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+main.o: main.c pplx_embed.h qwen_asr_kernels.h qwen_asr_tokenizer.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_kernels_generic.o: qwen_asr_kernels_generic.c qwen_asr_kernels_impl.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_kernels_neon.o: qwen_asr_kernels_neon.c qwen_asr_kernels_impl.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_kernels_avx.o: qwen_asr_kernels_avx.c qwen_asr_kernels_impl.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_tokenizer.o: qwen_asr_tokenizer.c qwen_asr_tokenizer.h qwen_asr_kernels.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+qwen_asr_safetensors.o: qwen_asr_safetensors.c qwen_asr_safetensors.h
+ $(CC) $(CFLAGS) -c -o $@ $<
+
+# =============================================================================
+clean:
+ rm -f $(OBJS) pplx_embed_mlx.o main.o $(TARGET)
diff --git a/main.c b/main.c
new file mode 100644
index 0000000..cc2e492
--- /dev/null
+++ b/main.c
@@ -0,0 +1,363 @@
+/*
+ * main.c - pplx_embed command-line tool
+ *
+ * Usage:
+ * pplx_embed -d <model_dir> [options] <text> [<text2> ...]
+ * pplx_embed -d <model_dir> [options] (read lines from stdin)
+ *
+ * Options:
+ * -d <dir> Model directory (required)
+ * -t <n> Number of threads (default: all CPU cores)
+ * -e Print raw embedding vector(s) instead of similarity matrix
+ * -v Verbose output (use -vv for debug)
+ *
+ * Output:
+ * 1 text : prints the 1024-dim embedding as space-separated floats
+ * 2+ texts: prints cosine-similarity matrix between every pair,
+ * then optionally the individual embeddings if -e is given
+ *
+ * Example:
+ * ./pplx_embed -d /path/to/model "hello world" "hi there"
+ * ./pplx_embed -d /path/to/model -e "query: what is AI?"
+ */
+
+#include "pplx_embed.h"
+#include "qwen_asr_kernels.h"
+#include "qwen_asr_tokenizer.h"
+
+#ifdef USE_MLX
+#include "pplx_embed_mlx.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+
+/* ========================================================================
+ * Helpers
+ * ======================================================================== */
+
+static double now_ms(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
+}
+
+static void print_embedding(const float *emb, int dim)
+{
+ for (int i = 0; i < dim; i++) {
+ if (i > 0) putchar(' ');
+ printf("%.8f", (double)emb[i]);
+ }
+ putchar('\n');
+}
+
+static void print_usage(const char *prog)
+{
+ fprintf(stderr,
+ "Usage: %s -d <model_dir> [options] [text...]\n"
+ "\n"
+ "Options:\n"
+ " -d <dir> Model directory (model.safetensors + vocab.json + merges.txt)\n"
+ " -t <n> Number of threads (default: all CPU cores, CPU backend)\n"
+ " --mlx Use Apple MLX GPU backend (Apple Silicon only)\n"
+ " -e Always print raw embedding vector(s)\n"
+ " -v Verbose (repeat for more: -vv)\n"
+ " -h Show this help\n"
+ "\n"
+ "With 1 text: prints the 1024-dim embedding vector\n"
+ "With 2+ texts: prints cosine similarity between each pair\n"
+ " (add -e to also print the individual embeddings)\n"
+ "\n"
+ "If no text arguments are given, reads one text per line from stdin.\n"
+ "\n"
+ "Example:\n"
+ " %s -d ./model \"query: what is AI?\" \"document: AI is artificial intelligence\"\n"
+ " %s -d ./model --mlx \"query: what is AI?\"\n",
+ prog, prog, prog);
+}
+
+/* ========================================================================
+ * Process one text: tokenize -> embed, returns malloc'd float[dim]
+ * ======================================================================== */
+
+static float *process_text(pplx_ctx_t *ctx,
+#ifdef USE_MLX
+ pplx_mlx_ctx_t *mlx_ctx,
+#endif
+ qwen_tokenizer_t *tok,
+ const char *text,
+ int print_tokens)
+{
+ /* Tokenize */
+ int n_tokens = 0;
+ int *token_ids = qwen_tokenizer_encode(tok, text, &n_tokens);
+ if (!token_ids || n_tokens == 0) {
+ fprintf(stderr, "pplx_embed: tokenization failed for: %s\n", text);
+ free(token_ids);
+ return NULL;
+ }
+
+ if (print_tokens || pplx_verbose >= 1) {
+ fprintf(stderr, "tokens (%d): ", n_tokens);
+ for (int i = 0; i < n_tokens && i < 20; i++)
+ fprintf(stderr, "%d ", token_ids[i]);
+ if (n_tokens > 20) fprintf(stderr, "...");
+ fprintf(stderr, "\n");
+ }
+
+ /* Embed */
+ double t0 = now_ms();
+ float *emb = NULL;
+#ifdef USE_MLX
+ if (mlx_ctx)
+ emb = pplx_mlx_embed(mlx_ctx, token_ids, n_tokens);
+ else
+#endif
+ emb = pplx_embed(ctx, token_ids, n_tokens);
+ double dt = now_ms() - t0;
+ free(token_ids);
+
+ if (!emb) {
+ fprintf(stderr, "pplx_embed: forward pass failed\n");
+ return NULL;
+ }
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "embed: %d tokens in %.1f ms\n", n_tokens, dt);
+
+ return emb;
+}
+
+/* ========================================================================
+ * Main
+ * ======================================================================== */
+
+int main(int argc, char *argv[])
+{
+ const char *model_dir = NULL;
+ int n_threads = 0; /* 0 = auto */
+ int print_embs = 0;
+ int verbose = 0;
+ int use_mlx = 0;
+
+ /* ---- Parse flags ---- */
+ int arg_start = 1;
+ while (arg_start < argc && argv[arg_start][0] == '-') {
+ const char *flag = argv[arg_start];
+ if (strcmp(flag, "-d") == 0) {
+ if (arg_start + 1 >= argc) {
+ fprintf(stderr, "pplx_embed: -d requires an argument\n");
+ return 1;
+ }
+ model_dir = argv[++arg_start];
+ } else if (strcmp(flag, "-t") == 0) {
+ if (arg_start + 1 >= argc) {
+ fprintf(stderr, "pplx_embed: -t requires an argument\n");
+ return 1;
+ }
+ n_threads = atoi(argv[++arg_start]);
+ } else if (strcmp(flag, "-e") == 0) {
+ print_embs = 1;
+ } else if (strcmp(flag, "--mlx") == 0) {
+#ifdef USE_MLX
+ use_mlx = 1;
+#else
+ fprintf(stderr, "pplx_embed: --mlx not available (build with: make mlx)\n");
+ return 1;
+#endif
+ } else if (strcmp(flag, "-v") == 0) {
+ verbose++;
+ } else if (strcmp(flag, "-vv") == 0) {
+ verbose = 2;
+ } else if (strcmp(flag, "-h") == 0 || strcmp(flag, "--help") == 0) {
+ print_usage(argv[0]);
+ return 0;
+ } else {
+ /* Stop flag parsing on first non-flag argument */
+ break;
+ }
+ arg_start++;
+ }
+
+ if (!model_dir || model_dir[0] == '\0') {
+ fprintf(stderr, "pplx_embed: model directory required (-d <dir>)\n"
+ " Hint: if using MODEL=.. ./pplx_embed -d \"$MODEL\",\n"
+ " make sure MODEL is exported or use: MODEL=.. && ./pplx_embed -d \"$MODEL\"\n");
+ print_usage(argv[0]);
+ return 1;
+ }
+
+ /* ---- Set verbosity ---- */
+ pplx_verbose = verbose;
+ qwen_verbose = verbose; /* kernels share the same flag name */
+
+ /* ---- Set thread count (CPU backend only) ---- */
+ if (!use_mlx) {
+ if (n_threads <= 0)
+ n_threads = qwen_get_num_cpus();
+ qwen_set_threads(n_threads);
+ if (verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ } else {
+ if (verbose >= 1)
+ fprintf(stderr, "Using MLX GPU backend\n");
+ }
+
+ /* ---- Load tokenizer ---- */
+ char vocab_path[1024];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+
+ double t0_load = now_ms();
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ if (!tok) {
+ fprintf(stderr, "pplx_embed: failed to load tokenizer from %s\n", vocab_path);
+ return 1;
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Tokenizer loaded in %.0f ms\n", now_ms() - t0_load);
+
+ /* ---- Load model ---- */
+ pplx_ctx_t *ctx = NULL;
+#ifdef USE_MLX
+ pplx_mlx_ctx_t *mlx_ctx = NULL;
+#endif
+
+ double t0_model = now_ms();
+
+#ifdef USE_MLX
+ if (use_mlx) {
+ mlx_ctx = pplx_mlx_load(model_dir);
+ if (!mlx_ctx) {
+ fprintf(stderr, "pplx_embed: failed to load MLX model from %s\n", model_dir);
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ } else
+#endif
+ {
+ ctx = pplx_load(model_dir);
+ if (!ctx) {
+ fprintf(stderr, "pplx_embed: failed to load model from %s\n", model_dir);
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Model loaded in %.0f ms%s\n", now_ms() - t0_model,
+ use_mlx ? " (MLX)" : "");
+
+ int dim = PPLX_HIDDEN_SIZE;
+
+ /* ---- Gather input texts ---- */
+ /* texts[] and embeddings[] are grown dynamically */
+ int n_texts = 0;
+ int texts_cap = 0;
+ char **texts = NULL;
+ float **embeddings = NULL;
+
+ /* Helper to append a text */
+#define APPEND_TEXT(s) do { \
+ if (n_texts == texts_cap) { \
+ texts_cap = texts_cap ? texts_cap * 2 : 8; \
+ texts = realloc(texts, texts_cap * sizeof(char *)); \
+ embeddings = realloc(embeddings, texts_cap * sizeof(float *)); \
+ if (!texts || !embeddings) { \
+ fprintf(stderr, "pplx_embed: out of memory\n"); return 1; \
+ } \
+ } \
+ texts[n_texts] = strdup(s); \
+ embeddings[n_texts] = NULL; \
+ n_texts++; \
+} while (0)
+
+ if (arg_start < argc) {
+ /* Text arguments on command line */
+ for (int i = arg_start; i < argc; i++)
+ APPEND_TEXT(argv[i]);
+ } else {
+ /* Read lines from stdin */
+ char line[65536];
+ while (fgets(line, sizeof(line), stdin)) {
+ /* Strip trailing newline */
+ size_t l = strlen(line);
+ while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
+ line[--l] = '\0';
+ if (l == 0) continue;
+ APPEND_TEXT(line);
+ }
+ }
+
+ if (n_texts == 0) {
+ fprintf(stderr, "pplx_embed: no input texts\n");
+ pplx_free(ctx);
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+
+ /* ---- Embed all texts ---- */
+ for (int i = 0; i < n_texts; i++) {
+ if (verbose >= 1)
+ fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
+
+ embeddings[i] = process_text(ctx,
+#ifdef USE_MLX
+ mlx_ctx,
+#endif
+ tok, texts[i],
+ /*print_tokens=*/ verbose >= 2);
+ if (!embeddings[i]) {
+ /* Error already printed */
+ goto cleanup;
+ }
+ }
+
+ /* ---- Output ---- */
+ if (n_texts == 1) {
+ /* Single text: always print the embedding */
+ print_embedding(embeddings[0], dim);
+ } else {
+ /* Multiple texts: print cosine similarity matrix */
+ printf("Cosine similarity matrix (%d texts):\n", n_texts);
+
+ /* Header */
+ printf("%-4s", "");
+ for (int j = 0; j < n_texts; j++) printf(" [%d] ", j);
+ printf("\n");
+
+ for (int i = 0; i < n_texts; i++) {
+ printf("[%d] ", i);
+ for (int j = 0; j < n_texts; j++) {
+ float sim = pplx_cosine_similarity(embeddings[i], embeddings[j], dim);
+ printf(" %.4f", (double)sim);
+ }
+ printf(" \"%s\"\n", texts[i]);
+ }
+
+ /* Print raw embeddings if requested */
+ if (print_embs) {
+ printf("\nEmbeddings:\n");
+ for (int i = 0; i < n_texts; i++) {
+ printf("[%d] ", i);
+ print_embedding(embeddings[i], dim);
+ }
+ }
+ }
+
+cleanup:
+ for (int i = 0; i < n_texts; i++) {
+ free(texts[i]);
+ free(embeddings[i]);
+ }
+ free(texts);
+ free(embeddings);
+ if (ctx) pplx_free(ctx);
+#ifdef USE_MLX
+ if (mlx_ctx) pplx_mlx_free(mlx_ctx);
+#endif
+ qwen_tokenizer_free(tok);
+ return 0;
+}
diff --git a/pplx_embed.c b/pplx_embed.c
new file mode 100644
index 0000000..2220760
--- /dev/null
+++ b/pplx_embed.c
@@ -0,0 +1,447 @@
+/*
+ * pplx_embed.c - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
+ *
+ * Transformer forward pass:
+ * embed -> 28x[RMSNorm -> QKV (no bias) -> Q/K RMSNorm -> RoPE
+ * -> Bidirectional GQA attention -> O proj -> residual
+ * -> RMSNorm -> SwiGLU MLP -> residual]
+ * -> final RMSNorm -> mean pool -> L2 normalize
+ *
+ * Key implementation choices:
+ * - All weights are F32 (mmap'd direct, no copy)
+ * - Bidirectional attention: reuse qwen_causal_attention with
+ * q_offset = (seq_len - 1), which removes the causal mask
+ * (every query sees all seq_len keys)
+ * - GQA ratio 2:1 (16 Q-heads, 8 KV-heads) handled by causal_attention
+ * - RoPE cache grown lazily, positions [0 .. seq_len-1]
+ */
+
+#include "pplx_embed.h"
+#include "qwen_asr_kernels.h"
+#include "qwen_asr_safetensors.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+/* ========================================================================
+ * Global verbose flags
+ * qwen_verbose is declared extern in qwen_asr_kernels.h and qwen_asr_tokenizer.c;
+ * we own its definition here since we don't link qwen_asr.c.
+ * ======================================================================== */
+
+int pplx_verbose = 0;
+int qwen_verbose = 0; /* required by qwen_asr_kernels.c and qwen_asr_tokenizer.c */
+
+/* ========================================================================
+ * Internal helpers: weight loading
+ * ======================================================================== */
+
+/*
+ * Return a direct pointer into the mmap'd safetensors region for an F32 tensor.
+ * No allocation, no copy; caller must NOT free the returned pointer.
+ */
+static const float *load_f32_direct(multi_safetensors_t *ms, const char *name)
+{
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "pplx_embed: tensor not found: %s\n", name);
+ return NULL;
+ }
+ if (t->dtype != DTYPE_F32) {
+ fprintf(stderr, "pplx_embed: expected F32 for %s, got dtype=%d\n",
+ name, (int)t->dtype);
+ return NULL;
+ }
+ return (const float *)safetensors_data(sf, t);
+}
+
+/* ========================================================================
+ * Internal helpers: working buffers
+ * ======================================================================== */
+
+static int ensure_buffers(pplx_ctx_t *ctx, int seq)
+{
+ if (seq <= ctx->buf_seq_cap) return 0;
+
+ int new_cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64;
+ while (new_cap < seq) new_cap *= 2;
+
+ int hidden = ctx->config.hidden_size;
+ int q_dim = ctx->config.q_dim;
+ int kv_dim = ctx->config.kv_dim;
+ int inter = ctx->config.intermediate_size;
+
+#define PPLX_REALLOC(ptr, n) do { \
+ void *_tmp = realloc((ptr), (size_t)(n) * sizeof(float)); \
+ if (!_tmp) return -1; \
+ (ptr) = (float *)_tmp; \
+} while (0)
+
+ PPLX_REALLOC(ctx->x, new_cap * hidden);
+ PPLX_REALLOC(ctx->x_norm, new_cap * hidden);
+ PPLX_REALLOC(ctx->q, new_cap * q_dim);
+ PPLX_REALLOC(ctx->k, new_cap * kv_dim);
+ PPLX_REALLOC(ctx->v, new_cap * kv_dim);
+ PPLX_REALLOC(ctx->attn_out, new_cap * q_dim);
+ PPLX_REALLOC(ctx->proj_out, new_cap * hidden);
+ PPLX_REALLOC(ctx->ffn_gate, new_cap * inter);
+ PPLX_REALLOC(ctx->ffn_up, new_cap * inter);
+ PPLX_REALLOC(ctx->ffn_out, new_cap * hidden);
+
+#undef PPLX_REALLOC
+
+ ctx->buf_seq_cap = new_cap;
+ return 0;
+}
+
+/* ========================================================================
+ * Internal helpers: RoPE cache
+ *
+ * Stores cos/sin for positions [0 .. rope_cache_cap-1].
+ * Layout: [pos, head_dim] - values are duplicated across the two halves
+ * so that qwen_apply_rope_neox can load contiguous cos[d] and cos[half+d]
+ * from a single row: cos_row[d] == cos_row[half+d].
+ * ======================================================================== */
+
+static int ensure_rope_cache(pplx_ctx_t *ctx, int n_pos)
+{
+ if (n_pos <= ctx->rope_cache_cap) return 0;
+
+ int new_cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512;
+ while (new_cap < n_pos) new_cap *= 2;
+
+ int head_dim = ctx->config.head_dim;
+ int half = head_dim / 2;
+ float theta = ctx->config.rope_theta;
+
+ size_t n = (size_t)new_cap * head_dim;
+ float *new_cos = (float *)realloc(ctx->rope_cos, n * sizeof(float));
+ float *new_sin = (float *)realloc(ctx->rope_sin, n * sizeof(float));
+ if (!new_cos || !new_sin) {
+ /* If realloc failed, partial state is still valid up to old cap */
+ if (new_cos) ctx->rope_cos = new_cos;
+ if (new_sin) ctx->rope_sin = new_sin;
+ return -1;
+ }
+ ctx->rope_cos = new_cos;
+ ctx->rope_sin = new_sin;
+
+ /* Compute only the newly needed positions */
+ for (int pos = ctx->rope_cache_cap; pos < new_cap; pos++) {
+ float fp = (float)pos;
+ float *c = ctx->rope_cos + (size_t)pos * head_dim;
+ float *s = ctx->rope_sin + (size_t)pos * head_dim;
+ for (int d = 0; d < half; d++) {
+ /* inv_freq[d] = 1 / (theta ^ (2d / head_dim)) */
+ float inv_freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
+ float angle = fp * inv_freq;
+ float cv = cosf(angle);
+ float sv = sinf(angle);
+ /* Duplicate across both halves for the NeoX rotation kernel */
+ c[d] = cv;
+ c[half + d] = cv;
+ s[d] = sv;
+ s[half + d] = sv;
+ }
+ }
+
+ ctx->rope_cache_cap = new_cap;
+ return 0;
+}
+
+/* ========================================================================
+ * Load
+ * ======================================================================== */
+
+pplx_ctx_t *pplx_load(const char *model_dir)
+{
+ pplx_ctx_t *ctx = (pplx_ctx_t *)calloc(1, sizeof(pplx_ctx_t));
+ if (!ctx) return NULL;
+
+ /* ---- Hardcoded config (mirrors config.json) ---- */
+ pplx_config_t *cfg = &ctx->config;
+ cfg->hidden_size = PPLX_HIDDEN_SIZE;
+ cfg->n_layers = PPLX_NUM_LAYERS;
+ cfg->n_heads = PPLX_NUM_HEADS;
+ cfg->n_kv_heads = PPLX_NUM_KV_HEADS;
+ cfg->head_dim = PPLX_HEAD_DIM;
+ cfg->q_dim = PPLX_Q_DIM;
+ cfg->kv_dim = PPLX_KV_DIM;
+ cfg->intermediate_size = PPLX_INTERMEDIATE_SIZE;
+ cfg->vocab_size = PPLX_VOCAB_SIZE;
+ cfg->rms_norm_eps = 1e-6f;
+ cfg->rope_theta = 1000000.0f;
+
+ /* ---- Open safetensors ---- */
+ multi_safetensors_t *ms = multi_safetensors_open(model_dir);
+ if (!ms) {
+ fprintf(stderr, "pplx_load: failed to open safetensors in %s\n", model_dir);
+ free(ctx);
+ return NULL;
+ }
+ ctx->safetensors = ms;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "pplx_load: loading weights from %s\n", model_dir);
+
+ /* ---- Token embeddings ---- */
+ pplx_weights_t *w = &ctx->weights;
+ w->embed_tokens = load_f32_direct(ms, "embed_tokens.weight");
+ if (!w->embed_tokens) goto fail;
+
+ /* ---- Per-layer weights ---- */
+ char name[256];
+ for (int i = 0; i < cfg->n_layers; i++) {
+ pplx_layer_t *l = &w->layers[i];
+
+#define LOAD(field, fmt) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ l->field = load_f32_direct(ms, name); \
+ if (!l->field) goto fail; \
+} while (0)
+
+ LOAD(wq, "layers.%d.self_attn.q_proj.weight");
+ LOAD(wk, "layers.%d.self_attn.k_proj.weight");
+ LOAD(wv, "layers.%d.self_attn.v_proj.weight");
+ LOAD(wo, "layers.%d.self_attn.o_proj.weight");
+ LOAD(q_norm, "layers.%d.self_attn.q_norm.weight");
+ LOAD(k_norm, "layers.%d.self_attn.k_norm.weight");
+ LOAD(input_norm, "layers.%d.input_layernorm.weight");
+ LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight");
+ LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight");
+ LOAD(up_proj, "layers.%d.mlp.up_proj.weight");
+ LOAD(down_proj, "layers.%d.mlp.down_proj.weight");
+
+#undef LOAD
+
+ if (pplx_verbose >= 2)
+ fprintf(stderr, " layer %d loaded\n", i);
+ }
+
+ /* ---- Final RMSNorm ---- */
+ w->norm = load_f32_direct(ms, "norm.weight");
+ if (!w->norm) goto fail;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "pplx_load: all weights loaded\n");
+
+ return ctx;
+
+fail:
+ pplx_free(ctx);
+ return NULL;
+}
+
+/* ========================================================================
+ * Free
+ * ======================================================================== */
+
+void pplx_free(pplx_ctx_t *ctx)
+{
+ if (!ctx) return;
+ if (ctx->safetensors)
+ multi_safetensors_close((multi_safetensors_t *)ctx->safetensors);
+
+ free(ctx->x);
+ free(ctx->x_norm);
+ free(ctx->q);
+ free(ctx->k);
+ free(ctx->v);
+ free(ctx->attn_out);
+ free(ctx->proj_out);
+ free(ctx->ffn_gate);
+ free(ctx->ffn_up);
+ free(ctx->ffn_out);
+ free(ctx->rope_cos);
+ free(ctx->rope_sin);
+ free(ctx);
+}
+
+/* ========================================================================
+ * Forward pass
+ * ======================================================================== */
+
+float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ const pplx_config_t *cfg = &ctx->config;
+ const pplx_weights_t *w = &ctx->weights;
+
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
+ int n_kv_heads = cfg->n_kv_heads;
+ int head_dim = cfg->head_dim;
+ int q_dim = cfg->q_dim;
+ int kv_dim = cfg->kv_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->rms_norm_eps;
+ int seq = n_tokens;
+
+ /* Allocate/grow working buffers */
+ if (ensure_buffers(ctx, seq) != 0) {
+ fprintf(stderr, "pplx_embed: buffer allocation failed\n");
+ return NULL;
+ }
+
+ /* Pre-compute RoPE tables for positions [0, seq-1] */
+ if (ensure_rope_cache(ctx, seq) != 0) {
+ fprintf(stderr, "pplx_embed: RoPE cache allocation failed\n");
+ return NULL;
+ }
+ const float *rope_cos = ctx->rope_cos; /* [seq_cap, head_dim] */
+ const float *rope_sin = ctx->rope_sin;
+
+ float *x = ctx->x;
+ float *x_norm = ctx->x_norm;
+ float *q = ctx->q;
+ float *k = ctx->k;
+ float *v = ctx->v;
+ float *attn_out = ctx->attn_out;
+ float *proj_out = ctx->proj_out;
+ float *ffn_gate = ctx->ffn_gate;
+ float *ffn_up = ctx->ffn_up;
+ float *ffn_out = ctx->ffn_out;
+
+ /* ------------------------------------------------------------------
+ * Step 1: Token embedding lookup
+ * x[i, :] = embed_tokens[token_ids[i], :]
+ * ------------------------------------------------------------------ */
+ for (int i = 0; i < seq; i++) {
+ int id = token_ids[i];
+ if (id < 0 || id >= cfg->vocab_size) {
+ fprintf(stderr, "pplx_embed: invalid token id %d at position %d\n", id, i);
+ return NULL;
+ }
+ memcpy(x + i * hidden,
+ w->embed_tokens + (size_t)id * hidden,
+ hidden * sizeof(float));
+ }
+
+ float scale = 1.0f / sqrtf((float)head_dim);
+
+ /* ------------------------------------------------------------------
+ * Steps 2..29: 28 transformer layers
+ * ------------------------------------------------------------------ */
+ for (int layer = 0; layer < cfg->n_layers; layer++) {
+ const pplx_layer_t *l = &w->layers[layer];
+
+ /* ---- 2a. Input RMSNorm ---- */
+ qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps);
+
+ /* ---- 2b. QKV projections (no bias) ----
+ * q: [seq, q_dim] = x_norm @ wq^T (wq is [q_dim, hidden])
+ * k: [seq, kv_dim] = x_norm @ wk^T
+ * v: [seq, kv_dim] = x_norm @ wv^T
+ */
+ qwen_linear_nobias(q, x_norm, l->wq, seq, hidden, q_dim);
+ qwen_linear_nobias(k, x_norm, l->wk, seq, hidden, kv_dim);
+ qwen_linear_nobias(v, x_norm, l->wv, seq, hidden, kv_dim);
+
+ /* ---- 2c. Per-head Q/K RMSNorm (Qwen3 feature) ----
+ * Normalize each head's [head_dim] segment independently.
+ */
+ qwen_rms_norm_per_head(q, l->q_norm, seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(k, l->k_norm, seq, n_kv_heads, head_dim, eps);
+
+ /* ---- 2d. Apply NeoX-style RoPE to Q and K ----
+ * rope_cos/sin: [seq_cap, head_dim], positions 0..seq-1 cached
+ */
+ qwen_apply_rope_neox(q, rope_cos, rope_sin, seq, n_heads, head_dim);
+ qwen_apply_rope_neox(k, rope_cos, rope_sin, seq, n_kv_heads, head_dim);
+
+ /* ---- 2e. Bidirectional GQA attention ----
+ *
+ * We reuse qwen_causal_attention with q_offset = (seq - 1).
+ *
+ * Inside the kernel for query position i:
+ * global_pos = q_offset + i = (seq-1) + i
+ * k_end = min(global_pos + 1, seq_k)
+ * = min(seq + i, seq) = seq for all i >= 0
+ *
+ * So every query attends to ALL seq key positions - full attention,
+ * no causal mask, while still benefiting from the GQA + threading
+ * in qwen_causal_attention.
+ */
+ qwen_causal_attention(attn_out, q, k, v,
+ seq, /* seq_q */
+ seq, /* seq_k */
+ n_heads, n_kv_heads, head_dim,
+ scale,
+ seq - 1); /* q_offset - removes causal mask */
+
+ /* ---- 2f. Output projection + residual ----
+ * proj_out = attn_out @ wo^T (wo is [hidden, q_dim])
+ * x += proj_out
+ */
+ qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden);
+ qwen_add_inplace(x, proj_out, seq * hidden);
+
+ /* ---- 2g. Post-attention RMSNorm ---- */
+ qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps);
+
+ /* ---- 2h. SwiGLU MLP ----
+ * gate = gate_proj(x_norm) [seq, intermediate]
+ * up = up_proj(x_norm) [seq, intermediate]
+ * mid = SiLU(gate) * up [seq, intermediate]
+ * out = down_proj(mid) [seq, hidden]
+ */
+ qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter);
+ qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter);
+
+ qwen_silu(ffn_gate, seq * inter); /* SiLU(gate) in-place */
+ qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); /* gate *= up */
+
+ qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden);
+ qwen_add_inplace(x, ffn_out, seq * hidden);
+ }
+
+ /* ------------------------------------------------------------------
+ * Step 3: Final RMSNorm (in-place on x)
+ * ------------------------------------------------------------------ */
+ qwen_rms_norm(x, x, w->norm, seq, hidden, eps);
+
+ /* ------------------------------------------------------------------
+ * Step 4: Mean pooling across all token positions
+ * embedding[d] = mean over i in [0, seq) of x[i, d]
+ * ------------------------------------------------------------------ */
+ float *embedding = (float *)calloc(hidden, sizeof(float));
+ if (!embedding) return NULL;
+
+ for (int i = 0; i < seq; i++) {
+ const float *row = x + i * hidden;
+ for (int d = 0; d < hidden; d++) {
+ embedding[d] += row[d];
+ }
+ }
+ float inv_seq = 1.0f / (float)seq;
+ for (int d = 0; d < hidden; d++) embedding[d] *= inv_seq;
+
+ /* ------------------------------------------------------------------
+ * Step 5: L2 normalization
+ * embedding /= ||embedding||_2
+ * ------------------------------------------------------------------ */
+ float norm_sq = 0.0f;
+ for (int d = 0; d < hidden; d++) norm_sq += embedding[d] * embedding[d];
+ float norm_val = sqrtf(norm_sq);
+ if (norm_val > 1e-8f) {
+ float inv = 1.0f / norm_val;
+ for (int d = 0; d < hidden; d++) embedding[d] *= inv;
+ }
+
+ return embedding;
+}
+
+/* ========================================================================
+ * Cosine similarity (dot product of two L2-normalized vectors)
+ * ======================================================================== */
+
+float pplx_cosine_similarity(const float *a, const float *b, int dim)
+{
+ float dot = 0.0f;
+ for (int i = 0; i < dim; i++) dot += a[i] * b[i];
+ return dot;
+}
diff --git a/pplx_embed.h b/pplx_embed.h
new file mode 100644
index 0000000..69b67c3
--- /dev/null
+++ b/pplx_embed.h
@@ -0,0 +1,186 @@
+/*
+ * pplx_embed.h - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
+ *
+ * Architecture: Qwen3-family transformer with BIDIRECTIONAL (non-causal)
+ * attention. Based on antirez's qwen-asr kernels. Produces dense float32
+ * embedding vectors via mean-pooling + L2-normalization.
+ *
+ * Model spec:
+ * hidden_size = 1024
+ * num_hidden_layers = 28
+ * num_attention_heads = 16 (query heads)
+ * num_key_value_heads = 8 (GQA: 2 query heads share 1 KV head)
+ * head_dim = 128
+ * intermediate_size = 3072
+ * vocab_size = 151936
+ * rms_norm_eps = 1e-6
+ * rope_theta = 1000000.0 (1e6)
+ * weights dtype = F32 (all weights float32)
+ * tie_word_embeddings = true
+ * use_bidirectional_attention = true
+ */
+
+#ifndef PPLX_EMBED_H
+#define PPLX_EMBED_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* ========================================================================
+ * Constants
+ * ======================================================================== */
+
+#define PPLX_HIDDEN_SIZE 1024
+#define PPLX_NUM_LAYERS 28
+#define PPLX_NUM_HEADS 16
+#define PPLX_NUM_KV_HEADS 8
+#define PPLX_HEAD_DIM 128
+#define PPLX_Q_DIM (PPLX_NUM_HEADS * PPLX_HEAD_DIM) /* 2048 */
+#define PPLX_KV_DIM (PPLX_NUM_KV_HEADS * PPLX_HEAD_DIM) /* 1024 */
+#define PPLX_INTERMEDIATE_SIZE 3072
+#define PPLX_VOCAB_SIZE 151936
+
+/* ========================================================================
+ * Model Configuration
+ * ======================================================================== */
+
+typedef struct {
+ int hidden_size; /* 1024 */
+ int n_layers; /* 28 */
+ int n_heads; /* 16 */
+ int n_kv_heads; /* 8 */
+ int head_dim; /* 128 */
+ int q_dim; /* n_heads * head_dim = 2048 */
+ int kv_dim; /* n_kv_heads * head_dim = 1024 */
+ int intermediate_size; /* 3072 */
+ int vocab_size; /* 151936 */
+ float rms_norm_eps; /* 1e-6 */
+ float rope_theta; /* 1e6 */
+} pplx_config_t;
+
+/* ========================================================================
+ * Per-Layer Weights (all F32, direct mmap pointers - read-only)
+ *
+ * Tensor shapes (row-major):
+ * wq [q_dim, hidden] = [2048, 1024]
+ * wk [kv_dim, hidden] = [1024, 1024]
+ * wv [kv_dim, hidden] = [1024, 1024]
+ * wo [hidden, q_dim] = [1024, 2048]
+ * q_norm [head_dim] = [128]
+ * k_norm [head_dim] = [128]
+ * input_norm [hidden] = [1024]
+ * post_attn_norm [hidden] = [1024]
+ * gate_proj [intermediate, hidden] = [3072, 1024]
+ * up_proj [intermediate, hidden] = [3072, 1024]
+ * down_proj [hidden, intermediate] = [1024, 3072]
+ * ======================================================================== */
+
+typedef struct {
+ /* Self-attention projections (no bias) */
+ const float *wq; /* [q_dim, hidden] */
+ const float *wk; /* [kv_dim, hidden] */
+ const float *wv; /* [kv_dim, hidden] */
+ const float *wo; /* [hidden, q_dim] */
+
+ /* Per-head Q/K RMSNorm weights (Qwen3 feature) */
+ const float *q_norm; /* [head_dim] */
+ const float *k_norm; /* [head_dim] */
+
+ /* Layer norms (no bias - RMSNorm only) */
+ const float *input_norm; /* [hidden] pre-attention */
+ const float *post_attn_norm;/* [hidden] pre-MLP */
+
+ /* SwiGLU MLP (no bias) */
+ const float *gate_proj; /* [intermediate, hidden] */
+ const float *up_proj; /* [intermediate, hidden] */
+ const float *down_proj; /* [hidden, intermediate] */
+} pplx_layer_t;
+
+/* ========================================================================
+ * Full Model Weights
+ * ======================================================================== */
+
+typedef struct {
+ /* Token embedding table (tied with output head, but we don't need lm_head) */
+ const float *embed_tokens; /* [vocab_size, hidden] */
+
+ /* Transformer layers */
+ pplx_layer_t layers[PPLX_NUM_LAYERS];
+
+ /* Final RMSNorm */
+ const float *norm; /* [hidden] */
+} pplx_weights_t;
+
+/* ========================================================================
+ * Context (model + working buffers)
+ * ======================================================================== */
+
+typedef struct {
+ pplx_config_t config;
+ pplx_weights_t weights;
+
+ /* Open safetensors handle (keeps mmap alive) */
+ void *safetensors; /* multi_safetensors_t* */
+
+ /* Working buffers, grown lazily to buf_seq_cap */
+ int buf_seq_cap;
+ float *x; /* [seq, hidden] main activations */
+ float *x_norm; /* [seq, hidden] after RMSNorm */
+ float *q; /* [seq, q_dim] query vectors */
+ float *k; /* [seq, kv_dim] key vectors */
+ float *v; /* [seq, kv_dim] value vectors */
+ float *attn_out; /* [seq, q_dim] attention output */
+ float *proj_out; /* [seq, hidden] after o_proj */
+ float *ffn_gate; /* [seq, intermediate] gate branch */
+ float *ffn_up; /* [seq, intermediate] up branch */
+ float *ffn_out; /* [seq, hidden] after down_proj */
+
+ /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily */
+ float *rope_cos;
+ float *rope_sin;
+ int rope_cache_cap; /* number of cached positions */
+} pplx_ctx_t;
+
+/* ========================================================================
+ * API
+ * ======================================================================== */
+
+/*
+ * Load the model from a directory containing:
+ * model.safetensors - weight tensors (F32)
+ * vocab.json - BPE vocabulary
+ * merges.txt - BPE merge rules
+ *
+ * Returns NULL on error.
+ */
+pplx_ctx_t *pplx_load(const char *model_dir);
+
+/* Free all resources */
+void pplx_free(pplx_ctx_t *ctx);
+
+/*
+ * Compute the embedding for a sequence of token IDs.
+ *
+ * Performs:
+ * 1. Token embedding lookup
+ * 2. 28 × bidirectional transformer layers (GQA, RoPE, SwiGLU)
+ * 3. Final RMSNorm
+ * 4. Mean pooling across all token positions
+ * 5. L2 normalization
+ *
+ * Returns a malloc'd float[hidden_size] (caller must free).
+ * Returns NULL on allocation error or empty input.
+ */
+float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
+
+/*
+ * Cosine similarity between two L2-normalized embedding vectors.
+ * (This is just the dot product since both are unit vectors.)
+ */
+float pplx_cosine_similarity(const float *a, const float *b, int dim);
+
+/* Verbose level: 0=quiet, 1=info, 2=debug */
+extern int pplx_verbose;
+extern int qwen_verbose; /* same flag, used internally by kernel/tokenizer */
+
+#endif /* PPLX_EMBED_H */
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
new file mode 100644
index 0000000..d6637cd
--- /dev/null
+++ b/pplx_embed_mlx.c
@@ -0,0 +1,465 @@
+/*
+ * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference
+ *
+ * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference.
+ * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm.
+ *
+ * Key operations used:
+ * mlx_take - embedding lookup
+ * mlx_matmul - linear projections (no bias)
+ * mlx_fast_rms_norm - RMSNorm (fused Metal kernel)
+ * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel)
+ * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel)
+ * sigmoid * gate, multiply - SwiGLU activation
+ */
+
+#include "pplx_embed_mlx.h"
+#include "pplx_embed.h" /* config constants */
+#include "qwen_asr_safetensors.h" /* safetensors reader */
+
+#include <mlx/c/mlx.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+/* ========================================================================
+ * Per-layer weight arrays (MLX)
+ * ======================================================================== */
+
+typedef struct {
+ mlx_array wq; /* [q_dim, hidden] */
+ mlx_array wk; /* [kv_dim, hidden] */
+ mlx_array wv; /* [kv_dim, hidden] */
+ mlx_array wo; /* [hidden, q_dim] */
+ mlx_array q_norm; /* [head_dim] */
+ mlx_array k_norm; /* [head_dim] */
+ mlx_array input_norm; /* [hidden] */
+ mlx_array post_attn_norm; /* [hidden] */
+ mlx_array gate_proj; /* [intermediate, hidden] */
+ mlx_array up_proj; /* [intermediate, hidden] */
+ mlx_array down_proj; /* [hidden, intermediate] */
+} mlx_layer_t;
+
+struct pplx_mlx_ctx {
+ mlx_array embed_tokens; /* [vocab, hidden] */
+ mlx_layer_t layers[PPLX_NUM_LAYERS];
+ mlx_array norm; /* [hidden] */
+ mlx_stream stream;
+
+ /* Safetensors kept alive for mmap */
+ multi_safetensors_t *ms;
+};
+
+/* ========================================================================
+ * Helpers
+ * ======================================================================== */
+
+/* Default GPU stream */
+static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; }
+
+/*
+ * Load an F32 tensor from safetensors into an mlx_array.
+ * The data is copied into MLX (which will move it to GPU as needed).
+ */
+static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
+ const int *shape, int ndim)
+{
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "mlx: tensor not found: %s\n", name);
+ return (mlx_array){0};
+ }
+ if (t->dtype != DTYPE_F32) {
+ fprintf(stderr, "mlx: expected F32 for %s\n", name);
+ return (mlx_array){0};
+ }
+ const void *data = safetensors_data(sf, t);
+ return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32);
+}
+
+/* Quick check: is arr valid? */
+static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+
+/* ========================================================================
+ * Load
+ * ======================================================================== */
+
+pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
+{
+ pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
+ if (!ctx) return NULL;
+
+ ctx->stream = mlx_default_gpu_stream_new();
+
+ /* Open safetensors */
+ ctx->ms = multi_safetensors_open(model_dir);
+ if (!ctx->ms) {
+ fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
+ free(ctx);
+ return NULL;
+ }
+
+ multi_safetensors_t *ms = ctx->ms;
+
+ /* Embedding table: [vocab_size, hidden] */
+ int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE};
+ ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens)) goto fail;
+
+ /* Per-layer weights */
+ char name[256];
+ for (int i = 0; i < PPLX_NUM_LAYERS; i++) {
+ mlx_layer_t *l = &ctx->layers[i];
+ int hidden = PPLX_HIDDEN_SIZE;
+ int q_dim = PPLX_Q_DIM;
+ int kv_dim = PPLX_KV_DIM;
+ int inter = PPLX_INTERMEDIATE_SIZE;
+ int hdim = PPLX_HEAD_DIM;
+
+#define LOAD(field, fmt, ...) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ int nd = sizeof(sh) / sizeof(sh[0]); \
+ l->field = load_tensor(ms, name, sh, nd); \
+ if (!arr_ok(l->field)) goto fail; \
+} while (0)
+
+ LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden);
+ LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden);
+ LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden);
+ LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim);
+ LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim);
+ LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim);
+ LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden);
+ LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden);
+ LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden);
+ LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden);
+ LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter);
+
+#undef LOAD
+
+ if (pplx_verbose >= 2)
+ fprintf(stderr, " mlx layer %d loaded\n", i);
+ }
+
+ /* Final norm */
+ int norm_shape[] = {PPLX_HIDDEN_SIZE};
+ ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1);
+ if (!arr_ok(ctx->norm)) goto fail;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "mlx: all weights loaded\n");
+
+ return ctx;
+
+fail:
+ pplx_mlx_free(ctx);
+ return NULL;
+}
+
+/* ========================================================================
+ * Free
+ * ======================================================================== */
+
+static void free_layer(mlx_layer_t *l)
+{
+ mlx_array_free(l->wq);
+ mlx_array_free(l->wk);
+ mlx_array_free(l->wv);
+ mlx_array_free(l->wo);
+ mlx_array_free(l->q_norm);
+ mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm);
+ mlx_array_free(l->post_attn_norm);
+ mlx_array_free(l->gate_proj);
+ mlx_array_free(l->up_proj);
+ mlx_array_free(l->down_proj);
+}
+
+void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
+{
+ if (!ctx) return;
+ mlx_array_free(ctx->embed_tokens);
+ for (int i = 0; i < PPLX_NUM_LAYERS; i++)
+ free_layer(&ctx->layers[i]);
+ mlx_array_free(ctx->norm);
+ mlx_stream_free(ctx->stream);
+ if (ctx->ms) multi_safetensors_close(ctx->ms);
+ free(ctx);
+}
+
+/* ========================================================================
+ * Forward pass (all on GPU stream, lazy-evaluated)
+ *
+ * Notation: S = gpu_stream(ctx)
+ *
+ * Linear without bias: y = x @ W^T
+ * Since mlx_matmul(x, W^T) requires transposing W, and our weights
+ * are stored as [out, in], we do: y = x @ W^T via
+ * Wt = transpose(W) [in, out]
+ * y = matmul(x, Wt) [seq, out]
+ *
+ * Alternatively: y = matmul(x, transpose(W))
+ * All MLX ops are lazy (graph-built) so transpose is free.
+ * ======================================================================== */
+
+/* Convenience: transpose a 2D matrix */
+static mlx_array t2d(mlx_array a, mlx_stream s)
+{
+ mlx_array res = mlx_array_new();
+ mlx_transpose(&res, a, s);
+ return res;
+}
+
+/* y = x @ W^T (linear, no bias) */
+static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
+{
+ mlx_array Wt = t2d(W, s);
+ mlx_array res = mlx_array_new();
+ mlx_matmul(&res, x, Wt, s);
+ mlx_array_free(Wt);
+ return res;
+}
+
+float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ mlx_stream S = gpu_stream(ctx);
+ int hidden = PPLX_HIDDEN_SIZE;
+ int n_heads = PPLX_NUM_HEADS;
+ int n_kv_heads = PPLX_NUM_KV_HEADS;
+ int head_dim = PPLX_HEAD_DIM;
+ int q_dim = PPLX_Q_DIM;
+ int seq = n_tokens;
+ float scale = 1.0f / sqrtf((float)head_dim);
+
+ (void)n_heads; (void)n_kv_heads; /* used only in shape literals */
+
+ /* ------------------------------------------------------------------
+ * 1. Token embedding lookup: x = embed_tokens[token_ids]
+ * ------------------------------------------------------------------ */
+ int ids_shape[] = {seq};
+ mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32);
+ mlx_array x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S);
+ mlx_array_free(ids);
+ /* x: [seq, hidden] */
+
+ /* ------------------------------------------------------------------
+ * 2. Transformer layers (28x)
+ * ------------------------------------------------------------------ */
+ for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) {
+ mlx_layer_t *l = &ctx->layers[layer];
+
+ /* ---- 2a. Input RMSNorm ---- */
+ mlx_array xn = mlx_array_new();
+ mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S);
+
+ /* ---- 2b. QKV projections (no bias) ----
+ * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim]
+ */
+ mlx_array q_flat = linear(xn, l->wq, S);
+ mlx_array k_flat = linear(xn, l->wk, S);
+ mlx_array v_flat = linear(xn, l->wv, S);
+ mlx_array_free(xn);
+
+ /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */
+ int q_shape[] = {seq, n_heads, head_dim};
+ int k_shape[] = {seq, n_kv_heads, head_dim};
+
+ mlx_array q = mlx_array_new();
+ mlx_array k = mlx_array_new();
+ mlx_array v = mlx_array_new();
+ mlx_reshape(&q, q_flat, q_shape, 3, S);
+ mlx_reshape(&k, k_flat, k_shape, 3, S);
+ mlx_reshape(&v, v_flat, k_shape, 3, S);
+ mlx_array_free(q_flat);
+ mlx_array_free(k_flat);
+ mlx_array_free(v_flat);
+
+ /* ---- 2d. Per-head Q/K RMSNorm ----
+ * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim)
+ */
+ mlx_array qn = mlx_array_new();
+ mlx_array kn = mlx_array_new();
+ mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S);
+ mlx_array_free(q);
+ mlx_array_free(k);
+
+ /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ----
+ * MLX rope expects (..., T, D) where T is second-to-last.
+ * [seq, n_heads, head_dim]
+ * - expand_dims(0) [1, seq, n_heads, head_dim]
+ * - transpose [0,2,1,3] [1, n_heads, seq, head_dim]
+ */
+ int perm[] = {0, 2, 1, 3};
+
+ mlx_array qe = mlx_array_new();
+ mlx_array ke = mlx_array_new();
+ mlx_array ve = mlx_array_new();
+ mlx_expand_dims(&qe, qn, 0, S);
+ mlx_expand_dims(&ke, kn, 0, S);
+ mlx_expand_dims(&ve, v, 0, S);
+ mlx_array_free(qn);
+ mlx_array_free(kn);
+
+ mlx_array qt = mlx_array_new();
+ mlx_array kt = mlx_array_new();
+ mlx_array vt = mlx_array_new();
+ mlx_transpose_axes(&qt, qe, perm, 4, S);
+ mlx_transpose_axes(&kt, ke, perm, 4, S);
+ mlx_transpose_axes(&vt, ve, perm, 4, S);
+ mlx_array_free(qe);
+ mlx_array_free(ke);
+ mlx_array_free(ve);
+ mlx_array_free(v);
+
+ /* ---- 2f. RoPE (NeoX-style: traditional=false) ----
+ * qt: [1, n_heads, seq, head_dim]
+ * kt: [1, n_kv_heads, seq, head_dim]
+ * MLX rope: T = second-to-last dim = seq
+ */
+ mlx_optional_float base = {.value = 1000000.0f, .has_value = true};
+ mlx_array null_freqs = (mlx_array){0};
+
+ mlx_array qr = mlx_array_new();
+ mlx_array kr = mlx_array_new();
+ mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false,
+ base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
+ mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false,
+ base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
+ mlx_array_free(qt);
+ mlx_array_free(kt);
+
+ /* ---- 2g. Bidirectional GQA Attention ----
+ * qr: [1, n_heads, seq, head_dim]
+ * kr: [1, n_kv_heads, seq, head_dim]
+ * vt: [1, n_kv_heads, seq, head_dim]
+ * mask_mode="" > no causal mask > full bidirectional
+ * GQA: SDPA broadcasts KV heads to match Q heads automatically.
+ */
+ mlx_array attn = mlx_array_new();
+ mlx_array null_mask = (mlx_array){0};
+ mlx_array null_sinks = (mlx_array){0};
+ mlx_fast_scaled_dot_product_attention(
+ &attn, qr, kr, vt, scale,
+ /*mask_mode=*/"",
+ null_mask, null_sinks, S);
+ mlx_array_free(qr);
+ mlx_array_free(kr);
+ mlx_array_free(vt);
+ /* attn: [1, n_heads, seq, head_dim] */
+
+ /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */
+ mlx_array attn_t = mlx_array_new();
+ mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */
+ mlx_array_free(attn);
+
+ mlx_array attn_sq = mlx_array_new();
+ mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */
+ mlx_array_free(attn_t);
+
+ int attn_flat_shape[] = {seq, q_dim};
+ mlx_array attn_flat = mlx_array_new();
+ mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S);
+ mlx_array_free(attn_sq);
+
+ /* ---- 2i. Output projection + residual ---- */
+ mlx_array proj = linear(attn_flat, l->wo, S);
+ mlx_array_free(attn_flat);
+
+ mlx_array x2 = mlx_array_new();
+ mlx_add(&x2, x, proj, S);
+ mlx_array_free(x);
+ mlx_array_free(proj);
+ x = x2;
+
+ /* ---- 2j. Post-attention RMSNorm ---- */
+ mlx_array xn2 = mlx_array_new();
+ mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S);
+
+ /* ---- 2k. SwiGLU MLP ----
+ * gate = x_norm @ gate_proj^T [seq, inter]
+ * up = x_norm @ up_proj^T [seq, inter]
+ * mid = silu(gate) * up = (gate * sigmoid(gate)) * up
+ * out = mid @ down_proj^T [seq, hidden]
+ */
+ mlx_array gate = linear(xn2, l->gate_proj, S);
+ mlx_array up = linear(xn2, l->up_proj, S);
+ mlx_array_free(xn2);
+
+ /* SiLU(gate) = gate * sigmoid(gate) */
+ mlx_array gate_sig = mlx_array_new();
+ mlx_sigmoid(&gate_sig, gate, S);
+ mlx_array silu_gate = mlx_array_new();
+ mlx_multiply(&silu_gate, gate, gate_sig, S);
+ mlx_array_free(gate);
+ mlx_array_free(gate_sig);
+
+ /* mid = silu_gate * up */
+ mlx_array mid = mlx_array_new();
+ mlx_multiply(&mid, silu_gate, up, S);
+ mlx_array_free(silu_gate);
+ mlx_array_free(up);
+
+ /* down projection */
+ mlx_array ffn = linear(mid, l->down_proj, S);
+ mlx_array_free(mid);
+
+ /* Residual */
+ mlx_array x3 = mlx_array_new();
+ mlx_add(&x3, x, ffn, S);
+ mlx_array_free(x);
+ mlx_array_free(ffn);
+ x = x3;
+ }
+
+ /* ------------------------------------------------------------------
+ * 3. Final RMSNorm
+ * ------------------------------------------------------------------ */
+ mlx_array x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S);
+ mlx_array_free(x);
+
+ /* ------------------------------------------------------------------
+ * 4. Mean pooling: mean(x, axis=0) [hidden]
+ * ------------------------------------------------------------------ */
+ mlx_array emb = mlx_array_new();
+ mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S);
+ mlx_array_free(x_normed);
+
+ /* ------------------------------------------------------------------
+ * 5. L2 normalization: emb / ||emb||_2
+ * ------------------------------------------------------------------ */
+ int norm_axes[] = {0};
+ mlx_array norm_val = mlx_array_new();
+ mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S);
+
+ mlx_array emb_normed = mlx_array_new();
+ mlx_divide(&emb_normed, emb, norm_val, S);
+ mlx_array_free(emb);
+ mlx_array_free(norm_val);
+
+ /* ------------------------------------------------------------------
+ * 6. Evaluate and copy result to CPU
+ * ------------------------------------------------------------------ */
+ mlx_array_eval(emb_normed);
+
+ const float *data = mlx_array_data_float32(emb_normed);
+ if (!data) {
+ fprintf(stderr, "mlx: failed to get embedding data\n");
+ mlx_array_free(emb_normed);
+ return NULL;
+ }
+
+ float *out = (float *)malloc(hidden * sizeof(float));
+ if (out) memcpy(out, data, hidden * sizeof(float));
+
+ mlx_array_free(emb_normed);
+ return out;
+}
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
new file mode 100644
index 0000000..5327fd0
--- /dev/null
+++ b/pplx_embed_mlx.h
@@ -0,0 +1,35 @@
+/*
+ * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference
+ *
+ * Uses Apple's mlx-c to run the transformer on Metal GPU.
+ * All operations run on the GPU default stream; the result is copied
+ * back to CPU as a float* embedding.
+ */
+
+#ifndef PPLX_EMBED_MLX_H
+#define PPLX_EMBED_MLX_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* Opaque MLX model context */
+typedef struct pplx_mlx_ctx pplx_mlx_ctx_t;
+
+/*
+ * Load model into MLX arrays from safetensors in model_dir.
+ * Returns NULL on error.
+ */
+pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir);
+
+/*
+ * Free all MLX resources.
+ */
+void pplx_mlx_free(pplx_mlx_ctx_t *ctx);
+
+/*
+ * Compute embedding for token_ids[0..n_tokens-1].
+ * Returns malloc'd float[1024] (caller frees). NULL on error.
+ */
+float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
+
+#endif /* PPLX_EMBED_MLX_H */
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
new file mode 100644
index 0000000..c0d9d70
--- /dev/null
+++ b/qwen_asr_kernels.c
@@ -0,0 +1,1247 @@
+/*
+ * qwen_asr_kernels.c - Math kernels for Qwen3-ASR inference
+ * Adapted from voxtral-realtime project.
+ */
+
+#include "qwen_asr_kernels.h"
+#include "qwen_asr_kernels_impl.h"
+#include <math.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <pthread.h>
+#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
+#include <immintrin.h>
+#endif
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#else
+#include <unistd.h>
+#endif
+
+#ifdef USE_BLAS
+#ifdef __APPLE__
+#include <Accelerate/Accelerate.h>
+#else
+#include <cblas.h>
+#endif
+#endif
+
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
+/* ========================================================================
+ * Thread Pool
+ * ======================================================================== */
+
+#define QWEN_MAX_THREADS 16
+
+typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg);
+
+static struct {
+ pthread_t threads[QWEN_MAX_THREADS - 1];
+ int tids[QWEN_MAX_THREADS - 1];
+ int n_threads;
+ int shutdown;
+
+ parallel_fn_t fn;
+ void *arg;
+ int generation;
+
+ pthread_mutex_t mutex;
+ pthread_cond_t cond_work;
+ pthread_cond_t cond_done;
+ int n_done;
+} tp = {
+ .n_threads = 1,
+ .shutdown = 0,
+ .generation = 0,
+ .mutex = PTHREAD_MUTEX_INITIALIZER,
+ .cond_work = PTHREAD_COND_INITIALIZER,
+ .cond_done = PTHREAD_COND_INITIALIZER,
+};
+
+static void *worker_loop(void *arg) {
+ int tid = *(int *)arg;
+ int my_gen = 0;
+
+ for (;;) {
+ pthread_mutex_lock(&tp.mutex);
+ while (tp.generation == my_gen && !tp.shutdown)
+ pthread_cond_wait(&tp.cond_work, &tp.mutex);
+ if (tp.shutdown) {
+ pthread_mutex_unlock(&tp.mutex);
+ return NULL;
+ }
+ my_gen = tp.generation;
+ parallel_fn_t fn = tp.fn;
+ void *a = tp.arg;
+ int nt = tp.n_threads;
+ pthread_mutex_unlock(&tp.mutex);
+
+ fn(tid, nt, a);
+
+ pthread_mutex_lock(&tp.mutex);
+ if (++tp.n_done >= tp.n_threads - 1)
+ pthread_cond_signal(&tp.cond_done);
+ pthread_mutex_unlock(&tp.mutex);
+ }
+}
+
+void qwen_set_threads(int n) {
+ if (n < 1) n = 1;
+ if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS;
+
+ /* Shutdown existing workers */
+ if (tp.n_threads > 1) {
+ pthread_mutex_lock(&tp.mutex);
+ tp.shutdown = 1;
+ pthread_cond_broadcast(&tp.cond_work);
+ pthread_mutex_unlock(&tp.mutex);
+ for (int i = 0; i < tp.n_threads - 1; i++)
+ pthread_join(tp.threads[i], NULL);
+ tp.shutdown = 0;
+ tp.generation = 0;
+ }
+
+ tp.n_threads = n;
+ if (n <= 1) return;
+
+ for (int i = 0; i < n - 1; i++) {
+ tp.tids[i] = i + 1;
+ pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]);
+ }
+
+ if (qwen_verbose >= 2)
+ fprintf(stderr, "Thread pool: %d threads\n", n);
+}
+
+int qwen_get_num_cpus(void) {
+#ifdef __APPLE__
+ int n = 0;
+ size_t len = sizeof(n);
+ sysctlbyname("hw.ncpu", &n, &len, NULL, 0);
+ return n > 0 ? n : 1;
+#else
+ int n = (int)sysconf(_SC_NPROCESSORS_ONLN);
+ return n > 0 ? n : 1;
+#endif
+}
+
+/* Dispatch work to all threads; main thread is tid=0 */
+static void parallel_for(parallel_fn_t fn, void *arg) {
+ if (tp.n_threads <= 1) {
+ fn(0, 1, arg);
+ return;
+ }
+
+ pthread_mutex_lock(&tp.mutex);
+ tp.fn = fn;
+ tp.arg = arg;
+ tp.n_done = 0;
+ tp.generation++;
+ pthread_cond_broadcast(&tp.cond_work);
+ pthread_mutex_unlock(&tp.mutex);
+
+ fn(0, tp.n_threads, arg);
+
+ pthread_mutex_lock(&tp.mutex);
+ while (tp.n_done < tp.n_threads - 1)
+ pthread_cond_wait(&tp.cond_done, &tp.mutex);
+ pthread_mutex_unlock(&tp.mutex);
+}
+
+/* ========================================================================
+ * Basic Element-wise Operations
+ * ======================================================================== */
+
+void qwen_add_inplace(float *a, const float *b, int n) {
+ for (int i = 0; i < n; i++) a[i] += b[i];
+}
+
+void qwen_mul_inplace(float *a, const float *b, int n) {
+ for (int i = 0; i < n; i++) a[i] *= b[i];
+}
+
+void qwen_scale(float *x, float s, int n) {
+ for (int i = 0; i < n; i++) x[i] *= s;
+}
+
+void qwen_copy(float *dst, const float *src, int n) {
+ memcpy(dst, src, n * sizeof(float));
+}
+
+/* ========================================================================
+ * Matrix Operations
+ * ======================================================================== */
+
+void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) {
+#ifdef USE_BLAS
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
+ M, N, K, 1.0f, A, K, B, K, 0.0f, C, N);
+#else
+ for (int m = 0; m < M; m++) {
+ for (int n = 0; n < N; n++) {
+ float sum = 0.0f;
+ for (int k = 0; k < K; k++) {
+ sum += A[m * K + k] * B[n * K + k];
+ }
+ C[m * N + n] = sum;
+ }
+ }
+#endif
+}
+
+void qwen_linear(float *y, const float *x, const float *W, const float *b,
+ int seq_len, int in_dim, int out_dim) {
+#ifdef USE_BLAS
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
+ seq_len, out_dim, in_dim,
+ 1.0f, x, in_dim, W, in_dim,
+ 0.0f, y, out_dim);
+ if (b != NULL) {
+ for (int s = 0; s < seq_len; s++) {
+ for (int o = 0; o < out_dim; o++) {
+ y[s * out_dim + o] += b[o];
+ }
+ }
+ }
+#else
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + s * in_dim;
+ float *y_row = y + s * out_dim;
+ for (int o = 0; o < out_dim; o++) {
+ const float *w_row = W + o * in_dim;
+ float sum = (b != NULL) ? b[o] : 0.0f;
+ for (int i = 0; i < in_dim; i++) {
+ sum += x_row[i] * w_row[i];
+ }
+ y_row[o] = sum;
+ }
+ }
+#endif
+}
+
+void qwen_linear_nobias(float *y, const float *x, const float *W,
+ int seq_len, int in_dim, int out_dim) {
+ qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim);
+}
+
+/* Convert bf16 buffer to f32 buffer */
+static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
+ uint32_t *d = (uint32_t *)(void *)dst;
+ for (size_t i = 0; i < n; i++)
+ d[i] = ((uint32_t)src[i]) << 16;
+}
+
+/* Reusable scratch buffer for bf16->f32 conversion */
+static float *bf16_scratch = NULL;
+static size_t bf16_scratch_cap = 0;
+
+static float *bf16_get_scratch(size_t n) {
+ if (n > bf16_scratch_cap) {
+ free(bf16_scratch);
+ bf16_scratch = (float *)malloc(n * sizeof(float));
+ bf16_scratch_cap = bf16_scratch ? n : 0;
+ }
+ return bf16_scratch;
+}
+
+typedef struct {
+ const uint16_t *src;
+ size_t n;
+ float *dst_f32;
+} bf16_cache_entry_t;
+
+static bf16_cache_entry_t *bf16_cache = NULL;
+static int bf16_cache_len = 0;
+static int bf16_cache_cap = 0;
+static size_t bf16_cache_bytes = 0;
+static size_t bf16_cache_limit_bytes = 0;
+static int bf16_cache_limit_init = 0;
+
+static void bf16_cache_init_limit(void) {
+ if (bf16_cache_limit_init) return;
+ bf16_cache_limit_init = 1;
+
+ /* Default OFF. Override with QWEN_BF16_CACHE_MB=<n> to enable. */
+ unsigned long long mb = 0;
+ const char *env = getenv("QWEN_BF16_CACHE_MB");
+ if (env && env[0] != '\0') {
+ char *end = NULL;
+ unsigned long long v = strtoull(env, &end, 10);
+ if (end != env) mb = v;
+ }
+ bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL);
+
+ if (qwen_verbose >= 2) {
+ fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb);
+ }
+}
+
+static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) {
+ bf16_cache_init_limit();
+
+ for (int i = 0; i < bf16_cache_len; i++) {
+ if (bf16_cache[i].src == src && bf16_cache[i].n == n) {
+ return bf16_cache[i].dst_f32;
+ }
+ }
+
+ if (bf16_cache_limit_bytes == 0) return NULL;
+
+ size_t bytes = n * sizeof(float);
+ if (bytes > bf16_cache_limit_bytes) return NULL;
+ if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL;
+
+ float *dst = (float *)malloc(bytes);
+ if (!dst) return NULL;
+ bf16_to_f32_buf(dst, src, n);
+
+ if (bf16_cache_len == bf16_cache_cap) {
+ int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256;
+ bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc(
+ bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t));
+ if (!tmp) {
+ free(dst);
+ return NULL;
+ }
+ bf16_cache = tmp;
+ bf16_cache_cap = new_cap;
+ }
+
+ bf16_cache[bf16_cache_len].src = src;
+ bf16_cache[bf16_cache_len].n = n;
+ bf16_cache[bf16_cache_len].dst_f32 = dst;
+ bf16_cache_len++;
+ bf16_cache_bytes += bytes;
+ return dst;
+}
+
+static const float *bf16_get_f32_view(const uint16_t *src, size_t n) {
+ const float *cached = bf16_get_cached_f32(src, n);
+ if (cached) return cached;
+
+ float *scratch = bf16_get_scratch(n);
+ if (!scratch) return NULL;
+ bf16_to_f32_buf(scratch, src, n);
+ return scratch;
+}
+
+/*
+ * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias
+ * Processes 2 output rows at a time to amortize x vector loads.
+ */
+static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim);
+}
+
+/* Threaded matvec: split output rows across threads */
+typedef struct {
+ float *y;
+ const float *x;
+ const uint16_t *W_bf16;
+ const float *bias;
+ int in_dim;
+ int out_dim;
+} matvec_task_t;
+
+static void matvec_worker(int tid, int n_threads, void *arg) {
+ matvec_task_t *t = (matvec_task_t *)arg;
+ int chunk = (t->out_dim + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->out_dim) end = t->out_dim;
+ if (start >= end) return;
+
+ bf16_matvec_fused(t->y + start, t->x,
+ t->W_bf16 + (size_t)start * t->in_dim,
+ t->bias ? t->bias + start : NULL,
+ t->in_dim, end - start);
+}
+
+static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ if (tp.n_threads <= 1) {
+ bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim);
+ return;
+ }
+ matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim };
+ parallel_for(matvec_worker, &task);
+}
+
+typedef struct {
+ float *q;
+ float *k;
+ float *v;
+ const float *x;
+ const uint16_t *Wq_bf16;
+ const uint16_t *Wk_bf16;
+ const uint16_t *Wv_bf16;
+ int in_dim;
+ int q_dim;
+ int kv_dim;
+ int total_dim;
+} qkv_matvec_task_t;
+
+static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
+ qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg;
+ int chunk = (t->total_dim + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->total_dim) end = t->total_dim;
+ if (start >= end) return;
+
+ int q_end = t->q_dim;
+ int k_end = q_end + t->kv_dim;
+ int v_end = k_end + t->kv_dim;
+
+ if (start < q_end) {
+ int s = start;
+ int e = end < q_end ? end : q_end;
+ if (s < e) {
+ bf16_matvec_fused(t->q + s, t->x,
+ t->Wq_bf16 + (size_t)s * t->in_dim,
+ NULL, t->in_dim, e - s);
+ }
+ }
+
+ if (end > q_end && start < k_end) {
+ int s = start > q_end ? start - q_end : 0;
+ int e_abs = end < k_end ? end : k_end;
+ int e = e_abs - q_end;
+ if (s < e) {
+ bf16_matvec_fused(t->k + s, t->x,
+ t->Wk_bf16 + (size_t)s * t->in_dim,
+ NULL, t->in_dim, e - s);
+ }
+ }
+
+ if (end > k_end && start < v_end) {
+ int s = start > k_end ? start - k_end : 0;
+ int e_abs = end < v_end ? end : v_end;
+ int e = e_abs - k_end;
+ if (s < e) {
+ bf16_matvec_fused(t->v + s, t->x,
+ t->Wv_bf16 + (size_t)s * t->in_dim,
+ NULL, t->in_dim, e - s);
+ }
+ }
+}
+
+void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
+ const uint16_t *Wq_bf16,
+ const uint16_t *Wk_bf16,
+ const uint16_t *Wv_bf16,
+ int in_dim, int q_dim, int kv_dim) {
+ if (tp.n_threads <= 1) {
+ bf16_matvec_fused(q, x, Wq_bf16, NULL, in_dim, q_dim);
+ bf16_matvec_fused(k, x, Wk_bf16, NULL, in_dim, kv_dim);
+ bf16_matvec_fused(v, x, Wv_bf16, NULL, in_dim, kv_dim);
+ return;
+ }
+
+ qkv_matvec_task_t task = {
+ .q = q,
+ .k = k,
+ .v = v,
+ .x = x,
+ .Wq_bf16 = Wq_bf16,
+ .Wk_bf16 = Wk_bf16,
+ .Wv_bf16 = Wv_bf16,
+ .in_dim = in_dim,
+ .q_dim = q_dim,
+ .kv_dim = kv_dim,
+ .total_dim = q_dim + 2 * kv_dim,
+ };
+ parallel_for(qkv_matvec_worker, &task);
+}
+
+void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
+ int seq_len, int in_dim, int out_dim) {
+ if (seq_len == 1) {
+ bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
+ return;
+ }
+ size_t n = (size_t)out_dim * in_dim;
+ const float *W_f32 = bf16_get_f32_view(W_bf16, n);
+ if (!W_f32) return;
+ qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim);
+}
+
+void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
+ const float *b, int seq_len, int in_dim, int out_dim) {
+ if (seq_len == 1) {
+ bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim);
+ return;
+ }
+ size_t n = (size_t)out_dim * in_dim;
+ const float *W_f32 = bf16_get_f32_view(W_bf16, n);
+ if (!W_f32) return;
+ qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim);
+}
+
+/* Find argmax over a range of output rows [start, end).
+ * Uses 2-row processing to amortize x vector loads (same as bf16_matvec_fused). */
+static void argmax_bf16_range(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out) {
+ qwen_argmax_bf16_range_impl(x, W_bf16, in_dim, start, end, best_out, best_val_out);
+}
+
+typedef struct {
+ const float *x;
+ const uint16_t *W_bf16;
+ int in_dim;
+ int out_dim;
+ int best_idx[QWEN_MAX_THREADS];
+ float best_val[QWEN_MAX_THREADS];
+} argmax_task_t;
+
+static void argmax_worker(int tid, int n_threads, void *arg) {
+ argmax_task_t *t = (argmax_task_t *)arg;
+ int chunk = (t->out_dim + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->out_dim) end = t->out_dim;
+ if (start >= end) {
+ t->best_val[tid] = -1e30f;
+ t->best_idx[tid] = 0;
+ return;
+ }
+ argmax_bf16_range(t->x, t->W_bf16, t->in_dim, start, end,
+ &t->best_idx[tid], &t->best_val[tid]);
+}
+
+int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16,
+ int in_dim, int out_dim) {
+ if (tp.n_threads <= 1) {
+ int best;
+ float best_val;
+ argmax_bf16_range(x, W_bf16, in_dim, 0, out_dim, &best, &best_val);
+ return best;
+ }
+
+ argmax_task_t task;
+ task.x = x;
+ task.W_bf16 = W_bf16;
+ task.in_dim = in_dim;
+ task.out_dim = out_dim;
+ parallel_for(argmax_worker, &task);
+
+ int best = task.best_idx[0];
+ float best_val = task.best_val[0];
+ for (int i = 1; i < tp.n_threads; i++) {
+ if (task.best_val[i] > best_val) {
+ best_val = task.best_val[i];
+ best = task.best_idx[i];
+ }
+ }
+ return best;
+}
+
+void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
+ int M, int K, int N) {
+ if (M == 1) {
+ bf16_matvec_threaded(C, A, B_bf16, NULL, K, N);
+ } else {
+ size_t n = (size_t)N * K;
+ const float *B_f32 = bf16_get_f32_view(B_bf16, n);
+ if (!B_f32) return;
+ qwen_matmul_t(C, A, B_f32, M, K, N);
+ }
+}
+
+/* ========================================================================
+ * 2D Convolution (im2col + BLAS sgemm)
+ * ======================================================================== */
+
+/*
+ * im2col: Unroll input patches into a column matrix for GEMM-based conv2d.
+ * Input: [C_in, H_in, W_in]
+ * Output columns: [C_in * kH * kW, H_out * W_out]
+ */
+static void im2col(const float *in, float *cols,
+ int c_in, int h_in, int w_in,
+ int kh, int kw, int stride, int padding,
+ int h_out, int w_out) {
+ int col_len = h_out * w_out;
+ for (int ic = 0; ic < c_in; ic++) {
+ for (int ki = 0; ki < kh; ki++) {
+ for (int kj = 0; kj < kw; kj++) {
+ int col_row = (ic * kh + ki) * kw + kj;
+ float *col_ptr = cols + (size_t)col_row * col_len;
+ for (int oh = 0; oh < h_out; oh++) {
+ int ih = oh * stride - padding + ki;
+ for (int ow = 0; ow < w_out; ow++) {
+ int iw = ow * stride - padding + kj;
+ if (ih >= 0 && ih < h_in && iw >= 0 && iw < w_in) {
+ col_ptr[oh * w_out + ow] = in[ic * h_in * w_in + ih * w_in + iw];
+ } else {
+ col_ptr[oh * w_out + ow] = 0.0f;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias,
+ int c_in, int c_out, int h_in, int w_in,
+ int kh, int kw, int stride, int padding) {
+ int h_out = (h_in + 2 * padding - kh) / stride + 1;
+ int w_out = (w_in + 2 * padding - kw) / stride + 1;
+ int patch_size = c_in * kh * kw;
+ int spatial_out = h_out * w_out;
+
+ /* im2col: input -> column matrix [patch_size, spatial_out] */
+ float *cols = (float *)malloc((size_t)patch_size * spatial_out * sizeof(float));
+ im2col(in, cols, c_in, h_in, w_in, kh, kw, stride, padding, h_out, w_out);
+
+ /* GEMM: weight[c_out, patch_size] @ cols[patch_size, spatial_out] = out[c_out, spatial_out] */
+#ifdef USE_BLAS
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
+ c_out, spatial_out, patch_size,
+ 1.0f, weight, patch_size, cols, spatial_out,
+ 0.0f, out, spatial_out);
+#else
+ for (int oc = 0; oc < c_out; oc++) {
+ for (int s = 0; s < spatial_out; s++) {
+ float sum = 0.0f;
+ for (int p = 0; p < patch_size; p++) {
+ sum += weight[oc * patch_size + p] * cols[p * spatial_out + s];
+ }
+ out[oc * spatial_out + s] = sum;
+ }
+ }
+#endif
+
+ free(cols);
+
+ /* Add bias */
+ if (bias) {
+ for (int oc = 0; oc < c_out; oc++) {
+ float b = bias[oc];
+ float *row = out + oc * spatial_out;
+ for (int s = 0; s < spatial_out; s++) {
+ row[s] += b;
+ }
+ }
+ }
+}
+
+/* ========================================================================
+ * Normalization
+ * ======================================================================== */
+
+void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias,
+ int seq_len, int hidden, float eps) {
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + s * hidden;
+ float *out_row = out + s * hidden;
+
+ /* Compute mean */
+#if defined(__AVX512F__)
+ __m512 sumv = _mm512_setzero_ps();
+ int i = 0;
+ for (; i + 16 <= hidden; i += 16) {
+ sumv = _mm512_add_ps(sumv, _mm512_loadu_ps(x_row + i));
+ }
+ float mean = _mm512_reduce_add_ps(sumv);
+ for (; i < hidden; i++) mean += x_row[i];
+#elif defined(__AVX2__)
+ __m256 sumv = _mm256_setzero_ps();
+ int i = 0;
+ for (; i + 8 <= hidden; i += 8) {
+ sumv = _mm256_add_ps(sumv, _mm256_loadu_ps(x_row + i));
+ }
+ __m128 sum128 = _mm_add_ps(_mm256_castps256_ps128(sumv), _mm256_extractf128_ps(sumv, 1));
+ sum128 = _mm_hadd_ps(sum128, sum128);
+ sum128 = _mm_hadd_ps(sum128, sum128);
+ float mean = _mm_cvtss_f32(sum128);
+ for (; i < hidden; i++) mean += x_row[i];
+#else
+ float mean = 0.0f;
+ for (int i = 0; i < hidden; i++) mean += x_row[i];
+#endif
+ mean /= hidden;
+
+ /* Compute variance */
+#if defined(__AVX512F__) && defined(__FMA__)
+ __m512 meanv = _mm512_set1_ps(mean);
+ __m512 accv = _mm512_setzero_ps();
+ int j = 0;
+ for (; j + 16 <= hidden; j += 16) {
+ __m512 v = _mm512_sub_ps(_mm512_loadu_ps(x_row + j), meanv);
+ accv = _mm512_fmadd_ps(v, v, accv);
+ }
+ float var = _mm512_reduce_add_ps(accv);
+ for (; j < hidden; j++) {
+ float d = x_row[j] - mean;
+ var += d * d;
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ __m256 meanv = _mm256_set1_ps(mean);
+ __m256 accv = _mm256_setzero_ps();
+ int j = 0;
+ for (; j + 8 <= hidden; j += 8) {
+ __m256 v = _mm256_sub_ps(_mm256_loadu_ps(x_row + j), meanv);
+ accv = _mm256_fmadd_ps(v, v, accv);
+ }
+ __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ float var = _mm_cvtss_f32(acc128);
+ for (; j < hidden; j++) {
+ float d = x_row[j] - mean;
+ var += d * d;
+ }
+#else
+ float var = 0.0f;
+ for (int i = 0; i < hidden; i++) {
+ float d = x_row[i] - mean;
+ var += d * d;
+ }
+#endif
+ var /= hidden;
+
+ float inv_std = 1.0f / sqrtf(var + eps);
+#if defined(__AVX512F__) && defined(__FMA__)
+ __m512 meanv2 = _mm512_set1_ps(mean);
+ __m512 invv = _mm512_set1_ps(inv_std);
+ int k = 0;
+ for (; k + 16 <= hidden; k += 16) {
+ __m512 vx = _mm512_sub_ps(_mm512_loadu_ps(x_row + k), meanv2);
+ __m512 vw = _mm512_loadu_ps(weight + k);
+ __m512 vb = _mm512_loadu_ps(bias + k);
+ __m512 v = _mm512_mul_ps(_mm512_mul_ps(vx, invv), vw);
+ v = _mm512_add_ps(v, vb);
+ _mm512_storeu_ps(out_row + k, v);
+ }
+ for (; k < hidden; k++) {
+ out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k];
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ __m256 meanv2 = _mm256_set1_ps(mean);
+ __m256 invv = _mm256_set1_ps(inv_std);
+ int k = 0;
+ for (; k + 8 <= hidden; k += 8) {
+ __m256 vx = _mm256_sub_ps(_mm256_loadu_ps(x_row + k), meanv2);
+ __m256 vw = _mm256_loadu_ps(weight + k);
+ __m256 vb = _mm256_loadu_ps(bias + k);
+ __m256 v = _mm256_mul_ps(_mm256_mul_ps(vx, invv), vw);
+ v = _mm256_add_ps(v, vb);
+ _mm256_storeu_ps(out_row + k, v);
+ }
+ for (; k < hidden; k++) {
+ out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k];
+ }
+#else
+ for (int i = 0; i < hidden; i++) {
+ out_row[i] = (x_row[i] - mean) * inv_std * weight[i] + bias[i];
+ }
+#endif
+ }
+}
+
+void qwen_rms_norm(float *out, const float *x, const float *weight,
+ int seq_len, int hidden, float eps) {
+ for (int s = 0; s < seq_len; s++) {
+ const float *x_row = x + s * hidden;
+ float *out_row = out + s * hidden;
+
+#if defined(__AVX512F__) && defined(__FMA__)
+ __m512 accv = _mm512_setzero_ps();
+ int i = 0;
+ for (; i + 16 <= hidden; i += 16) {
+ __m512 v = _mm512_loadu_ps(x_row + i);
+ accv = _mm512_fmadd_ps(v, v, accv);
+ }
+ float sum_sq = _mm512_reduce_add_ps(accv);
+ for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i];
+#elif defined(__AVX2__) && defined(__FMA__)
+ __m256 accv = _mm256_setzero_ps();
+ int i = 0;
+ for (; i + 8 <= hidden; i += 8) {
+ __m256 v = _mm256_loadu_ps(x_row + i);
+ accv = _mm256_fmadd_ps(v, v, accv);
+ }
+ __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ float sum_sq = _mm_cvtss_f32(acc128);
+ for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i];
+#else
+ float sum_sq = 0.0f;
+ for (int i = 0; i < hidden; i++) {
+ sum_sq += x_row[i] * x_row[i];
+ }
+#endif
+ float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps);
+
+#if defined(__AVX512F__)
+ __m512 scale = _mm512_set1_ps(rms_inv);
+ int j = 0;
+ for (; j + 16 <= hidden; j += 16) {
+ __m512 vx = _mm512_loadu_ps(x_row + j);
+ __m512 vw = _mm512_loadu_ps(weight + j);
+ _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale));
+ }
+ for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j];
+#elif defined(__AVX2__)
+ __m256 scale = _mm256_set1_ps(rms_inv);
+ int j = 0;
+ for (; j + 8 <= hidden; j += 8) {
+ __m256 vx = _mm256_loadu_ps(x_row + j);
+ __m256 vw = _mm256_loadu_ps(weight + j);
+ _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale));
+ }
+ for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j];
+#else
+ for (int i = 0; i < hidden; i++) {
+ out_row[i] = x_row[i] * rms_inv * weight[i];
+ }
+#endif
+ }
+}
+
+void qwen_rms_norm_per_head(float *x, const float *weight,
+ int seq_len, int n_heads, int head_dim, float eps) {
+ /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
+ int hidden = n_heads * head_dim;
+ for (int s = 0; s < seq_len; s++) {
+ for (int h = 0; h < n_heads; h++) {
+ float *vec = x + s * hidden + h * head_dim;
+
+#if defined(__AVX512F__) && defined(__FMA__)
+ __m512 accv = _mm512_setzero_ps();
+ int d = 0;
+ for (; d + 16 <= head_dim; d += 16) {
+ __m512 v = _mm512_loadu_ps(vec + d);
+ accv = _mm512_fmadd_ps(v, v, accv);
+ }
+ float sum_sq = _mm512_reduce_add_ps(accv);
+ for (; d < head_dim; d++) sum_sq += vec[d] * vec[d];
+#elif defined(__AVX2__) && defined(__FMA__)
+ __m256 accv = _mm256_setzero_ps();
+ int d = 0;
+ for (; d + 8 <= head_dim; d += 8) {
+ __m256 v = _mm256_loadu_ps(vec + d);
+ accv = _mm256_fmadd_ps(v, v, accv);
+ }
+ __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ acc128 = _mm_hadd_ps(acc128, acc128);
+ float sum_sq = _mm_cvtss_f32(acc128);
+ for (; d < head_dim; d++) sum_sq += vec[d] * vec[d];
+#else
+ float sum_sq = 0.0f;
+ for (int d = 0; d < head_dim; d++) {
+ sum_sq += vec[d] * vec[d];
+ }
+#endif
+ float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps);
+
+#if defined(__AVX512F__)
+ __m512 scale = _mm512_set1_ps(rms_inv);
+ int j = 0;
+ for (; j + 16 <= head_dim; j += 16) {
+ __m512 v = _mm512_loadu_ps(vec + j);
+ __m512 w = _mm512_loadu_ps(weight + j);
+ _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale));
+ }
+ for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j];
+#elif defined(__AVX2__)
+ __m256 scale = _mm256_set1_ps(rms_inv);
+ int j = 0;
+ for (; j + 8 <= head_dim; j += 8) {
+ __m256 v = _mm256_loadu_ps(vec + j);
+ __m256 w = _mm256_loadu_ps(weight + j);
+ _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale));
+ }
+ for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j];
+#else
+ for (int d = 0; d < head_dim; d++) {
+ vec[d] = vec[d] * rms_inv * weight[d];
+ }
+#endif
+ }
+ }
+}
+
+/* ========================================================================
+ * Activation Functions
+ * ======================================================================== */
+
+void qwen_silu(float *x, int n) {
+ for (int i = 0; i < n; i++) {
+ float val = x[i];
+ x[i] = val / (1.0f + expf(-val));
+ }
+}
+
+void qwen_gelu(float *x, int n) {
+ for (int i = 0; i < n; i++) {
+ float val = x[i];
+ float x3 = val * val * val;
+ float inner = 0.7978845608028654f * (val + 0.044715f * x3);
+ x[i] = 0.5f * val * (1.0f + tanhf(inner));
+ }
+}
+
+typedef struct {
+ float *out;
+ const float *gate_up;
+ int seq_len;
+ int intermediate;
+} swiglu_task_t;
+
+static void swiglu_worker(int tid, int n_threads, void *arg) {
+ swiglu_task_t *t = (swiglu_task_t *)arg;
+ int chunk = (t->seq_len + n_threads - 1) / n_threads;
+ int s0 = tid * chunk;
+ int s1 = s0 + chunk;
+ if (s1 > t->seq_len) s1 = t->seq_len;
+ if (s0 >= s1) return;
+
+ int inter = t->intermediate;
+ int alias_inplace = (t->out == t->gate_up);
+ for (int s = s0; s < s1; s++) {
+ const float *gu = t->gate_up + (size_t)s * 2 * inter;
+ float *o = t->out + (size_t)s * inter;
+ if (!alias_inplace) {
+#if defined(__APPLE__) && defined(USE_BLAS)
+ /* Fast path for prefill: vectorized exp(-g) using Accelerate/vForce. */
+ for (int j = 0; j < inter; j++) o[j] = -gu[2 * j];
+ int n = inter;
+ vvexpf(o, o, &n);
+ for (int j = 0; j < inter; j++) {
+ float g = gu[2 * j];
+ float u = gu[2 * j + 1];
+ o[j] = (g / (1.0f + o[j])) * u;
+ }
+#else
+ for (int j = 0; j < inter; j++) {
+ float g = gu[2 * j];
+ float u = gu[2 * j + 1];
+ g = g / (1.0f + expf(-g)); /* SiLU */
+ o[j] = g * u;
+ }
+#endif
+ } else {
+ /* In-place mode (decode seq=1): keep single-pass scalar to avoid alias hazards. */
+ for (int j = 0; j < inter; j++) {
+ float g = gu[2 * j];
+ float u = gu[2 * j + 1];
+ g = g / (1.0f + expf(-g)); /* SiLU */
+ o[j] = g * u;
+ }
+ }
+ }
+}
+
+void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate) {
+ swiglu_task_t task = {
+ .out = out,
+ .gate_up = gate_up,
+ .seq_len = seq_len,
+ .intermediate = intermediate
+ };
+
+ if (tp.n_threads > 1 && seq_len >= 2 && intermediate >= 256) {
+ parallel_for(swiglu_worker, &task);
+ } else {
+ swiglu_worker(0, 1, &task);
+ }
+}
+
+void qwen_softmax(float *x, int rows, int cols) {
+ for (int r = 0; r < rows; r++) {
+ float *row = x + r * cols;
+ float max_val = row[0];
+ for (int c = 1; c < cols; c++) {
+ if (row[c] > max_val) max_val = row[c];
+ }
+ float sum = 0.0f;
+ for (int c = 0; c < cols; c++) {
+ row[c] = expf(row[c] - max_val);
+ sum += row[c];
+ }
+ float inv_sum = 1.0f / sum;
+ for (int c = 0; c < cols; c++) {
+ row[c] *= inv_sum;
+ }
+ }
+}
+
+/* ========================================================================
+ * Attention Operations
+ * ======================================================================== */
+
+static inline float qwen_dot_f32(const float *a, const float *b, int n) {
+ return qwen_dot_f32_impl(a, b, n);
+}
+
+/* dst = dst * scale */
+static inline void qwen_vec_scale_inplace(float *dst, float scale, int n) {
+ qwen_vec_scale_inplace_impl(dst, scale, n);
+}
+
+/* dst += alpha * src */
+static inline void qwen_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
+ qwen_vec_axpy_inplace_impl(dst, src, alpha, n);
+}
+
+/* dst = dst * correction + src */
+static inline void qwen_vec_scale_add(float *dst, const float *src, float correction, int n) {
+ qwen_vec_scale_add_impl(dst, src, correction, n);
+}
+
+void qwen_bidirectional_attention(float *out, const float *Q, const float *K,
+ const float *V, int seq __attribute__((unused)),
+ int n_heads, int head_dim, float scale,
+ const int *window_starts, int n_windows) {
+ int hidden = n_heads * head_dim;
+
+ for (int h = 0; h < n_heads; h++) {
+ for (int w = 0; w < n_windows; w++) {
+ int ws = window_starts[w];
+ int we = window_starts[w + 1];
+
+ for (int i = ws; i < we; i++) {
+ const float *q_row = Q + i * hidden + h * head_dim;
+ float *o_row = out + i * hidden + h * head_dim;
+
+ /* Online softmax */
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+
+ for (int j = ws; j < we; j++) {
+ const float *k_row = K + j * hidden + h * head_dim;
+ const float *v_row = V + j * hidden + h * head_dim;
+
+ float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ qwen_vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f) {
+ float inv_sum = 1.0f / sum_exp;
+ qwen_vec_scale_inplace(o_row, inv_sum, head_dim);
+ }
+ }
+ }
+ }
+}
+
+static void qwen_causal_attention_heads(float *out, const float *Q, const float *K,
+ const float *V, int seq_q, int seq_k,
+ int n_heads, int n_kv_heads, int head_dim,
+ float scale, int q_offset,
+ int head_start, int head_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+
+ for (int h = head_start; h < head_end; h++) {
+ int kv_h = h / heads_per_kv;
+
+ for (int i = 0; i < seq_q; i++) {
+ const float *q_row = Q + i * q_hidden + h * head_dim;
+ float *o_row = out + i * q_hidden + h * head_dim;
+ int global_pos = q_offset + i;
+ int k_end = global_pos + 1;
+ if (k_end > seq_k) k_end = seq_k;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+
+ for (int j = 0; j < k_end; j++) {
+ const float *k_row = K + j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + j * kv_hidden + kv_h * head_dim;
+
+ float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ qwen_vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f) {
+ float inv_sum = 1.0f / sum_exp;
+ qwen_vec_scale_inplace(o_row, inv_sum, head_dim);
+ }
+ }
+ }
+}
+
+typedef struct {
+ float *out;
+ const float *Q;
+ const float *K;
+ const float *V;
+ int seq_q, seq_k;
+ int n_heads, n_kv_heads;
+ int head_dim;
+ float scale;
+ int q_offset;
+} causal_attn_task_t;
+
+static void causal_attn_worker(int tid, int n_threads, void *arg) {
+ causal_attn_task_t *t = (causal_attn_task_t *)arg;
+ int chunk = (t->n_heads + n_threads - 1) / n_threads;
+ int h0 = tid * chunk;
+ int h1 = h0 + chunk;
+ if (h1 > t->n_heads) h1 = t->n_heads;
+ if (h0 >= h1) return;
+
+ qwen_causal_attention_heads(t->out, t->Q, t->K, t->V,
+ t->seq_q, t->seq_k, t->n_heads, t->n_kv_heads,
+ t->head_dim, t->scale, t->q_offset, h0, h1);
+}
+
+void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V,
+ int seq_q, int seq_k, int n_heads, int n_kv_heads,
+ int head_dim, float scale, int q_offset) {
+ if (tp.n_threads > 1 && n_heads >= 2 && (seq_q >= 2 || seq_k >= 128)) {
+ causal_attn_task_t task = {
+ .out = out, .Q = Q, .K = K, .V = V,
+ .seq_q = seq_q, .seq_k = seq_k,
+ .n_heads = n_heads, .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim, .scale = scale, .q_offset = q_offset
+ };
+ parallel_for(causal_attn_worker, &task);
+ return;
+ }
+
+ qwen_causal_attention_heads(out, Q, K, V,
+ seq_q, seq_k, n_heads, n_kv_heads,
+ head_dim, scale, q_offset, 0, n_heads);
+}
+
+/* ========================================================================
+ * Position Embeddings
+ * ======================================================================== */
+
+void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model) {
+ int half = d_model / 2;
+ float log_timescale = logf(10000.0f) / (float)(half - 1);
+
+ for (int p = 0; p < n_pos; p++) {
+ float *row = pe + p * d_model;
+ for (int d = 0; d < half; d++) {
+ float inv_timescale = expf(-(float)d * log_timescale);
+ float angle = (float)p * inv_timescale;
+ row[d] = sinf(angle); /* first half: sin */
+ row[half + d] = cosf(angle); /* second half: cos */
+ }
+ }
+}
+
+void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions,
+ int seq, int head_dim, float theta) {
+ int half = head_dim / 2;
+
+ for (int s = 0; s < seq; s++) {
+ float pos = (float)positions[s];
+ for (int d = 0; d < half; d++) {
+ float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
+ float angle = pos * freq;
+ float c = cosf(angle);
+ float sn = sinf(angle);
+ /* Duplicate for full head_dim */
+ cos_out[s * head_dim + d] = c;
+ cos_out[s * head_dim + half + d] = c;
+ sin_out[s * head_dim + d] = sn;
+ sin_out[s * head_dim + half + d] = sn;
+ }
+ }
+}
+
+void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals,
+ int seq, int n_heads, int head_dim) {
+ /*
+ * NeoX split-half style:
+ * x1 = x[..., :half], x2 = x[..., half:]
+ * rotated = cat(-x2, x1)
+ * result = x * cos + rotated * sin
+ */
+ int half = head_dim / 2;
+ int hidden = n_heads * head_dim;
+
+ for (int s = 0; s < seq; s++) {
+ const float *c = cos_vals + s * head_dim;
+ const float *sn = sin_vals + s * head_dim;
+
+ for (int h = 0; h < n_heads; h++) {
+ float *vec = x + s * hidden + h * head_dim;
+
+#if defined(__AVX512F__) && defined(__FMA__)
+ int d = 0;
+ for (; d + 16 <= half; d += 16) {
+ __m512 x1 = _mm512_loadu_ps(vec + d);
+ __m512 x2 = _mm512_loadu_ps(vec + half + d);
+ /* RoPE cache duplicates cos/sin across halves. */
+ __m512 cc = _mm512_loadu_ps(c + d);
+ __m512 ss = _mm512_loadu_ps(sn + d);
+ __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss));
+ __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss));
+ _mm512_storeu_ps(vec + d, new1);
+ _mm512_storeu_ps(vec + half + d, new2);
+ }
+ for (; d < half; d++) {
+ float x1 = vec[d];
+ float x2 = vec[half + d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ int d = 0;
+ for (; d + 8 <= half; d += 8) {
+ __m256 x1 = _mm256_loadu_ps(vec + d);
+ __m256 x2 = _mm256_loadu_ps(vec + half + d);
+ __m256 cc = _mm256_loadu_ps(c + d);
+ __m256 ss = _mm256_loadu_ps(sn + d);
+ __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss));
+ __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss));
+ _mm256_storeu_ps(vec + d, new1);
+ _mm256_storeu_ps(vec + half + d, new2);
+ }
+ for (; d < half; d++) {
+ float x1 = vec[d];
+ float x2 = vec[half + d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#else
+ for (int d = 0; d < half; d++) {
+ float x1 = vec[d]; /* first half */
+ float x2 = vec[half + d]; /* second half */
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#endif
+ }
+ }
+}
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
new file mode 100644
index 0000000..c59aeff
--- /dev/null
+++ b/qwen_asr_kernels.h
@@ -0,0 +1,173 @@
+/*
+ * qwen_asr_kernels.h - Math kernels for Qwen3-ASR inference
+ *
+ * Low-level math operations. All operate on float32 tensors in row-major order.
+ * Adapted from voxtral-realtime project.
+ */
+
+#ifndef QWEN_ASR_KERNELS_H
+#define QWEN_ASR_KERNELS_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* ========================================================================
+ * Basic Operations
+ * ======================================================================== */
+
+void qwen_add_inplace(float *a, const float *b, int n);
+void qwen_mul_inplace(float *a, const float *b, int n);
+void qwen_scale(float *x, float s, int n);
+void qwen_copy(float *dst, const float *src, int n);
+
+/* ========================================================================
+ * Matrix Operations
+ * ======================================================================== */
+
+/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */
+void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
+
+/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */
+void qwen_linear(float *y, const float *x, const float *W, const float *b,
+ int seq_len, int in_dim, int out_dim);
+
+void qwen_linear_nobias(float *y, const float *x, const float *W,
+ int seq_len, int in_dim, int out_dim);
+
+/* bf16 weight variants */
+void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16,
+ const float *b, int seq_len, int in_dim, int out_dim);
+
+void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
+ int seq_len, int in_dim, int out_dim);
+
+/* seq=1 decoder fast path: compute Q/K/V matvecs with one threaded dispatch */
+void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
+ const uint16_t *Wq_bf16,
+ const uint16_t *Wk_bf16,
+ const uint16_t *Wv_bf16,
+ int in_dim, int q_dim, int kv_dim);
+
+void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
+ int M, int K, int N);
+
+/* ========================================================================
+ * 2D Convolution (for audio encoder conv stem)
+ * ======================================================================== */
+
+/*
+ * 2D Convolution: out = conv2d(in, weight, bias)
+ * in: [C_in, H, W]
+ * weight: [C_out, C_in, kH, kW]
+ * bias: [C_out] (can be NULL)
+ * out: [C_out, H_out, W_out]
+ * H_out = (H + 2*padding - kH) / stride + 1
+ * W_out = (W + 2*padding - kW) / stride + 1
+ */
+void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias,
+ int c_in, int c_out, int h_in, int w_in,
+ int kh, int kw, int stride, int padding);
+
+/* ========================================================================
+ * Normalization
+ * ======================================================================== */
+
+/* LayerNorm with bias: out = (x - mean) / sqrt(var + eps) * weight + bias */
+void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias,
+ int seq_len, int hidden, float eps);
+
+/* RMS Normalization: out = x / rms(x) * weight */
+void qwen_rms_norm(float *out, const float *x, const float *weight,
+ int seq_len, int hidden, float eps);
+
+/* Per-head RMS Normalization for Q/K norms in decoder
+ * x: [seq, n_heads, head_dim], weight: [head_dim]
+ * Normalizes each head independently */
+void qwen_rms_norm_per_head(float *x, const float *weight,
+ int seq_len, int n_heads, int head_dim, float eps);
+
+/* ========================================================================
+ * Activation Functions
+ * ======================================================================== */
+
+void qwen_silu(float *x, int n);
+void qwen_gelu(float *x, int n);
+void qwen_softmax(float *x, int rows, int cols);
+/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */
+void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate);
+
+/* ========================================================================
+ * Attention Operations
+ * ======================================================================== */
+
+/*
+ * Bidirectional windowed attention (encoder).
+ * Q, K, V: [seq, n_heads * head_dim]
+ * out: [seq, n_heads * head_dim]
+ * window_starts: array of window start positions
+ * window_starts[n_windows] = seq (sentinel)
+ * All heads have same dimensions (no GQA in encoder).
+ */
+void qwen_bidirectional_attention(float *out, const float *Q, const float *K,
+ const float *V, int seq, int n_heads,
+ int head_dim, float scale,
+ const int *window_starts, int n_windows);
+
+/*
+ * Causal attention with GQA (decoder).
+ * Q: [seq_q, n_heads * head_dim]
+ * K: [seq_k, n_kv_heads * head_dim]
+ * V: [seq_k, n_kv_heads * head_dim]
+ * q_offset: global position of first query (for causal mask)
+ */
+void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V,
+ int seq_q, int seq_k, int n_heads, int n_kv_heads,
+ int head_dim, float scale, int q_offset);
+
+/* ========================================================================
+ * Position Embeddings
+ * ======================================================================== */
+
+/*
+ * Sinusoidal position embeddings (encoder).
+ * pe: output [n_pos, d_model]
+ * First half = sin, second half = cos.
+ */
+void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model);
+
+/*
+ * NeoX-style RoPE: compute cos/sin for positions.
+ * cos_out, sin_out: [seq, head_dim]
+ * cos[d] and cos[half+d] are the same (duplicated for full head_dim).
+ */
+void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions,
+ int seq, int head_dim, float theta);
+
+/*
+ * Apply NeoX-style RoPE to Q or K.
+ * x: [seq, n_heads * head_dim] (in-place)
+ * cos_vals, sin_vals: [seq, head_dim]
+ */
+void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals,
+ int seq, int n_heads, int head_dim);
+
+/* Streaming argmax: finds argmax(W_bf16 @ x) without materializing full logits.
+ * Returns the index of the row with highest dot product. */
+int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16,
+ int in_dim, int out_dim);
+
+/* ========================================================================
+ * Threading
+ * ======================================================================== */
+
+/* Set number of threads for parallel operations (default: 1).
+ * Creates a persistent thread pool. Call before inference. */
+void qwen_set_threads(int n);
+
+/* Get number of available CPU cores */
+int qwen_get_num_cpus(void);
+
+/* Global verbose flag */
+extern int qwen_verbose;
+
+#endif /* QWEN_ASR_KERNELS_H */
diff --git a/qwen_asr_kernels_avx.c b/qwen_asr_kernels_avx.c
new file mode 100644
index 0000000..3fa2194
--- /dev/null
+++ b/qwen_asr_kernels_avx.c
@@ -0,0 +1,502 @@
+/*
+ * qwen_asr_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available)
+ *
+ * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16.
+ * Uses AVX-512F+BW for 16-wide bf16 matvec/argmax (dominant for seq_len==1), and
+ * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data).
+ *
+ * The bf16 matvec processes 4 output rows simultaneously to reduce instruction
+ * overhead and improve out-of-order execution on memory-bound workloads.
+ */
+
+#include "qwen_asr_kernels_impl.h"
+
+#if defined(__AVX2__) && defined(__FMA__)
+
+#include <immintrin.h>
+#include <string.h>
+
+/* =====================================================================
+ * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise
+ * ===================================================================== */
+
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+
+void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ int o = 0;
+
+ /* Process 4 output rows at a time */
+ for (; o + 3 < out_dim; o += 4) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ const uint16_t *w2 = w1 + in_dim;
+ const uint16_t *w3 = w2 + in_dim;
+
+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
+ __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps();
+ __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps();
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ __m512 xv0 = _mm512_loadu_ps(x + k);
+ __m512 xv1 = _mm512_loadu_ps(x + k + 16);
+
+ /* Row 0 */
+ __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
+ a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+
+ /* Row 1 */
+ __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
+ a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+
+ /* Row 2 */
+ __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k));
+ __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16));
+ a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4);
+ a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5);
+
+ /* Row 3 */
+ __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k));
+ __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16));
+ a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6);
+ a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7);
+ }
+
+ for (; k + 16 <= in_dim; k += 16) {
+ __m512 xv = _mm512_loadu_ps(x + k);
+ __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k));
+ __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k));
+ a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4);
+ a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6);
+ }
+
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f);
+ float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f);
+ float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f);
+
+ for (; k < in_dim; k++) {
+ float xk = x[k];
+ uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
+ uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16;
+ float v0,v1,v2,v3;
+ memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4);
+ s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk;
+ }
+ y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3;
+ }
+
+ /* Handle 2-row remainder */
+ for (; o + 1 < out_dim; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ __m512 xv0 = _mm512_loadu_ps(x + k);
+ __m512 xv1 = _mm512_loadu_ps(x + k + 16);
+ __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
+ __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
+ a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+ a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+ }
+ for (; k + 16 <= in_dim; k += 16) {
+ __m512 xv = _mm512_loadu_ps(x + k);
+ __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ }
+
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f);
+
+ for (; k < in_dim; k++) {
+ uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
+ float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4);
+ s0+=v0*x[k]; s1+=v1*x[k];
+ }
+ y[o]=s0; y[o+1]=s1;
+ }
+
+ /* Single remaining row */
+ for (; o < out_dim; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m512 acc = _mm512_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k));
+ acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
+ _mm512_loadu_ps(x + k), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
+ sum += wv*x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out) {
+ int best = start;
+ float best_val = -1e30f;
+ int o = start;
+
+ /* Process 4 rows at a time */
+ for (; o + 3 < end; o += 4) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ const uint16_t *w2 = w1 + in_dim;
+ const uint16_t *w3 = w2 + in_dim;
+
+ __m512 a0=_mm512_setzero_ps(), a1=_mm512_setzero_ps();
+ __m512 a2=_mm512_setzero_ps(), a3=_mm512_setzero_ps();
+ __m512 a4=_mm512_setzero_ps(), a5=_mm512_setzero_ps();
+ __m512 a6=_mm512_setzero_ps(), a7=_mm512_setzero_ps();
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ __m512 xv0 = _mm512_loadu_ps(x + k);
+ __m512 xv1 = _mm512_loadu_ps(x + k + 16);
+ __m256i r0a=_mm256_loadu_si256((const __m256i*)(w0+k));
+ __m256i r0b=_mm256_loadu_si256((const __m256i*)(w0+k+16));
+ a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a),16)),xv0,a0);
+ a1=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b),16)),xv1,a1);
+ __m256i r1a=_mm256_loadu_si256((const __m256i*)(w1+k));
+ __m256i r1b=_mm256_loadu_si256((const __m256i*)(w1+k+16));
+ a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a),16)),xv0,a2);
+ a3=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b),16)),xv1,a3);
+ __m256i r2a=_mm256_loadu_si256((const __m256i*)(w2+k));
+ __m256i r2b=_mm256_loadu_si256((const __m256i*)(w2+k+16));
+ a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a),16)),xv0,a4);
+ a5=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b),16)),xv1,a5);
+ __m256i r3a=_mm256_loadu_si256((const __m256i*)(w3+k));
+ __m256i r3b=_mm256_loadu_si256((const __m256i*)(w3+k+16));
+ a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a),16)),xv0,a6);
+ a7=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b),16)),xv1,a7);
+ }
+ for (; k + 16 <= in_dim; k += 16) {
+ __m512 xv = _mm512_loadu_ps(x + k);
+ __m256i r0=_mm256_loadu_si256((const __m256i*)(w0+k));
+ __m256i r1=_mm256_loadu_si256((const __m256i*)(w1+k));
+ __m256i r2=_mm256_loadu_si256((const __m256i*)(w2+k));
+ __m256i r3=_mm256_loadu_si256((const __m256i*)(w3+k));
+ a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0),16)),xv,a0);
+ a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1),16)),xv,a2);
+ a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2),16)),xv,a4);
+ a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3),16)),xv,a6);
+ }
+
+ float s0=_mm512_reduce_add_ps(_mm512_add_ps(a0,a1));
+ float s1=_mm512_reduce_add_ps(_mm512_add_ps(a2,a3));
+ float s2=_mm512_reduce_add_ps(_mm512_add_ps(a4,a5));
+ float s3=_mm512_reduce_add_ps(_mm512_add_ps(a6,a7));
+
+ for (; k < in_dim; k++) {
+ float xk=x[k];
+ uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
+ uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16;
+ float v0,v1,v2,v3;
+ memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4);
+ s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk;
+ }
+
+ if (s0>best_val){best_val=s0;best=o;}
+ if (s1>best_val){best_val=s1;best=o+1;}
+ if (s2>best_val){best_val=s2;best=o+2;}
+ if (s3>best_val){best_val=s3;best=o+3;}
+ }
+
+ /* Remaining 1-3 rows */
+ for (; o < end; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m512 acc = _mm512_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k));
+ acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
+ _mm512_loadu_ps(x + k), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc);
+ for (; k < in_dim; k++) {
+ uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
+ sum += wv*x[k];
+ }
+ if (sum > best_val) { best_val = sum; best = o; }
+ }
+
+ *best_out = best;
+ *best_val_out = best_val;
+}
+
+#else /* AVX2 only (or AVX-512F without BW) */
+
+/* Helper: bf16 to f32 for 8 elements */
+static inline __m256 bf16x8_to_f32(__m128i raw) {
+ return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16));
+}
+
+static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) {
+ __m256i raw = _mm256_loadu_si256((const __m256i *)src);
+ *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw));
+ *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1));
+}
+
+void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ int o = 0;
+
+ for (; o + 1 < out_dim; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps();
+ __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256 xlo = _mm256_loadu_ps(x + k);
+ __m256 xhi = _mm256_loadu_ps(x + k + 8);
+ __m256 wlo, whi;
+ bf16x16_to_f32(w0 + k, &wlo, &whi);
+ a0 = _mm256_fmadd_ps(wlo, xlo, a0);
+ a1 = _mm256_fmadd_ps(whi, xhi, a1);
+ bf16x16_to_f32(w1 + k, &wlo, &whi);
+ a2 = _mm256_fmadd_ps(wlo, xlo, a2);
+ a3 = _mm256_fmadd_ps(whi, xhi, a3);
+ }
+ a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3);
+ __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
+ __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1));
+ r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0);
+ float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
+ float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4);
+ s0+=v0*x[k]; s1+=v1*x[k];
+ }
+ y[o]=s0; y[o+1]=s1;
+ }
+ for (; o < out_dim; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256 wlo, whi;
+ bf16x16_to_f32(w + k, &wlo, &whi);
+ a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0);
+ a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1);
+ }
+ a0 = _mm256_add_ps(a0, a1);
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
+ r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
+ sum+=wv*x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out) {
+ int best = start;
+ float best_val = -1e30f;
+
+ for (int o = start; o < end; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256 wlo, whi;
+ bf16x16_to_f32(w + k, &wlo, &whi);
+ a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0);
+ a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1);
+ }
+ a0 = _mm256_add_ps(a0, a1);
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
+ r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r);
+ for (; k < in_dim; k++) {
+ uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
+ sum+=wv*x[k];
+ }
+ if (sum > best_val) { best_val = sum; best = o; }
+ }
+ *best_out = best;
+ *best_val_out = best_val;
+}
+
+#endif /* AVX-512F+BW vs AVX2 for bf16 */
+
+/* =====================================================================
+ * f32 attention helpers - AVX2+FMA, with AVX-512F when available
+ * (operates on L1-resident head vectors)
+ * ===================================================================== */
+
+float qwen_dot_f32_avx(const float *a, const float *b, int n) {
+#if defined(__AVX512F__)
+ int i = 0;
+ __m512 acc0 = _mm512_setzero_ps();
+ __m512 acc1 = _mm512_setzero_ps();
+ __m512 acc2 = _mm512_setzero_ps();
+ __m512 acc3 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
+ acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
+ acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
+ acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
+ }
+ __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
+ for (; i + 16 <= n; i += 16) {
+ acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc);
+ for (; i < n; i++) sum += a[i] * b[i];
+ return sum;
+#else
+ int i = 0;
+ __m256 acc0 = _mm256_setzero_ps();
+ __m256 acc1 = _mm256_setzero_ps();
+ __m256 acc2 = _mm256_setzero_ps();
+ __m256 acc3 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0);
+ acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1);
+ acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2);
+ acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3);
+ }
+ acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
+ for (; i + 8 <= n; i += 8) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0);
+ }
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
+ r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r);
+ for (; i < n; i++) sum += a[i] * b[i];
+ return sum;
+#endif
+}
+
+void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) {
+#if defined(__AVX512F__)
+ int i = 0;
+ __m512 s = _mm512_set1_ps(scale);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
+ _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
+ _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ }
+ for (; i < n; i++) dst[i] *= scale;
+#else
+ int i = 0;
+ __m256 s = _mm256_set1_ps(scale);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s));
+ _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s));
+ _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s));
+ _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s));
+ }
+ for (; i < n; i++) dst[i] *= scale;
+#endif
+}
+
+void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) {
+#if defined(__AVX512F__)
+ int i = 0;
+ __m512 a = _mm512_set1_ps(alpha);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ _mm512_storeu_ps(dst + i + 16,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16)));
+ _mm512_storeu_ps(dst + i + 32,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32)));
+ _mm512_storeu_ps(dst + i + 48,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ }
+ for (; i < n; i++) dst[i] += alpha * src[i];
+#else
+ int i = 0;
+ __m256 a = _mm256_set1_ps(alpha);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i)));
+ _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8)));
+ _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16)));
+ _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i)));
+ }
+ for (; i < n; i++) dst[i] += alpha * src[i];
+#endif
+}
+
+void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n) {
+#if defined(__AVX512F__)
+ int i = 0;
+ __m512 c = _mm512_set1_ps(correction);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ _mm512_storeu_ps(dst + i + 16,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16)));
+ _mm512_storeu_ps(dst + i + 32,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32)));
+ _mm512_storeu_ps(dst + i + 48,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ }
+ for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+#else
+ int i = 0;
+ __m256 c = _mm256_set1_ps(correction);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i)));
+ _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8)));
+ _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16)));
+ _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i)));
+ }
+ for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+#endif
+}
+
+#endif /* __AVX2__ && __FMA__ */
diff --git a/qwen_asr_kernels_generic.c b/qwen_asr_kernels_generic.c
new file mode 100644
index 0000000..d322d8e
--- /dev/null
+++ b/qwen_asr_kernels_generic.c
@@ -0,0 +1,65 @@
+/*
+ * qwen_asr_kernels_generic.c - architecture-generic hot kernels
+ */
+
+#include "qwen_asr_kernels_impl.h"
+
+#include <string.h>
+
+void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ for (int o = 0; o < out_dim; o++) {
+ const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
+ float sum = bias ? bias[o] : 0.0f;
+ for (int k = 0; k < in_dim; k++) {
+ uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
+ float w_val;
+ memcpy(&w_val, &f32_bits, sizeof(float));
+ sum += w_val * x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out) {
+ int best = start;
+ float best_val = -1e30f;
+
+ for (int o = start; o < end; o++) {
+ const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
+ float sum = 0.0f;
+ for (int k = 0; k < in_dim; k++) {
+ uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
+ float w_val;
+ memcpy(&w_val, &f32_bits, sizeof(float));
+ sum += w_val * x[k];
+ }
+ if (sum > best_val) {
+ best_val = sum;
+ best = o;
+ }
+ }
+
+ *best_out = best;
+ *best_val_out = best_val;
+}
+
+float qwen_dot_f32_generic(const float *a, const float *b, int n) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; i++) sum += a[i] * b[i];
+ return sum;
+}
+
+void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) {
+ for (int i = 0; i < n; i++) dst[i] *= scale;
+}
+
+void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
+ for (int i = 0; i < n; i++) dst[i] += alpha * src[i];
+}
+
+void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
+ for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i];
+}
diff --git a/qwen_asr_kernels_impl.h b/qwen_asr_kernels_impl.h
new file mode 100644
index 0000000..31b9cbb
--- /dev/null
+++ b/qwen_asr_kernels_impl.h
@@ -0,0 +1,65 @@
+/*
+ * qwen_asr_kernels_impl.h - internal architecture dispatch for hot kernels
+ */
+
+#ifndef QWEN_ASR_KERNELS_IMPL_H
+#define QWEN_ASR_KERNELS_IMPL_H
+
+#include <stdint.h>
+
+void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim);
+void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out);
+float qwen_dot_f32_generic(const float *a, const float *b, int n);
+void qwen_vec_scale_inplace_generic(float *dst, float scale, int n);
+void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
+void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n);
+
+#ifdef __ARM_NEON
+void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim);
+void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out);
+float qwen_dot_f32_neon(const float *a, const float *b, int n);
+void qwen_vec_scale_inplace_neon(float *dst, float scale, int n);
+void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n);
+void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n);
+
+#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon
+#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_neon
+#define qwen_dot_f32_impl qwen_dot_f32_neon
+#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon
+
+#elif defined(__AVX2__) && defined(__FMA__)
+void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim);
+void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out);
+float qwen_dot_f32_avx(const float *a, const float *b, int n);
+void qwen_vec_scale_inplace_avx(float *dst, float scale, int n);
+void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n);
+void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n);
+
+#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx
+#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_avx
+#define qwen_dot_f32_impl qwen_dot_f32_avx
+#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx
+
+#else
+#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic
+#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_generic
+#define qwen_dot_f32_impl qwen_dot_f32_generic
+#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic
+#endif
+
+#endif /* QWEN_ASR_KERNELS_IMPL_H */
diff --git a/qwen_asr_kernels_neon.c b/qwen_asr_kernels_neon.c
new file mode 100644
index 0000000..90d8ed9
--- /dev/null
+++ b/qwen_asr_kernels_neon.c
@@ -0,0 +1,274 @@
+/*
+ * qwen_asr_kernels_neon.c - ARM NEON hot kernels
+ */
+
+#include "qwen_asr_kernels_impl.h"
+
+#ifdef __ARM_NEON
+
+#include <arm_neon.h>
+#include <string.h>
+
+void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16,
+ const float *bias, int in_dim, int out_dim) {
+ int o = 0;
+
+ /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */
+ for (; o + 1 < out_dim; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim;
+ float s0 = bias ? bias[o] : 0.0f;
+ float s1 = bias ? bias[o + 1] : 0.0f;
+
+ float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f);
+ float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f);
+ float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f);
+ float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f);
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ float32x4_t x0 = vld1q_f32(x + k);
+ float32x4_t x1 = vld1q_f32(x + k + 4);
+ float32x4_t x2 = vld1q_f32(x + k + 8);
+ float32x4_t x3 = vld1q_f32(x + k + 12);
+ float32x4_t x4 = vld1q_f32(x + k + 16);
+ float32x4_t x5 = vld1q_f32(x + k + 20);
+ float32x4_t x6 = vld1q_f32(x + k + 24);
+ float32x4_t x7 = vld1q_f32(x + k + 28);
+
+ uint16x8_t r0a = vld1q_u16(w0 + k);
+ uint16x8_t r0b = vld1q_u16(w0 + k + 8);
+ uint16x8_t r0c = vld1q_u16(w0 + k + 16);
+ uint16x8_t r0d = vld1q_u16(w0 + k + 24);
+ a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0);
+ a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1);
+ a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2);
+ a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3);
+ a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4);
+ a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5);
+ a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6);
+ a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7);
+
+ uint16x8_t r1a = vld1q_u16(w1 + k);
+ uint16x8_t r1b = vld1q_u16(w1 + k + 8);
+ uint16x8_t r1c = vld1q_u16(w1 + k + 16);
+ uint16x8_t r1d = vld1q_u16(w1 + k + 24);
+ b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0);
+ b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1);
+ b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2);
+ b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3);
+ b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4);
+ b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5);
+ b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6);
+ b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7);
+ }
+ for (; k + 8 <= in_dim; k += 8) {
+ float32x4_t x0 = vld1q_f32(x + k);
+ float32x4_t x1 = vld1q_f32(x + k + 4);
+ uint16x8_t r0 = vld1q_u16(w0 + k);
+ uint16x8_t r1 = vld1q_u16(w1 + k);
+ a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0), 16)), x0);
+ a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0), 16)), x1);
+ b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1), 16)), x0);
+ b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1), 16)), x1);
+ }
+ s0 += vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3)));
+ s1 += vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3)));
+
+ for (; k < in_dim; k++) {
+ uint32_t bits0 = ((uint32_t)w0[k]) << 16;
+ uint32_t bits1 = ((uint32_t)w1[k]) << 16;
+ float wv0, wv1;
+ memcpy(&wv0, &bits0, sizeof(float));
+ memcpy(&wv1, &bits1, sizeof(float));
+ s0 += wv0 * x[k];
+ s1 += wv1 * x[k];
+ }
+ y[o] = s0;
+ y[o + 1] = s1;
+ }
+
+ /* Handle remaining odd row */
+ for (; o < out_dim; o++) {
+ const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
+ float sum = bias ? bias[o] : 0.0f;
+ int k = 0;
+
+ float32x4_t acc0 = vdupq_n_f32(0.0f);
+ float32x4_t acc1 = vdupq_n_f32(0.0f);
+ for (; k + 8 <= in_dim; k += 8) {
+ uint16x8_t bf = vld1q_u16(w_row + k);
+ acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)),
+ vld1q_f32(x + k));
+ acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)),
+ vld1q_f32(x + k + 4));
+ }
+ sum += vaddvq_f32(vaddq_f32(acc0, acc1));
+
+ for (; k < in_dim; k++) {
+ uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
+ float w_val;
+ memcpy(&w_val, &f32_bits, sizeof(float));
+ sum += w_val * x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16,
+ int in_dim, int start, int end,
+ int *best_out, float *best_val_out) {
+ int best = start;
+ float best_val = -1e30f;
+ int o = start;
+
+ /* Process 2 rows at a time, 32 elements/iter, 8 accumulators per row */
+ for (; o + 1 < end; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim;
+ float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f);
+ float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f);
+ float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f);
+ float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f);
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ float32x4_t x0 = vld1q_f32(x + k);
+ float32x4_t x1 = vld1q_f32(x + k + 4);
+ float32x4_t x2 = vld1q_f32(x + k + 8);
+ float32x4_t x3 = vld1q_f32(x + k + 12);
+ float32x4_t x4 = vld1q_f32(x + k + 16);
+ float32x4_t x5 = vld1q_f32(x + k + 20);
+ float32x4_t x6 = vld1q_f32(x + k + 24);
+ float32x4_t x7 = vld1q_f32(x + k + 28);
+
+ uint16x8_t r0a = vld1q_u16(w0 + k);
+ uint16x8_t r0b = vld1q_u16(w0 + k + 8);
+ uint16x8_t r0c = vld1q_u16(w0 + k + 16);
+ uint16x8_t r0d = vld1q_u16(w0 + k + 24);
+ a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0);
+ a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1);
+ a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2);
+ a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3);
+ a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4);
+ a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5);
+ a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6);
+ a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7);
+
+ uint16x8_t r1a = vld1q_u16(w1 + k);
+ uint16x8_t r1b = vld1q_u16(w1 + k + 8);
+ uint16x8_t r1c = vld1q_u16(w1 + k + 16);
+ uint16x8_t r1d = vld1q_u16(w1 + k + 24);
+ b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0);
+ b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1);
+ b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2);
+ b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3);
+ b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4);
+ b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5);
+ b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6);
+ b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7);
+ }
+
+ float s0 = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3)));
+ float s1 = vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3)));
+
+ for (; k < in_dim; k++) {
+ uint32_t bits0 = ((uint32_t)w0[k]) << 16;
+ uint32_t bits1 = ((uint32_t)w1[k]) << 16;
+ float wv0, wv1;
+ memcpy(&wv0, &bits0, sizeof(float));
+ memcpy(&wv1, &bits1, sizeof(float));
+ s0 += wv0 * x[k];
+ s1 += wv1 * x[k];
+ }
+
+ if (s0 > best_val) { best_val = s0; best = o; }
+ if (s1 > best_val) { best_val = s1; best = o + 1; }
+ }
+
+ for (; o < end; o++) {
+ const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
+ float sum = 0.0f;
+ int k = 0;
+
+ float32x4_t acc0 = vdupq_n_f32(0.0f), acc1 = vdupq_n_f32(0.0f);
+ for (; k + 8 <= in_dim; k += 8) {
+ uint16x8_t bf = vld1q_u16(w_row + k);
+ acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)),
+ vld1q_f32(x + k));
+ acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)),
+ vld1q_f32(x + k + 4));
+ }
+ sum += vaddvq_f32(vaddq_f32(acc0, acc1));
+
+ for (; k < in_dim; k++) {
+ uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
+ float w_val;
+ memcpy(&w_val, &f32_bits, sizeof(float));
+ sum += w_val * x[k];
+ }
+ if (sum > best_val) { best_val = sum; best = o; }
+ }
+
+ *best_out = best;
+ *best_val_out = best_val;
+}
+
+float qwen_dot_f32_neon(const float *a, const float *b, int n) {
+ int i = 0;
+ float32x4_t acc0 = vdupq_n_f32(0.0f);
+ float32x4_t acc1 = vdupq_n_f32(0.0f);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t a0 = vld1q_f32(a + i);
+ float32x4_t b0 = vld1q_f32(b + i);
+ float32x4_t a1 = vld1q_f32(a + i + 4);
+ float32x4_t b1 = vld1q_f32(b + i + 4);
+ acc0 = vfmaq_f32(acc0, a0, b0);
+ acc1 = vfmaq_f32(acc1, a1, b1);
+ }
+ float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
+ for (; i < n; i++) sum += a[i] * b[i];
+ return sum;
+}
+
+void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) {
+ int i = 0;
+ float32x4_t s = vdupq_n_f32(scale);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
+ vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
+ }
+ for (; i < n; i++) dst[i] *= scale;
+}
+
+void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) {
+ int i = 0;
+ float32x4_t a = vdupq_n_f32(alpha);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
+ vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
+ }
+ for (; i < n; i++) dst[i] += alpha * src[i];
+}
+
+void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) {
+ int i = 0;
+ float32x4_t c = vdupq_n_f32(correction);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
+ vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
+ }
+ for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+}
+
+#endif /* __ARM_NEON */
diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c
new file mode 100644
index 0000000..0bdf272
--- /dev/null
+++ b/qwen_asr_safetensors.c
@@ -0,0 +1,394 @@
+/*
+ * qwen_asr_safetensors.c - Safetensors reader with multi-shard support
+ * Adapted from voxtral-realtime project.
+ */
+
+#include "qwen_asr_safetensors.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <dirent.h>
+
+/* ========================================================================
+ * Minimal JSON parser for safetensors header
+ * ======================================================================== */
+
+static void skip_whitespace(const char **p) {
+ while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++;
+}
+
+static int parse_string(const char **p, char *out, size_t max_len) {
+ skip_whitespace(p);
+ if (**p != '"') return -1;
+ (*p)++;
+ size_t i = 0;
+ while (**p && **p != '"' && i < max_len - 1) {
+ if (**p == '\\') {
+ (*p)++;
+ if (**p == 'n') out[i++] = '\n';
+ else if (**p == 't') out[i++] = '\t';
+ else if (**p == '"') out[i++] = '"';
+ else if (**p == '\\') out[i++] = '\\';
+ else out[i++] = **p;
+ } else {
+ out[i++] = **p;
+ }
+ (*p)++;
+ }
+ out[i] = '\0';
+ if (**p != '"') return -1;
+ (*p)++;
+ return 0;
+}
+
+static int64_t parse_int(const char **p) {
+ skip_whitespace(p);
+ int64_t val = 0;
+ int neg = 0;
+ if (**p == '-') { neg = 1; (*p)++; }
+ while (**p >= '0' && **p <= '9') {
+ val = val * 10 + (**p - '0');
+ (*p)++;
+ }
+ return neg ? -val : val;
+}
+
+static safetensor_dtype_t parse_dtype(const char *s) {
+ if (strcmp(s, "F32") == 0) return DTYPE_F32;
+ if (strcmp(s, "F16") == 0) return DTYPE_F16;
+ if (strcmp(s, "BF16") == 0) return DTYPE_BF16;
+ if (strcmp(s, "I32") == 0) return DTYPE_I32;
+ if (strcmp(s, "I64") == 0) return DTYPE_I64;
+ if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL;
+ return DTYPE_UNKNOWN;
+}
+
+static int parse_tensor_entry(const char **p, safetensor_t *t) {
+ skip_whitespace(p);
+ if (**p != '{') return -1;
+ (*p)++;
+
+ t->dtype = DTYPE_UNKNOWN;
+ t->ndim = 0;
+ t->data_offset = 0;
+ t->data_size = 0;
+
+ while (**p && **p != '}') {
+ skip_whitespace(p);
+ if (**p == ',') { (*p)++; continue; }
+
+ char key[64];
+ if (parse_string(p, key, sizeof(key)) != 0) return -1;
+ skip_whitespace(p);
+ if (**p != ':') return -1;
+ (*p)++;
+ skip_whitespace(p);
+
+ if (strcmp(key, "dtype") == 0) {
+ char dtype_str[32];
+ if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1;
+ t->dtype = parse_dtype(dtype_str);
+ } else if (strcmp(key, "shape") == 0) {
+ if (**p != '[') return -1;
+ (*p)++;
+ t->ndim = 0;
+ while (**p && **p != ']' && t->ndim < 8) {
+ skip_whitespace(p);
+ if (**p == ',') { (*p)++; continue; }
+ t->shape[t->ndim++] = parse_int(p);
+ }
+ if (**p == ']') (*p)++;
+ } else if (strcmp(key, "data_offsets") == 0) {
+ if (**p != '[') return -1;
+ (*p)++;
+ skip_whitespace(p);
+ size_t start = (size_t)parse_int(p);
+ skip_whitespace(p);
+ if (**p == ',') (*p)++;
+ skip_whitespace(p);
+ size_t end = (size_t)parse_int(p);
+ t->data_offset = start;
+ t->data_size = end - start;
+ skip_whitespace(p);
+ if (**p == ']') (*p)++;
+ } else {
+ /* Skip unknown value */
+ if (**p == '"') {
+ (*p)++;
+ while (**p && **p != '"') {
+ if (**p == '\\') (*p)++;
+ if (**p) (*p)++;
+ }
+ if (**p == '"') (*p)++;
+ } else if (**p == '[') {
+ int depth = 1; (*p)++;
+ while (**p && depth > 0) {
+ if (**p == '[') depth++;
+ else if (**p == ']') depth--;
+ (*p)++;
+ }
+ } else if (**p == '{') {
+ int depth = 1; (*p)++;
+ while (**p && depth > 0) {
+ if (**p == '{') depth++;
+ else if (**p == '}') depth--;
+ (*p)++;
+ }
+ } else {
+ while (**p && **p != ',' && **p != '}') (*p)++;
+ }
+ }
+ }
+ if (**p == '}') (*p)++;
+ return 0;
+}
+
+static int parse_header(safetensors_file_t *sf) {
+ const char *p = sf->header_json;
+ skip_whitespace(&p);
+ if (*p != '{') return -1;
+ p++;
+
+ sf->num_tensors = 0;
+
+ while (*p && *p != '}' && sf->num_tensors < SAFETENSORS_MAX_TENSORS) {
+ skip_whitespace(&p);
+ if (*p == ',') { p++; continue; }
+ if (*p == '}') break;
+
+ char name[256];
+ if (parse_string(&p, name, sizeof(name)) != 0) return -1;
+ skip_whitespace(&p);
+ if (*p != ':') return -1;
+ p++;
+
+ if (strcmp(name, "__metadata__") == 0) {
+ skip_whitespace(&p);
+ if (*p == '{') {
+ int depth = 1; p++;
+ while (*p && depth > 0) {
+ if (*p == '{') depth++;
+ else if (*p == '}') depth--;
+ p++;
+ }
+ }
+ continue;
+ }
+
+ safetensor_t *t = &sf->tensors[sf->num_tensors];
+ snprintf(t->name, sizeof(t->name), "%s", name);
+ if (parse_tensor_entry(&p, t) != 0) return -1;
+ sf->num_tensors++;
+ }
+ return 0;
+}
+
+/* ========================================================================
+ * Single file operations
+ * ======================================================================== */
+
+safetensors_file_t *safetensors_open(const char *path) {
+ int fd = open(path, O_RDONLY);
+ if (fd < 0) return NULL;
+
+ struct stat st;
+ if (fstat(fd, &st) < 0) { close(fd); return NULL; }
+
+ size_t file_size = (size_t)st.st_size;
+ if (file_size < 8) { close(fd); return NULL; }
+
+ void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (data == MAP_FAILED) return NULL;
+
+ uint64_t header_size = 0;
+ memcpy(&header_size, data, 8);
+ if (header_size > file_size - 8) { munmap(data, file_size); return NULL; }
+
+ safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
+ if (!sf) { munmap(data, file_size); return NULL; }
+
+ sf->path = strdup(path);
+ sf->data = data;
+ sf->file_size = file_size;
+ sf->header_size = (size_t)header_size;
+
+ sf->header_json = malloc(header_size + 1);
+ if (!sf->header_json) { safetensors_close(sf); return NULL; }
+ memcpy(sf->header_json, (char *)data + 8, header_size);
+ sf->header_json[header_size] = '\0';
+
+ if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; }
+
+ return sf;
+}
+
+void safetensors_close(safetensors_file_t *sf) {
+ if (!sf) return;
+ if (sf->data) munmap(sf->data, sf->file_size);
+ free(sf->path);
+ free(sf->header_json);
+ free(sf);
+}
+
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
+ return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
+}
+
+int64_t safetensor_numel(const safetensor_t *t) {
+ int64_t n = 1;
+ for (int i = 0; i < t->ndim; i++) n *= t->shape[i];
+ return n;
+}
+
+static float bf16_to_f32(uint16_t bf16) {
+ uint32_t f32 = ((uint32_t)bf16) << 16;
+ float result;
+ memcpy(&result, &f32, sizeof(float));
+ return result;
+}
+
+float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
+ int64_t n = safetensor_numel(t);
+ if (n <= 0) return NULL;
+
+ float *out = malloc(n * sizeof(float));
+ if (!out) return NULL;
+
+ const void *data = safetensors_data(sf, t);
+
+ switch (t->dtype) {
+ case DTYPE_F32:
+ memcpy(out, data, n * sizeof(float));
+ break;
+ case DTYPE_BF16: {
+ const uint16_t *src = (const uint16_t *)data;
+ for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]);
+ break;
+ }
+ default:
+ free(out);
+ return NULL;
+ }
+ return out;
+}
+
+int safetensor_is_bf16(const safetensor_t *t) {
+ return t && t->dtype == DTYPE_BF16;
+}
+
+uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t) {
+ if (!sf || !t || t->dtype != DTYPE_BF16) return NULL;
+ return (uint16_t *)safetensors_data(sf, t);
+}
+
+void safetensor_print(const safetensor_t *t) {
+ const char *dtype_names[] = {"F32", "F16", "BF16", "I32", "I64", "BOOL"};
+ const char *dtype_name = t->dtype >= 0 && t->dtype <= 5 ?
+ dtype_names[t->dtype] : "UNKNOWN";
+ printf("%s: dtype=%s, shape=[", t->name, dtype_name);
+ for (int i = 0; i < t->ndim; i++) {
+ printf("%ld%s", (long)t->shape[i], i < t->ndim - 1 ? ", " : "");
+ }
+ printf("]\n");
+}
+
+void safetensors_print_all(const safetensors_file_t *sf) {
+ printf("File: %s (%d tensors)\n", sf->path, sf->num_tensors);
+ for (int i = 0; i < sf->num_tensors; i++) safetensor_print(&sf->tensors[i]);
+}
+
+/* ========================================================================
+ * Multi-shard operations
+ * ======================================================================== */
+
+multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
+ multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
+ if (!ms) return NULL;
+
+ char path[4096];
+
+ /* Try single file first */
+ snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
+ safetensors_file_t *sf = safetensors_open(path);
+ if (sf) {
+ ms->shards[0] = sf;
+ ms->num_shards = 1;
+ return ms;
+ }
+
+ /* Try multi-shard: model-00001-of-NNNNN.safetensors */
+ for (int i = 1; i <= SAFETENSORS_MAX_SHARDS; i++) {
+ snprintf(path, sizeof(path), "%s/model-%05d-of-%05d.safetensors",
+ model_dir, i, i); /* placeholder - need to detect count */
+ /* We don't know the total count yet, try common patterns */
+ break;
+ }
+
+ /* Scan directory for shard files */
+ DIR *dir = opendir(model_dir);
+ if (!dir) { free(ms); return NULL; }
+
+ struct dirent *entry;
+ char shard_names[SAFETENSORS_MAX_SHARDS][256];
+ int n_shards = 0;
+
+ while ((entry = readdir(dir)) != NULL && n_shards < SAFETENSORS_MAX_SHARDS) {
+ if (strncmp(entry->d_name, "model-", 6) == 0 &&
+ strstr(entry->d_name, ".safetensors") != NULL) {
+ snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]),
+ "%s", entry->d_name);
+ n_shards++;
+ }
+ }
+ closedir(dir);
+
+ if (n_shards == 0) {
+ fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
+ free(ms);
+ return NULL;
+ }
+
+ /* Sort shard names to ensure consistent ordering */
+ qsort(shard_names, n_shards, 256, (int(*)(const void*,const void*))strcmp);
+
+ /* Open each shard */
+ for (int i = 0; i < n_shards; i++) {
+ snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
+ ms->shards[i] = safetensors_open(path);
+ if (!ms->shards[i]) {
+ fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
+ multi_safetensors_close(ms);
+ return NULL;
+ }
+ }
+ ms->num_shards = n_shards;
+ return ms;
+}
+
+void multi_safetensors_close(multi_safetensors_t *ms) {
+ if (!ms) return;
+ for (int i = 0; i < ms->num_shards; i++) {
+ safetensors_close(ms->shards[i]);
+ }
+ free(ms);
+}
+
+const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
+ const char *name,
+ safetensors_file_t **out_sf) {
+ for (int s = 0; s < ms->num_shards; s++) {
+ safetensors_file_t *sf = ms->shards[s];
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0) {
+ if (out_sf) *out_sf = sf;
+ return &sf->tensors[i];
+ }
+ }
+ }
+ return NULL;
+}
diff --git a/qwen_asr_safetensors.h b/qwen_asr_safetensors.h
new file mode 100644
index 0000000..4cb09f7
--- /dev/null
+++ b/qwen_asr_safetensors.h
@@ -0,0 +1,77 @@
+/*
+ * qwen_asr_safetensors.h - Safetensors file format reader (multi-shard support)
+ * Adapted from voxtral-realtime project.
+ */
+
+#ifndef QWEN_ASR_SAFETENSORS_H
+#define QWEN_ASR_SAFETENSORS_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define SAFETENSORS_MAX_TENSORS 1024
+#define SAFETENSORS_MAX_SHARDS 8
+
+typedef enum {
+ DTYPE_F32 = 0,
+ DTYPE_F16 = 1,
+ DTYPE_BF16 = 2,
+ DTYPE_I32 = 3,
+ DTYPE_I64 = 4,
+ DTYPE_BOOL = 5,
+ DTYPE_UNKNOWN = -1
+} safetensor_dtype_t;
+
+typedef struct {
+ char name[256];
+ safetensor_dtype_t dtype;
+ int ndim;
+ int64_t shape[8];
+ size_t data_offset;
+ size_t data_size;
+} safetensor_t;
+
+typedef struct {
+ char *path;
+ void *data;
+ size_t file_size;
+ size_t header_size;
+ char *header_json;
+ int num_tensors;
+ safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
+} safetensors_file_t;
+
+/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
+typedef struct {
+ safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
+ int num_shards;
+} multi_safetensors_t;
+
+/* Open a single safetensors file (memory-mapped) */
+safetensors_file_t *safetensors_open(const char *path);
+void safetensors_close(safetensors_file_t *sf);
+
+/* Open model from directory (auto-detects single file or multi-shard) */
+multi_safetensors_t *multi_safetensors_open(const char *model_dir);
+void multi_safetensors_close(multi_safetensors_t *ms);
+
+/* Find a tensor by name across all shards */
+const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
+ const char *name,
+ safetensors_file_t **out_sf);
+
+/* Get raw pointer to tensor data (within mmap'd region) */
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
+
+/* Get tensor data as float32 (allocates, caller must free) */
+float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t);
+
+/* Get direct pointer to bf16 data in mmap'd region (no copy) */
+uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t);
+
+int safetensor_is_bf16(const safetensor_t *t);
+int64_t safetensor_numel(const safetensor_t *t);
+void safetensor_print(const safetensor_t *t);
+void safetensors_print_all(const safetensors_file_t *sf);
+
+#endif /* QWEN_ASR_SAFETENSORS_H */
diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c
new file mode 100644
index 0000000..7bbc5db
--- /dev/null
+++ b/qwen_asr_tokenizer.c
@@ -0,0 +1,650 @@
+/*
+ * qwen_asr_tokenizer.c - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
+ *
+ * Supports:
+ * - decode token IDs to UTF-8 text (for ASR output assembly)
+ * - encode UTF-8 text to token IDs using vocab.json + merges.txt
+ */
+
+#include "qwen_asr_tokenizer.h"
+#include "qwen_asr_kernels.h"
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ========================================================================
+ * GPT-2 Bytes-to-Unicode Mapping
+ * ======================================================================== */
+
+static int gpt2_byte_to_unicode[256];
+static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */
+static int gpt2_mapping_initialized = 0;
+
+static void init_gpt2_mapping(void) {
+ if (gpt2_mapping_initialized) return;
+
+ memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte));
+
+ int n = 0;
+ for (int b = 0; b < 256; b++) {
+ int is_normal = 0;
+ if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */
+ if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */
+ if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */
+
+ if (is_normal) gpt2_byte_to_unicode[b] = b;
+ else gpt2_byte_to_unicode[b] = 256 + n++;
+ }
+
+ for (int b = 0; b < 256; b++) {
+ int cp = gpt2_byte_to_unicode[b];
+ if (cp < 512) gpt2_unicode_to_byte[cp] = b;
+ }
+
+ gpt2_mapping_initialized = 1;
+}
+
+/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */
+static int utf8_encode_cp(int cp, char out[4]) {
+ if (cp < 0x80) {
+ out[0] = (char)cp;
+ return 1;
+ }
+ if (cp < 0x800) {
+ out[0] = (char)(0xC0 | (cp >> 6));
+ out[1] = (char)(0x80 | (cp & 0x3F));
+ return 2;
+ }
+ out[0] = (char)(0xE0 | (cp >> 12));
+ out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
+ out[2] = (char)(0x80 | (cp & 0x3F));
+ return 3;
+}
+
+/*
+ * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text.
+ * Returns allocated string (caller must free).
+ */
+static char *decode_gpt2_token(const char *token_str) {
+ init_gpt2_mapping();
+
+ size_t len = strlen(token_str);
+ unsigned char *bytes = (unsigned char *)malloc(len + 1);
+ if (!bytes) return NULL;
+
+ int byte_count = 0;
+ const unsigned char *p = (const unsigned char *)token_str;
+ const unsigned char *end = p + len;
+
+ while (p < end) {
+ int cp;
+ int nbytes;
+
+ if ((*p & 0x80) == 0) {
+ cp = *p;
+ nbytes = 1;
+ } else if ((*p & 0xE0) == 0xC0) {
+ cp = (*p & 0x1F) << 6;
+ if (p + 1 < end) cp |= (p[1] & 0x3F);
+ nbytes = 2;
+ } else if ((*p & 0xF0) == 0xE0) {
+ cp = (*p & 0x0F) << 12;
+ if (p + 1 < end) cp |= (p[1] & 0x3F) << 6;
+ if (p + 2 < end) cp |= (p[2] & 0x3F);
+ nbytes = 3;
+ } else if ((*p & 0xF8) == 0xF0) {
+ cp = (*p & 0x07) << 18;
+ if (p + 1 < end) cp |= (p[1] & 0x3F) << 12;
+ if (p + 2 < end) cp |= (p[2] & 0x3F) << 6;
+ if (p + 3 < end) cp |= (p[3] & 0x3F);
+ nbytes = 4;
+ } else {
+ cp = *p;
+ nbytes = 1;
+ }
+ p += nbytes;
+
+ if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) {
+ bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp];
+ } else {
+ bytes[byte_count++] = '?';
+ }
+ }
+ bytes[byte_count] = '\0';
+ return (char *)bytes;
+}
+
+/* ========================================================================
+ * Simple JSON parser for vocab.json
+ * ======================================================================== */
+
+static void skip_ws(const char **p) {
+ while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++;
+}
+
+static int parse_json_string(const char **p, char *out, size_t max_len) {
+ skip_ws(p);
+ if (**p != '"') return -1;
+ (*p)++;
+ size_t i = 0;
+ while (**p && **p != '"' && i < max_len - 1) {
+ if (**p == '\\') {
+ (*p)++;
+ if (**p == 'n') out[i++] = '\n';
+ else if (**p == 't') out[i++] = '\t';
+ else if (**p == '"') out[i++] = '"';
+ else if (**p == '\\') out[i++] = '\\';
+ else if (**p == '/') out[i++] = '/';
+ else if (**p == 'u') {
+ (*p)++;
+ unsigned int cp = 0;
+ for (int j = 0; j < 4 && **p; j++, (*p)++) {
+ cp <<= 4;
+ char c = **p;
+ if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0');
+ else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10);
+ else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10);
+ }
+ if (cp < 0x80 && i + 1 < max_len) {
+ out[i++] = (char)cp;
+ } else if (cp < 0x800 && i + 2 < max_len) {
+ out[i++] = (char)(0xC0 | (cp >> 6));
+ out[i++] = (char)(0x80 | (cp & 0x3F));
+ } else if (i + 3 < max_len) {
+ out[i++] = (char)(0xE0 | (cp >> 12));
+ out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F));
+ out[i++] = (char)(0x80 | (cp & 0x3F));
+ }
+ continue;
+ } else {
+ out[i++] = **p;
+ }
+ } else {
+ out[i++] = **p;
+ }
+ (*p)++;
+ }
+ out[i] = '\0';
+ if (**p != '"') return -1;
+ (*p)++;
+ return 0;
+}
+
+static int64_t parse_json_int(const char **p) {
+ skip_ws(p);
+ int neg = 0;
+ if (**p == '-') { neg = 1; (*p)++; }
+ int64_t val = 0;
+ while (**p >= '0' && **p <= '9') {
+ val = val * 10 + (**p - '0');
+ (*p)++;
+ }
+ return neg ? -val : val;
+}
+
+/* ========================================================================
+ * String->int hash map (open addressing)
+ * ======================================================================== */
+
+typedef struct {
+ char *key; /* NULL means empty slot */
+ int value;
+} str_int_entry_t;
+
+static uint64_t fnv1a_hash(const char *s) {
+ uint64_t h = 1469598103934665603ULL;
+ while (*s) {
+ h ^= (unsigned char)*s++;
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+static int next_pow2(int x) {
+ int p = 1;
+ while (p < x) p <<= 1;
+ return p;
+}
+
+static int map_insert(str_int_entry_t *map, int cap, char *key, int value) {
+ if (!map || cap <= 0 || !key) return -1;
+ int mask = cap - 1;
+ int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
+ for (int i = 0; i < cap; i++) {
+ int idx = (pos + i) & mask;
+ if (!map[idx].key) {
+ map[idx].key = key;
+ map[idx].value = value;
+ return 0;
+ }
+ if (strcmp(map[idx].key, key) == 0) {
+ map[idx].value = value;
+ return 0;
+ }
+ }
+ return -1;
+}
+
+static int map_get(const str_int_entry_t *map, int cap, const char *key) {
+ if (!map || cap <= 0 || !key) return -1;
+ int mask = cap - 1;
+ int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
+ for (int i = 0; i < cap; i++) {
+ int idx = (pos + i) & mask;
+ if (!map[idx].key) return -1;
+ if (strcmp(map[idx].key, key) == 0) return map[idx].value;
+ }
+ return -1;
+}
+
+/* ========================================================================
+ * BPE helpers
+ * ======================================================================== */
+
+static int utf8_char_len(unsigned char c) {
+ if ((c & 0x80) == 0) return 1;
+ if ((c & 0xE0) == 0xC0) return 2;
+ if ((c & 0xF0) == 0xE0) return 3;
+ if ((c & 0xF8) == 0xF0) return 4;
+ return 1;
+}
+
+static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) {
+ *out_syms = NULL;
+ *out_n = 0;
+ if (!s || !*s) return 0;
+
+ int n = 0;
+ for (const unsigned char *p = (const unsigned char *)s; *p; ) {
+ int l = utf8_char_len(*p);
+ p += l;
+ n++;
+ }
+ if (n <= 0) return 0;
+
+ char **syms = (char **)calloc((size_t)n, sizeof(char *));
+ if (!syms) return -1;
+
+ int i = 0;
+ for (const unsigned char *p = (const unsigned char *)s; *p; ) {
+ int l = utf8_char_len(*p);
+ syms[i] = (char *)malloc((size_t)l + 1);
+ if (!syms[i]) {
+ for (int j = 0; j < i; j++) free(syms[j]);
+ free(syms);
+ return -1;
+ }
+ memcpy(syms[i], p, (size_t)l);
+ syms[i][l] = '\0';
+ p += l;
+ i++;
+ }
+
+ *out_syms = syms;
+ *out_n = n;
+ return 0;
+}
+
+static char *str_concat2(const char *a, const char *b) {
+ size_t la = strlen(a), lb = strlen(b);
+ char *r = (char *)malloc(la + lb + 1);
+ if (!r) return NULL;
+ memcpy(r, a, la);
+ memcpy(r + la, b, lb);
+ r[la + lb] = '\0';
+ return r;
+}
+
+static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) {
+ if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX;
+
+ size_t la = strlen(a), lb = strlen(b);
+ char *pair = (char *)malloc(la + 1 + lb + 1);
+ if (!pair) return INT_MAX;
+ memcpy(pair, a, la);
+ pair[la] = ' ';
+ memcpy(pair + la + 1, b, lb);
+ pair[la + 1 + lb] = '\0';
+
+ int rank = map_get((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, pair);
+ free(pair);
+ return rank >= 0 ? rank : INT_MAX;
+}
+
+static int append_id(int **arr, int *n, int *cap, int id) {
+ if (*n >= *cap) {
+ int new_cap = (*cap == 0) ? 16 : (*cap * 2);
+ int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int));
+ if (!tmp) return -1;
+ *arr = tmp;
+ *cap = new_cap;
+ }
+ (*arr)[(*n)++] = id;
+ return 0;
+}
+
+/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */
+static char *text_to_bpe_unicode(const char *text) {
+ init_gpt2_mapping();
+ size_t len = strlen(text);
+ /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */
+ char *out = (char *)malloc(len * 3 + 1);
+ if (!out) return NULL;
+
+ size_t w = 0;
+ for (size_t i = 0; i < len; i++) {
+ int cp = gpt2_byte_to_unicode[(unsigned char)text[i]];
+ char tmp[4];
+ int n = utf8_encode_cp(cp, tmp);
+ for (int j = 0; j < n; j++) out[w++] = tmp[j];
+ }
+ out[w] = '\0';
+ return out;
+}
+
+/* Encode one mapped BPE unicode string to token IDs. */
+static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) {
+ *out_ids = NULL;
+ *out_n = 0;
+ if (!mapped || !*mapped) return 0;
+
+ char **syms = NULL;
+ int n_syms = 0;
+ if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1;
+ if (n_syms <= 0) {
+ free(syms);
+ return 0;
+ }
+
+ while (n_syms > 1) {
+ int best_rank = INT_MAX;
+ int best_i = -1;
+ for (int i = 0; i < n_syms - 1; i++) {
+ int r = merge_rank(tok, syms[i], syms[i + 1]);
+ if (r < best_rank) {
+ best_rank = r;
+ best_i = i;
+ }
+ }
+ if (best_i < 0 || best_rank == INT_MAX) break;
+
+ char *merged = str_concat2(syms[best_i], syms[best_i + 1]);
+ if (!merged) {
+ for (int i = 0; i < n_syms; i++) free(syms[i]);
+ free(syms);
+ return -1;
+ }
+ free(syms[best_i]);
+ free(syms[best_i + 1]);
+ syms[best_i] = merged;
+ for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1];
+ n_syms--;
+ }
+
+ int *ids = NULL;
+ int n_ids = 0, cap = 0;
+ for (int i = 0; i < n_syms; i++) {
+ int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]);
+ if (id < 0) {
+ /* Should not happen with valid vocab + merges + byte-level mapping. */
+ for (int k = 0; k < n_syms; k++) free(syms[k]);
+ free(syms);
+ free(ids);
+ return -1;
+ }
+ if (append_id(&ids, &n_ids, &cap, id) != 0) {
+ for (int k = 0; k < n_syms; k++) free(syms[k]);
+ free(syms);
+ free(ids);
+ return -1;
+ }
+ }
+
+ for (int i = 0; i < n_syms; i++) free(syms[i]);
+ free(syms);
+
+ *out_ids = ids;
+ *out_n = n_ids;
+ return 0;
+}
+
+static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) {
+ const char *slash = strrchr(vocab_path, '/');
+ if (!slash) {
+ if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1;
+ return 0;
+ }
+ size_t dir_len = (size_t)(slash - vocab_path);
+ if (dir_len + 12 >= out_cap) return -1;
+ memcpy(out_path, vocab_path, dir_len);
+ out_path[dir_len] = '\0';
+ snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt");
+ return 0;
+}
+
+static void trim_newline(char *line) {
+ size_t n = strlen(line);
+ while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0';
+}
+
+static int parse_merge_pair(char *line, char **a, char **b) {
+ *a = NULL;
+ *b = NULL;
+ trim_newline(line);
+ if (line[0] == '\0') return -1;
+ if (line[0] == '#') return -1;
+
+ char *sp = strchr(line, ' ');
+ if (!sp) return -1;
+ *sp = '\0';
+ char *p2 = sp + 1;
+ while (*p2 == ' ') p2++;
+ if (*line == '\0' || *p2 == '\0') return -1;
+ *a = line;
+ *b = p2;
+ return 0;
+}
+
+static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
+ FILE *f = fopen(merges_path, "rb");
+ if (!f) return -1;
+
+ int n_pairs = 0;
+ char line[8192];
+ while (fgets(line, sizeof(line), f)) {
+ char *a = NULL, *b = NULL;
+ if (parse_merge_pair(line, &a, &b) == 0) n_pairs++;
+ }
+ if (n_pairs <= 0) {
+ fclose(f);
+ return -1;
+ }
+
+ tok->merge_map_cap = next_pow2(n_pairs * 2);
+ tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t));
+ if (!tok->merge_map) {
+ fclose(f);
+ tok->merge_map_cap = 0;
+ return -1;
+ }
+
+ rewind(f);
+ int rank = 0;
+ while (fgets(line, sizeof(line), f)) {
+ char *a = NULL, *b = NULL;
+ if (parse_merge_pair(line, &a, &b) != 0) continue;
+
+ size_t la = strlen(a), lb = strlen(b);
+ char *key = (char *)malloc(la + 1 + lb + 1);
+ if (!key) continue;
+ memcpy(key, a, la);
+ key[la] = ' ';
+ memcpy(key + la + 1, b, lb);
+ key[la + 1 + lb] = '\0';
+
+ if (map_insert((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, key, rank) != 0) {
+ free(key);
+ }
+ rank++;
+ }
+ fclose(f);
+ return 0;
+}
+
+/* ========================================================================
+ * Public API
+ * ======================================================================== */
+
+qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
+ FILE *f = fopen(vocab_json_path, "rb");
+ if (!f) {
+ fprintf(stderr, "qwen_tokenizer_load: cannot open %s\n", vocab_json_path);
+ return NULL;
+ }
+
+ fseek(f, 0, SEEK_END);
+ long file_size = ftell(f);
+ fseek(f, 0, SEEK_SET);
+
+ char *json = (char *)malloc((size_t)file_size + 1);
+ if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) {
+ fclose(f);
+ free(json);
+ return NULL;
+ }
+ fclose(f);
+ json[file_size] = '\0';
+
+ int max_id = 0;
+ const char *p = json;
+ skip_ws(&p);
+ if (*p != '{') {
+ free(json);
+ return NULL;
+ }
+ p++;
+
+ const char *saved = p;
+ while (*p && *p != '}') {
+ skip_ws(&p);
+ if (*p == ',') { p++; continue; }
+ if (*p == '}') break;
+
+ char key[4096];
+ if (parse_json_string(&p, key, sizeof(key)) != 0) break;
+ skip_ws(&p);
+ if (*p != ':') break;
+ p++;
+ int64_t id = parse_json_int(&p);
+ if (id > max_id) max_id = (int)id;
+ }
+
+ int vocab_size = max_id + 1;
+ qwen_tokenizer_t *tok = (qwen_tokenizer_t *)calloc(1, sizeof(qwen_tokenizer_t));
+ if (!tok) {
+ free(json);
+ return NULL;
+ }
+ tok->vocab_size = vocab_size;
+ tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *));
+ tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *));
+ if (!tok->id_to_text || !tok->id_to_bpe) {
+ qwen_tokenizer_free(tok);
+ free(json);
+ return NULL;
+ }
+
+ p = saved;
+ while (*p && *p != '}') {
+ skip_ws(&p);
+ if (*p == ',') { p++; continue; }
+ if (*p == '}') break;
+
+ char key[4096];
+ if (parse_json_string(&p, key, sizeof(key)) != 0) break;
+ skip_ws(&p);
+ if (*p != ':') break;
+ p++;
+ int64_t id = parse_json_int(&p);
+
+ if (id >= 0 && id < vocab_size) {
+ free(tok->id_to_bpe[id]);
+ free(tok->id_to_text[id]);
+ tok->id_to_bpe[id] = strdup(key);
+ tok->id_to_text[id] = decode_gpt2_token(key);
+ }
+ }
+ free(json);
+
+ int n_vocab_entries = 0;
+ for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++;
+ tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1);
+ tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t));
+ if (!tok->vocab_map) {
+ qwen_tokenizer_free(tok);
+ return NULL;
+ }
+ for (int i = 0; i < vocab_size; i++) {
+ if (tok->id_to_bpe[i]) {
+ map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i);
+ }
+ }
+
+ char merges_path[1024];
+ if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) {
+ if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) {
+ fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n",
+ merges_path);
+ }
+ }
+
+ return tok;
+}
+
+const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) {
+ if (!tok || token_id < 0 || token_id >= tok->vocab_size) return "";
+ return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : "";
+}
+
+int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) {
+ if (out_n_tokens) *out_n_tokens = 0;
+ if (!tok || !text || text[0] == '\0') return NULL;
+
+ char *mapped = text_to_bpe_unicode(text);
+ if (!mapped) return NULL;
+
+ int *ids = NULL;
+ int n_ids = 0;
+ if (encode_bpe_word(tok, mapped, &ids, &n_ids) != 0) {
+ free(mapped);
+ free(ids);
+ return NULL;
+ }
+ free(mapped);
+
+ if (out_n_tokens) *out_n_tokens = n_ids;
+ return ids;
+}
+
+void qwen_tokenizer_free(qwen_tokenizer_t *tok) {
+ if (!tok) return;
+
+ if (tok->id_to_text) {
+ for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]);
+ free(tok->id_to_text);
+ }
+ if (tok->id_to_bpe) {
+ for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]);
+ free(tok->id_to_bpe);
+ }
+
+ if (tok->merge_map) {
+ str_int_entry_t *m = (str_int_entry_t *)tok->merge_map;
+ for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key);
+ free(tok->merge_map);
+ }
+ free(tok->vocab_map);
+ free(tok);
+}
diff --git a/qwen_asr_tokenizer.h b/qwen_asr_tokenizer.h
new file mode 100644
index 0000000..0cea192
--- /dev/null
+++ b/qwen_asr_tokenizer.h
@@ -0,0 +1,38 @@
+/*
+ * qwen_asr_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
+ *
+ * Supports:
+ * - decode token ID -> UTF-8 text
+ * - encode UTF-8 text -> token IDs (vocab.json + merges.txt)
+ */
+
+#ifndef QWEN_ASR_TOKENIZER_H
+#define QWEN_ASR_TOKENIZER_H
+
+typedef struct {
+ char **id_to_text; /* [vocab_size] decoded text strings */
+ char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
+ int vocab_size;
+
+ /* Internal hash maps (opaque to callers) */
+ void *vocab_map;
+ int vocab_map_cap;
+ void *merge_map;
+ int merge_map_cap;
+} qwen_tokenizer_t;
+
+/* Load tokenizer from vocab.json in model directory */
+qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
+
+/* Decode a single token ID to text. Returns pointer to internal string. */
+const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
+
+/* Encode UTF-8 text into token IDs using BPE.
+ * Returns malloc'd array of token IDs and sets *out_n_tokens.
+ * Returns NULL on error (and sets *out_n_tokens to 0). */
+int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+
+/* Free tokenizer */
+void qwen_tokenizer_free(qwen_tokenizer_t *tok);
+
+#endif /* QWEN_ASR_TOKENIZER_H */