From 34e9eefde7870edcc318209738cabb17c71b50ed Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 28 Mar 2026 20:01:21 +0100 Subject: Init --- Makefile | 128 +++++ main.c | 363 +++++++++++++ pplx_embed.c | 447 ++++++++++++++++ pplx_embed.h | 186 +++++++ pplx_embed_mlx.c | 465 +++++++++++++++++ pplx_embed_mlx.h | 35 ++ qwen_asr_kernels.c | 1247 ++++++++++++++++++++++++++++++++++++++++++++ qwen_asr_kernels.h | 173 ++++++ qwen_asr_kernels_avx.c | 502 ++++++++++++++++++ qwen_asr_kernels_generic.c | 65 +++ qwen_asr_kernels_impl.h | 65 +++ qwen_asr_kernels_neon.c | 274 ++++++++++ qwen_asr_safetensors.c | 394 ++++++++++++++ qwen_asr_safetensors.h | 77 +++ qwen_asr_tokenizer.c | 650 +++++++++++++++++++++++ qwen_asr_tokenizer.h | 38 ++ 16 files changed, 5109 insertions(+) create mode 100644 Makefile create mode 100644 main.c create mode 100644 pplx_embed.c create mode 100644 pplx_embed.h create mode 100644 pplx_embed_mlx.c create mode 100644 pplx_embed_mlx.h create mode 100644 qwen_asr_kernels.c create mode 100644 qwen_asr_kernels.h create mode 100644 qwen_asr_kernels_avx.c create mode 100644 qwen_asr_kernels_generic.c create mode 100644 qwen_asr_kernels_impl.h create mode 100644 qwen_asr_kernels_neon.c create mode 100644 qwen_asr_safetensors.c create mode 100644 qwen_asr_safetensors.h create mode 100644 qwen_asr_tokenizer.c create mode 100644 qwen_asr_tokenizer.h diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ee6b813 --- /dev/null +++ b/Makefile @@ -0,0 +1,128 @@ +# pplx_embed - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b +# Makefile + +CC = gcc +CFLAGS_BASE = -Wall -Wextra -O3 -march=native -ffast-math +LDFLAGS = -lm -lpthread + +UNAME_S := $(shell uname -s) + +# Source files +SRCS = pplx_embed.c \ + qwen_asr_kernels.c \ + qwen_asr_kernels_generic.c \ + qwen_asr_kernels_neon.c \ + qwen_asr_kernels_avx.c \ + qwen_asr_tokenizer.c \ + qwen_asr_safetensors.c + +MLX_SRCS = pplx_embed_mlx.c + +OBJS = $(SRCS:.c=.o) +MLX_OBJS = $(MLX_SRCS:.c=.o) +TARGET = pplx_embed + +.PHONY: all blas mlx debug clean help + +all: help + +help: + @echo "pplx_embed - Pure C inference for pplx-embed-v1-0.6b" + @echo "" + @echo "Build targets:" + @echo " make blas Build with BLAS acceleration (CPU)" + @echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)" + @echo " make debug Debug build with AddressSanitizer" + @echo " make clean Remove build artifacts" + @echo "" + @echo "Usage after build:" + @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\"" + @echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)" + +# ============================================================================= +# BLAS build (Apple Accelerate on macOS, OpenBLAS on Linux) +# ============================================================================= +ifeq ($(UNAME_S),Darwin) +blas: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK +blas: LDFLAGS += -framework Accelerate +else +blas: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -I/usr/include/openblas +blas: LDFLAGS += -lopenblas +endif +blas: + $(MAKE) clean + $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" + @echo "" + @echo "Built: $(TARGET) (CPU/BLAS)" + +# ============================================================================= +# MLX build (Apple Silicon GPU - uses mlx-c pure C API) +# Requires: brew install mlx mlx-c +# ============================================================================= +MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null) +MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null) + +mlx: SRCS += pplx_embed_mlx.c +mlx: OBJS = $(SRCS:.c=.o) pplx_embed_mlx.o +ifeq ($(UNAME_S),Darwin) +mlx: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX \ + -I$(MLXC_PREFIX)/include +mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \ + -L$(MLXC_PREFIX)/lib -lmlxc \ + -L$(MLX_PREFIX)/lib -lmlx \ + -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib +endif +mlx: + $(MAKE) clean + $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" + @echo "" + @echo "Built: $(TARGET) (CPU/BLAS + MLX GPU, use --mlx flag)" + +# ============================================================================= +# Debug build +# ============================================================================= +debug: CFLAGS = -Wall -Wextra -g -O0 -DDEBUG -fsanitize=address +debug: LDFLAGS += -fsanitize=address +debug: + $(MAKE) clean + $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" + +# ============================================================================= +# Link +# ============================================================================= +$(TARGET): $(OBJS) $(EXTRA_OBJS) main.o + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) + +# ============================================================================= +# Compile rules +# ============================================================================= +pplx_embed.o: pplx_embed.c pplx_embed.h qwen_asr_kernels.h qwen_asr_safetensors.h + $(CC) $(CFLAGS) -c -o $@ $< + +pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h + $(CC) $(CFLAGS) -c -o $@ $< + +main.o: main.c pplx_embed.h qwen_asr_kernels.h qwen_asr_tokenizer.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_kernels_generic.o: qwen_asr_kernels_generic.c qwen_asr_kernels_impl.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_kernels_neon.o: qwen_asr_kernels_neon.c qwen_asr_kernels_impl.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_kernels_avx.o: qwen_asr_kernels_avx.c qwen_asr_kernels_impl.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_tokenizer.o: qwen_asr_tokenizer.c qwen_asr_tokenizer.h qwen_asr_kernels.h + $(CC) $(CFLAGS) -c -o $@ $< + +qwen_asr_safetensors.o: qwen_asr_safetensors.c qwen_asr_safetensors.h + $(CC) $(CFLAGS) -c -o $@ $< + +# ============================================================================= +clean: + rm -f $(OBJS) pplx_embed_mlx.o main.o $(TARGET) diff --git a/main.c b/main.c new file mode 100644 index 0000000..cc2e492 --- /dev/null +++ b/main.c @@ -0,0 +1,363 @@ +/* + * main.c - pplx_embed command-line tool + * + * Usage: + * pplx_embed -d [options] [ ...] + * pplx_embed -d [options] (read lines from stdin) + * + * Options: + * -d Model directory (required) + * -t Number of threads (default: all CPU cores) + * -e Print raw embedding vector(s) instead of similarity matrix + * -v Verbose output (use -vv for debug) + * + * Output: + * 1 text : prints the 1024-dim embedding as space-separated floats + * 2+ texts: prints cosine-similarity matrix between every pair, + * then optionally the individual embeddings if -e is given + * + * Example: + * ./pplx_embed -d /path/to/model "hello world" "hi there" + * ./pplx_embed -d /path/to/model -e "query: what is AI?" + */ + +#include "pplx_embed.h" +#include "qwen_asr_kernels.h" +#include "qwen_asr_tokenizer.h" + +#ifdef USE_MLX +#include "pplx_embed_mlx.h" +#endif + +#include +#include +#include +#include +#include + +/* ======================================================================== + * Helpers + * ======================================================================== */ + +static double now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; +} + +static void print_embedding(const float *emb, int dim) +{ + for (int i = 0; i < dim; i++) { + if (i > 0) putchar(' '); + printf("%.8f", (double)emb[i]); + } + putchar('\n'); +} + +static void print_usage(const char *prog) +{ + fprintf(stderr, + "Usage: %s -d [options] [text...]\n" + "\n" + "Options:\n" + " -d Model directory (model.safetensors + vocab.json + merges.txt)\n" + " -t Number of threads (default: all CPU cores, CPU backend)\n" + " --mlx Use Apple MLX GPU backend (Apple Silicon only)\n" + " -e Always print raw embedding vector(s)\n" + " -v Verbose (repeat for more: -vv)\n" + " -h Show this help\n" + "\n" + "With 1 text: prints the 1024-dim embedding vector\n" + "With 2+ texts: prints cosine similarity between each pair\n" + " (add -e to also print the individual embeddings)\n" + "\n" + "If no text arguments are given, reads one text per line from stdin.\n" + "\n" + "Example:\n" + " %s -d ./model \"query: what is AI?\" \"document: AI is artificial intelligence\"\n" + " %s -d ./model --mlx \"query: what is AI?\"\n", + prog, prog, prog); +} + +/* ======================================================================== + * Process one text: tokenize -> embed, returns malloc'd float[dim] + * ======================================================================== */ + +static float *process_text(pplx_ctx_t *ctx, +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx_ctx, +#endif + qwen_tokenizer_t *tok, + const char *text, + int print_tokens) +{ + /* Tokenize */ + int n_tokens = 0; + int *token_ids = qwen_tokenizer_encode(tok, text, &n_tokens); + if (!token_ids || n_tokens == 0) { + fprintf(stderr, "pplx_embed: tokenization failed for: %s\n", text); + free(token_ids); + return NULL; + } + + if (print_tokens || pplx_verbose >= 1) { + fprintf(stderr, "tokens (%d): ", n_tokens); + for (int i = 0; i < n_tokens && i < 20; i++) + fprintf(stderr, "%d ", token_ids[i]); + if (n_tokens > 20) fprintf(stderr, "..."); + fprintf(stderr, "\n"); + } + + /* Embed */ + double t0 = now_ms(); + float *emb = NULL; +#ifdef USE_MLX + if (mlx_ctx) + emb = pplx_mlx_embed(mlx_ctx, token_ids, n_tokens); + else +#endif + emb = pplx_embed(ctx, token_ids, n_tokens); + double dt = now_ms() - t0; + free(token_ids); + + if (!emb) { + fprintf(stderr, "pplx_embed: forward pass failed\n"); + return NULL; + } + + if (pplx_verbose >= 1) + fprintf(stderr, "embed: %d tokens in %.1f ms\n", n_tokens, dt); + + return emb; +} + +/* ======================================================================== + * Main + * ======================================================================== */ + +int main(int argc, char *argv[]) +{ + const char *model_dir = NULL; + int n_threads = 0; /* 0 = auto */ + int print_embs = 0; + int verbose = 0; + int use_mlx = 0; + + /* ---- Parse flags ---- */ + int arg_start = 1; + while (arg_start < argc && argv[arg_start][0] == '-') { + const char *flag = argv[arg_start]; + if (strcmp(flag, "-d") == 0) { + if (arg_start + 1 >= argc) { + fprintf(stderr, "pplx_embed: -d requires an argument\n"); + return 1; + } + model_dir = argv[++arg_start]; + } else if (strcmp(flag, "-t") == 0) { + if (arg_start + 1 >= argc) { + fprintf(stderr, "pplx_embed: -t requires an argument\n"); + return 1; + } + n_threads = atoi(argv[++arg_start]); + } else if (strcmp(flag, "-e") == 0) { + print_embs = 1; + } else if (strcmp(flag, "--mlx") == 0) { +#ifdef USE_MLX + use_mlx = 1; +#else + fprintf(stderr, "pplx_embed: --mlx not available (build with: make mlx)\n"); + return 1; +#endif + } else if (strcmp(flag, "-v") == 0) { + verbose++; + } else if (strcmp(flag, "-vv") == 0) { + verbose = 2; + } else if (strcmp(flag, "-h") == 0 || strcmp(flag, "--help") == 0) { + print_usage(argv[0]); + return 0; + } else { + /* Stop flag parsing on first non-flag argument */ + break; + } + arg_start++; + } + + if (!model_dir || model_dir[0] == '\0') { + fprintf(stderr, "pplx_embed: model directory required (-d )\n" + " Hint: if using MODEL=.. ./pplx_embed -d \"$MODEL\",\n" + " make sure MODEL is exported or use: MODEL=.. && ./pplx_embed -d \"$MODEL\"\n"); + print_usage(argv[0]); + return 1; + } + + /* ---- Set verbosity ---- */ + pplx_verbose = verbose; + qwen_verbose = verbose; /* kernels share the same flag name */ + + /* ---- Set thread count (CPU backend only) ---- */ + if (!use_mlx) { + if (n_threads <= 0) + n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) + fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else { + if (verbose >= 1) + fprintf(stderr, "Using MLX GPU backend\n"); + } + + /* ---- Load tokenizer ---- */ + char vocab_path[1024]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); + + double t0_load = now_ms(); + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + if (!tok) { + fprintf(stderr, "pplx_embed: failed to load tokenizer from %s\n", vocab_path); + return 1; + } + if (verbose >= 1) + fprintf(stderr, "Tokenizer loaded in %.0f ms\n", now_ms() - t0_load); + + /* ---- Load model ---- */ + pplx_ctx_t *ctx = NULL; +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx_ctx = NULL; +#endif + + double t0_model = now_ms(); + +#ifdef USE_MLX + if (use_mlx) { + mlx_ctx = pplx_mlx_load(model_dir); + if (!mlx_ctx) { + fprintf(stderr, "pplx_embed: failed to load MLX model from %s\n", model_dir); + qwen_tokenizer_free(tok); + return 1; + } + } else +#endif + { + ctx = pplx_load(model_dir); + if (!ctx) { + fprintf(stderr, "pplx_embed: failed to load model from %s\n", model_dir); + qwen_tokenizer_free(tok); + return 1; + } + } + if (verbose >= 1) + fprintf(stderr, "Model loaded in %.0f ms%s\n", now_ms() - t0_model, + use_mlx ? " (MLX)" : ""); + + int dim = PPLX_HIDDEN_SIZE; + + /* ---- Gather input texts ---- */ + /* texts[] and embeddings[] are grown dynamically */ + int n_texts = 0; + int texts_cap = 0; + char **texts = NULL; + float **embeddings = NULL; + + /* Helper to append a text */ +#define APPEND_TEXT(s) do { \ + if (n_texts == texts_cap) { \ + texts_cap = texts_cap ? texts_cap * 2 : 8; \ + texts = realloc(texts, texts_cap * sizeof(char *)); \ + embeddings = realloc(embeddings, texts_cap * sizeof(float *)); \ + if (!texts || !embeddings) { \ + fprintf(stderr, "pplx_embed: out of memory\n"); return 1; \ + } \ + } \ + texts[n_texts] = strdup(s); \ + embeddings[n_texts] = NULL; \ + n_texts++; \ +} while (0) + + if (arg_start < argc) { + /* Text arguments on command line */ + for (int i = arg_start; i < argc; i++) + APPEND_TEXT(argv[i]); + } else { + /* Read lines from stdin */ + char line[65536]; + while (fgets(line, sizeof(line), stdin)) { + /* Strip trailing newline */ + size_t l = strlen(line); + while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) + line[--l] = '\0'; + if (l == 0) continue; + APPEND_TEXT(line); + } + } + + if (n_texts == 0) { + fprintf(stderr, "pplx_embed: no input texts\n"); + pplx_free(ctx); + qwen_tokenizer_free(tok); + return 1; + } + + /* ---- Embed all texts ---- */ + for (int i = 0; i < n_texts; i++) { + if (verbose >= 1) + fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); + + embeddings[i] = process_text(ctx, +#ifdef USE_MLX + mlx_ctx, +#endif + tok, texts[i], + /*print_tokens=*/ verbose >= 2); + if (!embeddings[i]) { + /* Error already printed */ + goto cleanup; + } + } + + /* ---- Output ---- */ + if (n_texts == 1) { + /* Single text: always print the embedding */ + print_embedding(embeddings[0], dim); + } else { + /* Multiple texts: print cosine similarity matrix */ + printf("Cosine similarity matrix (%d texts):\n", n_texts); + + /* Header */ + printf("%-4s", ""); + for (int j = 0; j < n_texts; j++) printf(" [%d] ", j); + printf("\n"); + + for (int i = 0; i < n_texts; i++) { + printf("[%d] ", i); + for (int j = 0; j < n_texts; j++) { + float sim = pplx_cosine_similarity(embeddings[i], embeddings[j], dim); + printf(" %.4f", (double)sim); + } + printf(" \"%s\"\n", texts[i]); + } + + /* Print raw embeddings if requested */ + if (print_embs) { + printf("\nEmbeddings:\n"); + for (int i = 0; i < n_texts; i++) { + printf("[%d] ", i); + print_embedding(embeddings[i], dim); + } + } + } + +cleanup: + for (int i = 0; i < n_texts; i++) { + free(texts[i]); + free(embeddings[i]); + } + free(texts); + free(embeddings); + if (ctx) pplx_free(ctx); +#ifdef USE_MLX + if (mlx_ctx) pplx_mlx_free(mlx_ctx); +#endif + qwen_tokenizer_free(tok); + return 0; +} diff --git a/pplx_embed.c b/pplx_embed.c new file mode 100644 index 0000000..2220760 --- /dev/null +++ b/pplx_embed.c @@ -0,0 +1,447 @@ +/* + * pplx_embed.c - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b + * + * Transformer forward pass: + * embed -> 28x[RMSNorm -> QKV (no bias) -> Q/K RMSNorm -> RoPE + * -> Bidirectional GQA attention -> O proj -> residual + * -> RMSNorm -> SwiGLU MLP -> residual] + * -> final RMSNorm -> mean pool -> L2 normalize + * + * Key implementation choices: + * - All weights are F32 (mmap'd direct, no copy) + * - Bidirectional attention: reuse qwen_causal_attention with + * q_offset = (seq_len - 1), which removes the causal mask + * (every query sees all seq_len keys) + * - GQA ratio 2:1 (16 Q-heads, 8 KV-heads) handled by causal_attention + * - RoPE cache grown lazily, positions [0 .. seq_len-1] + */ + +#include "pplx_embed.h" +#include "qwen_asr_kernels.h" +#include "qwen_asr_safetensors.h" + +#include +#include +#include +#include + +/* ======================================================================== + * Global verbose flags + * qwen_verbose is declared extern in qwen_asr_kernels.h and qwen_asr_tokenizer.c; + * we own its definition here since we don't link qwen_asr.c. + * ======================================================================== */ + +int pplx_verbose = 0; +int qwen_verbose = 0; /* required by qwen_asr_kernels.c and qwen_asr_tokenizer.c */ + +/* ======================================================================== + * Internal helpers: weight loading + * ======================================================================== */ + +/* + * Return a direct pointer into the mmap'd safetensors region for an F32 tensor. + * No allocation, no copy; caller must NOT free the returned pointer. + */ +static const float *load_f32_direct(multi_safetensors_t *ms, const char *name) +{ + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "pplx_embed: tensor not found: %s\n", name); + return NULL; + } + if (t->dtype != DTYPE_F32) { + fprintf(stderr, "pplx_embed: expected F32 for %s, got dtype=%d\n", + name, (int)t->dtype); + return NULL; + } + return (const float *)safetensors_data(sf, t); +} + +/* ======================================================================== + * Internal helpers: working buffers + * ======================================================================== */ + +static int ensure_buffers(pplx_ctx_t *ctx, int seq) +{ + if (seq <= ctx->buf_seq_cap) return 0; + + int new_cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64; + while (new_cap < seq) new_cap *= 2; + + int hidden = ctx->config.hidden_size; + int q_dim = ctx->config.q_dim; + int kv_dim = ctx->config.kv_dim; + int inter = ctx->config.intermediate_size; + +#define PPLX_REALLOC(ptr, n) do { \ + void *_tmp = realloc((ptr), (size_t)(n) * sizeof(float)); \ + if (!_tmp) return -1; \ + (ptr) = (float *)_tmp; \ +} while (0) + + PPLX_REALLOC(ctx->x, new_cap * hidden); + PPLX_REALLOC(ctx->x_norm, new_cap * hidden); + PPLX_REALLOC(ctx->q, new_cap * q_dim); + PPLX_REALLOC(ctx->k, new_cap * kv_dim); + PPLX_REALLOC(ctx->v, new_cap * kv_dim); + PPLX_REALLOC(ctx->attn_out, new_cap * q_dim); + PPLX_REALLOC(ctx->proj_out, new_cap * hidden); + PPLX_REALLOC(ctx->ffn_gate, new_cap * inter); + PPLX_REALLOC(ctx->ffn_up, new_cap * inter); + PPLX_REALLOC(ctx->ffn_out, new_cap * hidden); + +#undef PPLX_REALLOC + + ctx->buf_seq_cap = new_cap; + return 0; +} + +/* ======================================================================== + * Internal helpers: RoPE cache + * + * Stores cos/sin for positions [0 .. rope_cache_cap-1]. + * Layout: [pos, head_dim] - values are duplicated across the two halves + * so that qwen_apply_rope_neox can load contiguous cos[d] and cos[half+d] + * from a single row: cos_row[d] == cos_row[half+d]. + * ======================================================================== */ + +static int ensure_rope_cache(pplx_ctx_t *ctx, int n_pos) +{ + if (n_pos <= ctx->rope_cache_cap) return 0; + + int new_cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512; + while (new_cap < n_pos) new_cap *= 2; + + int head_dim = ctx->config.head_dim; + int half = head_dim / 2; + float theta = ctx->config.rope_theta; + + size_t n = (size_t)new_cap * head_dim; + float *new_cos = (float *)realloc(ctx->rope_cos, n * sizeof(float)); + float *new_sin = (float *)realloc(ctx->rope_sin, n * sizeof(float)); + if (!new_cos || !new_sin) { + /* If realloc failed, partial state is still valid up to old cap */ + if (new_cos) ctx->rope_cos = new_cos; + if (new_sin) ctx->rope_sin = new_sin; + return -1; + } + ctx->rope_cos = new_cos; + ctx->rope_sin = new_sin; + + /* Compute only the newly needed positions */ + for (int pos = ctx->rope_cache_cap; pos < new_cap; pos++) { + float fp = (float)pos; + float *c = ctx->rope_cos + (size_t)pos * head_dim; + float *s = ctx->rope_sin + (size_t)pos * head_dim; + for (int d = 0; d < half; d++) { + /* inv_freq[d] = 1 / (theta ^ (2d / head_dim)) */ + float inv_freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = fp * inv_freq; + float cv = cosf(angle); + float sv = sinf(angle); + /* Duplicate across both halves for the NeoX rotation kernel */ + c[d] = cv; + c[half + d] = cv; + s[d] = sv; + s[half + d] = sv; + } + } + + ctx->rope_cache_cap = new_cap; + return 0; +} + +/* ======================================================================== + * Load + * ======================================================================== */ + +pplx_ctx_t *pplx_load(const char *model_dir) +{ + pplx_ctx_t *ctx = (pplx_ctx_t *)calloc(1, sizeof(pplx_ctx_t)); + if (!ctx) return NULL; + + /* ---- Hardcoded config (mirrors config.json) ---- */ + pplx_config_t *cfg = &ctx->config; + cfg->hidden_size = PPLX_HIDDEN_SIZE; + cfg->n_layers = PPLX_NUM_LAYERS; + cfg->n_heads = PPLX_NUM_HEADS; + cfg->n_kv_heads = PPLX_NUM_KV_HEADS; + cfg->head_dim = PPLX_HEAD_DIM; + cfg->q_dim = PPLX_Q_DIM; + cfg->kv_dim = PPLX_KV_DIM; + cfg->intermediate_size = PPLX_INTERMEDIATE_SIZE; + cfg->vocab_size = PPLX_VOCAB_SIZE; + cfg->rms_norm_eps = 1e-6f; + cfg->rope_theta = 1000000.0f; + + /* ---- Open safetensors ---- */ + multi_safetensors_t *ms = multi_safetensors_open(model_dir); + if (!ms) { + fprintf(stderr, "pplx_load: failed to open safetensors in %s\n", model_dir); + free(ctx); + return NULL; + } + ctx->safetensors = ms; + + if (pplx_verbose >= 1) + fprintf(stderr, "pplx_load: loading weights from %s\n", model_dir); + + /* ---- Token embeddings ---- */ + pplx_weights_t *w = &ctx->weights; + w->embed_tokens = load_f32_direct(ms, "embed_tokens.weight"); + if (!w->embed_tokens) goto fail; + + /* ---- Per-layer weights ---- */ + char name[256]; + for (int i = 0; i < cfg->n_layers; i++) { + pplx_layer_t *l = &w->layers[i]; + +#define LOAD(field, fmt) do { \ + snprintf(name, sizeof(name), fmt, i); \ + l->field = load_f32_direct(ms, name); \ + if (!l->field) goto fail; \ +} while (0) + + LOAD(wq, "layers.%d.self_attn.q_proj.weight"); + LOAD(wk, "layers.%d.self_attn.k_proj.weight"); + LOAD(wv, "layers.%d.self_attn.v_proj.weight"); + LOAD(wo, "layers.%d.self_attn.o_proj.weight"); + LOAD(q_norm, "layers.%d.self_attn.q_norm.weight"); + LOAD(k_norm, "layers.%d.self_attn.k_norm.weight"); + LOAD(input_norm, "layers.%d.input_layernorm.weight"); + LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight"); + LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight"); + LOAD(up_proj, "layers.%d.mlp.up_proj.weight"); + LOAD(down_proj, "layers.%d.mlp.down_proj.weight"); + +#undef LOAD + + if (pplx_verbose >= 2) + fprintf(stderr, " layer %d loaded\n", i); + } + + /* ---- Final RMSNorm ---- */ + w->norm = load_f32_direct(ms, "norm.weight"); + if (!w->norm) goto fail; + + if (pplx_verbose >= 1) + fprintf(stderr, "pplx_load: all weights loaded\n"); + + return ctx; + +fail: + pplx_free(ctx); + return NULL; +} + +/* ======================================================================== + * Free + * ======================================================================== */ + +void pplx_free(pplx_ctx_t *ctx) +{ + if (!ctx) return; + if (ctx->safetensors) + multi_safetensors_close((multi_safetensors_t *)ctx->safetensors); + + free(ctx->x); + free(ctx->x_norm); + free(ctx->q); + free(ctx->k); + free(ctx->v); + free(ctx->attn_out); + free(ctx->proj_out); + free(ctx->ffn_gate); + free(ctx->ffn_up); + free(ctx->ffn_out); + free(ctx->rope_cos); + free(ctx->rope_sin); + free(ctx); +} + +/* ======================================================================== + * Forward pass + * ======================================================================== */ + +float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + const pplx_config_t *cfg = &ctx->config; + const pplx_weights_t *w = &ctx->weights; + + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; + int n_kv_heads = cfg->n_kv_heads; + int head_dim = cfg->head_dim; + int q_dim = cfg->q_dim; + int kv_dim = cfg->kv_dim; + int inter = cfg->intermediate_size; + float eps = cfg->rms_norm_eps; + int seq = n_tokens; + + /* Allocate/grow working buffers */ + if (ensure_buffers(ctx, seq) != 0) { + fprintf(stderr, "pplx_embed: buffer allocation failed\n"); + return NULL; + } + + /* Pre-compute RoPE tables for positions [0, seq-1] */ + if (ensure_rope_cache(ctx, seq) != 0) { + fprintf(stderr, "pplx_embed: RoPE cache allocation failed\n"); + return NULL; + } + const float *rope_cos = ctx->rope_cos; /* [seq_cap, head_dim] */ + const float *rope_sin = ctx->rope_sin; + + float *x = ctx->x; + float *x_norm = ctx->x_norm; + float *q = ctx->q; + float *k = ctx->k; + float *v = ctx->v; + float *attn_out = ctx->attn_out; + float *proj_out = ctx->proj_out; + float *ffn_gate = ctx->ffn_gate; + float *ffn_up = ctx->ffn_up; + float *ffn_out = ctx->ffn_out; + + /* ------------------------------------------------------------------ + * Step 1: Token embedding lookup + * x[i, :] = embed_tokens[token_ids[i], :] + * ------------------------------------------------------------------ */ + for (int i = 0; i < seq; i++) { + int id = token_ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "pplx_embed: invalid token id %d at position %d\n", id, i); + return NULL; + } + memcpy(x + i * hidden, + w->embed_tokens + (size_t)id * hidden, + hidden * sizeof(float)); + } + + float scale = 1.0f / sqrtf((float)head_dim); + + /* ------------------------------------------------------------------ + * Steps 2..29: 28 transformer layers + * ------------------------------------------------------------------ */ + for (int layer = 0; layer < cfg->n_layers; layer++) { + const pplx_layer_t *l = &w->layers[layer]; + + /* ---- 2a. Input RMSNorm ---- */ + qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps); + + /* ---- 2b. QKV projections (no bias) ---- + * q: [seq, q_dim] = x_norm @ wq^T (wq is [q_dim, hidden]) + * k: [seq, kv_dim] = x_norm @ wk^T + * v: [seq, kv_dim] = x_norm @ wv^T + */ + qwen_linear_nobias(q, x_norm, l->wq, seq, hidden, q_dim); + qwen_linear_nobias(k, x_norm, l->wk, seq, hidden, kv_dim); + qwen_linear_nobias(v, x_norm, l->wv, seq, hidden, kv_dim); + + /* ---- 2c. Per-head Q/K RMSNorm (Qwen3 feature) ---- + * Normalize each head's [head_dim] segment independently. + */ + qwen_rms_norm_per_head(q, l->q_norm, seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(k, l->k_norm, seq, n_kv_heads, head_dim, eps); + + /* ---- 2d. Apply NeoX-style RoPE to Q and K ---- + * rope_cos/sin: [seq_cap, head_dim], positions 0..seq-1 cached + */ + qwen_apply_rope_neox(q, rope_cos, rope_sin, seq, n_heads, head_dim); + qwen_apply_rope_neox(k, rope_cos, rope_sin, seq, n_kv_heads, head_dim); + + /* ---- 2e. Bidirectional GQA attention ---- + * + * We reuse qwen_causal_attention with q_offset = (seq - 1). + * + * Inside the kernel for query position i: + * global_pos = q_offset + i = (seq-1) + i + * k_end = min(global_pos + 1, seq_k) + * = min(seq + i, seq) = seq for all i >= 0 + * + * So every query attends to ALL seq key positions - full attention, + * no causal mask, while still benefiting from the GQA + threading + * in qwen_causal_attention. + */ + qwen_causal_attention(attn_out, q, k, v, + seq, /* seq_q */ + seq, /* seq_k */ + n_heads, n_kv_heads, head_dim, + scale, + seq - 1); /* q_offset - removes causal mask */ + + /* ---- 2f. Output projection + residual ---- + * proj_out = attn_out @ wo^T (wo is [hidden, q_dim]) + * x += proj_out + */ + qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden); + qwen_add_inplace(x, proj_out, seq * hidden); + + /* ---- 2g. Post-attention RMSNorm ---- */ + qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps); + + /* ---- 2h. SwiGLU MLP ---- + * gate = gate_proj(x_norm) [seq, intermediate] + * up = up_proj(x_norm) [seq, intermediate] + * mid = SiLU(gate) * up [seq, intermediate] + * out = down_proj(mid) [seq, hidden] + */ + qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter); + qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter); + + qwen_silu(ffn_gate, seq * inter); /* SiLU(gate) in-place */ + qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); /* gate *= up */ + + qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden); + qwen_add_inplace(x, ffn_out, seq * hidden); + } + + /* ------------------------------------------------------------------ + * Step 3: Final RMSNorm (in-place on x) + * ------------------------------------------------------------------ */ + qwen_rms_norm(x, x, w->norm, seq, hidden, eps); + + /* ------------------------------------------------------------------ + * Step 4: Mean pooling across all token positions + * embedding[d] = mean over i in [0, seq) of x[i, d] + * ------------------------------------------------------------------ */ + float *embedding = (float *)calloc(hidden, sizeof(float)); + if (!embedding) return NULL; + + for (int i = 0; i < seq; i++) { + const float *row = x + i * hidden; + for (int d = 0; d < hidden; d++) { + embedding[d] += row[d]; + } + } + float inv_seq = 1.0f / (float)seq; + for (int d = 0; d < hidden; d++) embedding[d] *= inv_seq; + + /* ------------------------------------------------------------------ + * Step 5: L2 normalization + * embedding /= ||embedding||_2 + * ------------------------------------------------------------------ */ + float norm_sq = 0.0f; + for (int d = 0; d < hidden; d++) norm_sq += embedding[d] * embedding[d]; + float norm_val = sqrtf(norm_sq); + if (norm_val > 1e-8f) { + float inv = 1.0f / norm_val; + for (int d = 0; d < hidden; d++) embedding[d] *= inv; + } + + return embedding; +} + +/* ======================================================================== + * Cosine similarity (dot product of two L2-normalized vectors) + * ======================================================================== */ + +float pplx_cosine_similarity(const float *a, const float *b, int dim) +{ + float dot = 0.0f; + for (int i = 0; i < dim; i++) dot += a[i] * b[i]; + return dot; +} diff --git a/pplx_embed.h b/pplx_embed.h new file mode 100644 index 0000000..69b67c3 --- /dev/null +++ b/pplx_embed.h @@ -0,0 +1,186 @@ +/* + * pplx_embed.h - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b + * + * Architecture: Qwen3-family transformer with BIDIRECTIONAL (non-causal) + * attention. Based on antirez's qwen-asr kernels. Produces dense float32 + * embedding vectors via mean-pooling + L2-normalization. + * + * Model spec: + * hidden_size = 1024 + * num_hidden_layers = 28 + * num_attention_heads = 16 (query heads) + * num_key_value_heads = 8 (GQA: 2 query heads share 1 KV head) + * head_dim = 128 + * intermediate_size = 3072 + * vocab_size = 151936 + * rms_norm_eps = 1e-6 + * rope_theta = 1000000.0 (1e6) + * weights dtype = F32 (all weights float32) + * tie_word_embeddings = true + * use_bidirectional_attention = true + */ + +#ifndef PPLX_EMBED_H +#define PPLX_EMBED_H + +#include +#include + +/* ======================================================================== + * Constants + * ======================================================================== */ + +#define PPLX_HIDDEN_SIZE 1024 +#define PPLX_NUM_LAYERS 28 +#define PPLX_NUM_HEADS 16 +#define PPLX_NUM_KV_HEADS 8 +#define PPLX_HEAD_DIM 128 +#define PPLX_Q_DIM (PPLX_NUM_HEADS * PPLX_HEAD_DIM) /* 2048 */ +#define PPLX_KV_DIM (PPLX_NUM_KV_HEADS * PPLX_HEAD_DIM) /* 1024 */ +#define PPLX_INTERMEDIATE_SIZE 3072 +#define PPLX_VOCAB_SIZE 151936 + +/* ======================================================================== + * Model Configuration + * ======================================================================== */ + +typedef struct { + int hidden_size; /* 1024 */ + int n_layers; /* 28 */ + int n_heads; /* 16 */ + int n_kv_heads; /* 8 */ + int head_dim; /* 128 */ + int q_dim; /* n_heads * head_dim = 2048 */ + int kv_dim; /* n_kv_heads * head_dim = 1024 */ + int intermediate_size; /* 3072 */ + int vocab_size; /* 151936 */ + float rms_norm_eps; /* 1e-6 */ + float rope_theta; /* 1e6 */ +} pplx_config_t; + +/* ======================================================================== + * Per-Layer Weights (all F32, direct mmap pointers - read-only) + * + * Tensor shapes (row-major): + * wq [q_dim, hidden] = [2048, 1024] + * wk [kv_dim, hidden] = [1024, 1024] + * wv [kv_dim, hidden] = [1024, 1024] + * wo [hidden, q_dim] = [1024, 2048] + * q_norm [head_dim] = [128] + * k_norm [head_dim] = [128] + * input_norm [hidden] = [1024] + * post_attn_norm [hidden] = [1024] + * gate_proj [intermediate, hidden] = [3072, 1024] + * up_proj [intermediate, hidden] = [3072, 1024] + * down_proj [hidden, intermediate] = [1024, 3072] + * ======================================================================== */ + +typedef struct { + /* Self-attention projections (no bias) */ + const float *wq; /* [q_dim, hidden] */ + const float *wk; /* [kv_dim, hidden] */ + const float *wv; /* [kv_dim, hidden] */ + const float *wo; /* [hidden, q_dim] */ + + /* Per-head Q/K RMSNorm weights (Qwen3 feature) */ + const float *q_norm; /* [head_dim] */ + const float *k_norm; /* [head_dim] */ + + /* Layer norms (no bias - RMSNorm only) */ + const float *input_norm; /* [hidden] pre-attention */ + const float *post_attn_norm;/* [hidden] pre-MLP */ + + /* SwiGLU MLP (no bias) */ + const float *gate_proj; /* [intermediate, hidden] */ + const float *up_proj; /* [intermediate, hidden] */ + const float *down_proj; /* [hidden, intermediate] */ +} pplx_layer_t; + +/* ======================================================================== + * Full Model Weights + * ======================================================================== */ + +typedef struct { + /* Token embedding table (tied with output head, but we don't need lm_head) */ + const float *embed_tokens; /* [vocab_size, hidden] */ + + /* Transformer layers */ + pplx_layer_t layers[PPLX_NUM_LAYERS]; + + /* Final RMSNorm */ + const float *norm; /* [hidden] */ +} pplx_weights_t; + +/* ======================================================================== + * Context (model + working buffers) + * ======================================================================== */ + +typedef struct { + pplx_config_t config; + pplx_weights_t weights; + + /* Open safetensors handle (keeps mmap alive) */ + void *safetensors; /* multi_safetensors_t* */ + + /* Working buffers, grown lazily to buf_seq_cap */ + int buf_seq_cap; + float *x; /* [seq, hidden] main activations */ + float *x_norm; /* [seq, hidden] after RMSNorm */ + float *q; /* [seq, q_dim] query vectors */ + float *k; /* [seq, kv_dim] key vectors */ + float *v; /* [seq, kv_dim] value vectors */ + float *attn_out; /* [seq, q_dim] attention output */ + float *proj_out; /* [seq, hidden] after o_proj */ + float *ffn_gate; /* [seq, intermediate] gate branch */ + float *ffn_up; /* [seq, intermediate] up branch */ + float *ffn_out; /* [seq, hidden] after down_proj */ + + /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily */ + float *rope_cos; + float *rope_sin; + int rope_cache_cap; /* number of cached positions */ +} pplx_ctx_t; + +/* ======================================================================== + * API + * ======================================================================== */ + +/* + * Load the model from a directory containing: + * model.safetensors - weight tensors (F32) + * vocab.json - BPE vocabulary + * merges.txt - BPE merge rules + * + * Returns NULL on error. + */ +pplx_ctx_t *pplx_load(const char *model_dir); + +/* Free all resources */ +void pplx_free(pplx_ctx_t *ctx); + +/* + * Compute the embedding for a sequence of token IDs. + * + * Performs: + * 1. Token embedding lookup + * 2. 28 × bidirectional transformer layers (GQA, RoPE, SwiGLU) + * 3. Final RMSNorm + * 4. Mean pooling across all token positions + * 5. L2 normalization + * + * Returns a malloc'd float[hidden_size] (caller must free). + * Returns NULL on allocation error or empty input. + */ +float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); + +/* + * Cosine similarity between two L2-normalized embedding vectors. + * (This is just the dot product since both are unit vectors.) + */ +float pplx_cosine_similarity(const float *a, const float *b, int dim); + +/* Verbose level: 0=quiet, 1=info, 2=debug */ +extern int pplx_verbose; +extern int qwen_verbose; /* same flag, used internally by kernel/tokenizer */ + +#endif /* PPLX_EMBED_H */ diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c new file mode 100644 index 0000000..d6637cd --- /dev/null +++ b/pplx_embed_mlx.c @@ -0,0 +1,465 @@ +/* + * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference + * + * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference. + * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm. + * + * Key operations used: + * mlx_take - embedding lookup + * mlx_matmul - linear projections (no bias) + * mlx_fast_rms_norm - RMSNorm (fused Metal kernel) + * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel) + * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel) + * sigmoid * gate, multiply - SwiGLU activation + */ + +#include "pplx_embed_mlx.h" +#include "pplx_embed.h" /* config constants */ +#include "qwen_asr_safetensors.h" /* safetensors reader */ + +#include + +#include +#include +#include +#include + +/* ======================================================================== + * Per-layer weight arrays (MLX) + * ======================================================================== */ + +typedef struct { + mlx_array wq; /* [q_dim, hidden] */ + mlx_array wk; /* [kv_dim, hidden] */ + mlx_array wv; /* [kv_dim, hidden] */ + mlx_array wo; /* [hidden, q_dim] */ + mlx_array q_norm; /* [head_dim] */ + mlx_array k_norm; /* [head_dim] */ + mlx_array input_norm; /* [hidden] */ + mlx_array post_attn_norm; /* [hidden] */ + mlx_array gate_proj; /* [intermediate, hidden] */ + mlx_array up_proj; /* [intermediate, hidden] */ + mlx_array down_proj; /* [hidden, intermediate] */ +} mlx_layer_t; + +struct pplx_mlx_ctx { + mlx_array embed_tokens; /* [vocab, hidden] */ + mlx_layer_t layers[PPLX_NUM_LAYERS]; + mlx_array norm; /* [hidden] */ + mlx_stream stream; + + /* Safetensors kept alive for mmap */ + multi_safetensors_t *ms; +}; + +/* ======================================================================== + * Helpers + * ======================================================================== */ + +/* Default GPU stream */ +static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; } + +/* + * Load an F32 tensor from safetensors into an mlx_array. + * The data is copied into MLX (which will move it to GPU as needed). + */ +static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, + const int *shape, int ndim) +{ + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "mlx: tensor not found: %s\n", name); + return (mlx_array){0}; + } + if (t->dtype != DTYPE_F32) { + fprintf(stderr, "mlx: expected F32 for %s\n", name); + return (mlx_array){0}; + } + const void *data = safetensors_data(sf, t); + return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32); +} + +/* Quick check: is arr valid? */ +static int arr_ok(mlx_array a) { return a.ctx != NULL; } + +/* ======================================================================== + * Load + * ======================================================================== */ + +pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) +{ + pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); + if (!ctx) return NULL; + + ctx->stream = mlx_default_gpu_stream_new(); + + /* Open safetensors */ + ctx->ms = multi_safetensors_open(model_dir); + if (!ctx->ms) { + fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); + free(ctx); + return NULL; + } + + multi_safetensors_t *ms = ctx->ms; + + /* Embedding table: [vocab_size, hidden] */ + int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE}; + ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) goto fail; + + /* Per-layer weights */ + char name[256]; + for (int i = 0; i < PPLX_NUM_LAYERS; i++) { + mlx_layer_t *l = &ctx->layers[i]; + int hidden = PPLX_HIDDEN_SIZE; + int q_dim = PPLX_Q_DIM; + int kv_dim = PPLX_KV_DIM; + int inter = PPLX_INTERMEDIATE_SIZE; + int hdim = PPLX_HEAD_DIM; + +#define LOAD(field, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + int nd = sizeof(sh) / sizeof(sh[0]); \ + l->field = load_tensor(ms, name, sh, nd); \ + if (!arr_ok(l->field)) goto fail; \ +} while (0) + + LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden); + LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden); + LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden); + LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim); + LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim); + LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim); + LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden); + LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden); + LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden); + LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden); + LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter); + +#undef LOAD + + if (pplx_verbose >= 2) + fprintf(stderr, " mlx layer %d loaded\n", i); + } + + /* Final norm */ + int norm_shape[] = {PPLX_HIDDEN_SIZE}; + ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1); + if (!arr_ok(ctx->norm)) goto fail; + + if (pplx_verbose >= 1) + fprintf(stderr, "mlx: all weights loaded\n"); + + return ctx; + +fail: + pplx_mlx_free(ctx); + return NULL; +} + +/* ======================================================================== + * Free + * ======================================================================== */ + +static void free_layer(mlx_layer_t *l) +{ + mlx_array_free(l->wq); + mlx_array_free(l->wk); + mlx_array_free(l->wv); + mlx_array_free(l->wo); + mlx_array_free(l->q_norm); + mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); + mlx_array_free(l->post_attn_norm); + mlx_array_free(l->gate_proj); + mlx_array_free(l->up_proj); + mlx_array_free(l->down_proj); +} + +void pplx_mlx_free(pplx_mlx_ctx_t *ctx) +{ + if (!ctx) return; + mlx_array_free(ctx->embed_tokens); + for (int i = 0; i < PPLX_NUM_LAYERS; i++) + free_layer(&ctx->layers[i]); + mlx_array_free(ctx->norm); + mlx_stream_free(ctx->stream); + if (ctx->ms) multi_safetensors_close(ctx->ms); + free(ctx); +} + +/* ======================================================================== + * Forward pass (all on GPU stream, lazy-evaluated) + * + * Notation: S = gpu_stream(ctx) + * + * Linear without bias: y = x @ W^T + * Since mlx_matmul(x, W^T) requires transposing W, and our weights + * are stored as [out, in], we do: y = x @ W^T via + * Wt = transpose(W) [in, out] + * y = matmul(x, Wt) [seq, out] + * + * Alternatively: y = matmul(x, transpose(W)) + * All MLX ops are lazy (graph-built) so transpose is free. + * ======================================================================== */ + +/* Convenience: transpose a 2D matrix */ +static mlx_array t2d(mlx_array a, mlx_stream s) +{ + mlx_array res = mlx_array_new(); + mlx_transpose(&res, a, s); + return res; +} + +/* y = x @ W^T (linear, no bias) */ +static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s) +{ + mlx_array Wt = t2d(W, s); + mlx_array res = mlx_array_new(); + mlx_matmul(&res, x, Wt, s); + mlx_array_free(Wt); + return res; +} + +float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + mlx_stream S = gpu_stream(ctx); + int hidden = PPLX_HIDDEN_SIZE; + int n_heads = PPLX_NUM_HEADS; + int n_kv_heads = PPLX_NUM_KV_HEADS; + int head_dim = PPLX_HEAD_DIM; + int q_dim = PPLX_Q_DIM; + int seq = n_tokens; + float scale = 1.0f / sqrtf((float)head_dim); + + (void)n_heads; (void)n_kv_heads; /* used only in shape literals */ + + /* ------------------------------------------------------------------ + * 1. Token embedding lookup: x = embed_tokens[token_ids] + * ------------------------------------------------------------------ */ + int ids_shape[] = {seq}; + mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32); + mlx_array x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S); + mlx_array_free(ids); + /* x: [seq, hidden] */ + + /* ------------------------------------------------------------------ + * 2. Transformer layers (28x) + * ------------------------------------------------------------------ */ + for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) { + mlx_layer_t *l = &ctx->layers[layer]; + + /* ---- 2a. Input RMSNorm ---- */ + mlx_array xn = mlx_array_new(); + mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S); + + /* ---- 2b. QKV projections (no bias) ---- + * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim] + */ + mlx_array q_flat = linear(xn, l->wq, S); + mlx_array k_flat = linear(xn, l->wk, S); + mlx_array v_flat = linear(xn, l->wv, S); + mlx_array_free(xn); + + /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */ + int q_shape[] = {seq, n_heads, head_dim}; + int k_shape[] = {seq, n_kv_heads, head_dim}; + + mlx_array q = mlx_array_new(); + mlx_array k = mlx_array_new(); + mlx_array v = mlx_array_new(); + mlx_reshape(&q, q_flat, q_shape, 3, S); + mlx_reshape(&k, k_flat, k_shape, 3, S); + mlx_reshape(&v, v_flat, k_shape, 3, S); + mlx_array_free(q_flat); + mlx_array_free(k_flat); + mlx_array_free(v_flat); + + /* ---- 2d. Per-head Q/K RMSNorm ---- + * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim) + */ + mlx_array qn = mlx_array_new(); + mlx_array kn = mlx_array_new(); + mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S); + mlx_array_free(q); + mlx_array_free(k); + + /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ---- + * MLX rope expects (..., T, D) where T is second-to-last. + * [seq, n_heads, head_dim] + * - expand_dims(0) [1, seq, n_heads, head_dim] + * - transpose [0,2,1,3] [1, n_heads, seq, head_dim] + */ + int perm[] = {0, 2, 1, 3}; + + mlx_array qe = mlx_array_new(); + mlx_array ke = mlx_array_new(); + mlx_array ve = mlx_array_new(); + mlx_expand_dims(&qe, qn, 0, S); + mlx_expand_dims(&ke, kn, 0, S); + mlx_expand_dims(&ve, v, 0, S); + mlx_array_free(qn); + mlx_array_free(kn); + + mlx_array qt = mlx_array_new(); + mlx_array kt = mlx_array_new(); + mlx_array vt = mlx_array_new(); + mlx_transpose_axes(&qt, qe, perm, 4, S); + mlx_transpose_axes(&kt, ke, perm, 4, S); + mlx_transpose_axes(&vt, ve, perm, 4, S); + mlx_array_free(qe); + mlx_array_free(ke); + mlx_array_free(ve); + mlx_array_free(v); + + /* ---- 2f. RoPE (NeoX-style: traditional=false) ---- + * qt: [1, n_heads, seq, head_dim] + * kt: [1, n_kv_heads, seq, head_dim] + * MLX rope: T = second-to-last dim = seq + */ + mlx_optional_float base = {.value = 1000000.0f, .has_value = true}; + mlx_array null_freqs = (mlx_array){0}; + + mlx_array qr = mlx_array_new(); + mlx_array kr = mlx_array_new(); + mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false, + base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S); + mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false, + base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S); + mlx_array_free(qt); + mlx_array_free(kt); + + /* ---- 2g. Bidirectional GQA Attention ---- + * qr: [1, n_heads, seq, head_dim] + * kr: [1, n_kv_heads, seq, head_dim] + * vt: [1, n_kv_heads, seq, head_dim] + * mask_mode="" > no causal mask > full bidirectional + * GQA: SDPA broadcasts KV heads to match Q heads automatically. + */ + mlx_array attn = mlx_array_new(); + mlx_array null_mask = (mlx_array){0}; + mlx_array null_sinks = (mlx_array){0}; + mlx_fast_scaled_dot_product_attention( + &attn, qr, kr, vt, scale, + /*mask_mode=*/"", + null_mask, null_sinks, S); + mlx_array_free(qr); + mlx_array_free(kr); + mlx_array_free(vt); + /* attn: [1, n_heads, seq, head_dim] */ + + /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */ + mlx_array attn_t = mlx_array_new(); + mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */ + mlx_array_free(attn); + + mlx_array attn_sq = mlx_array_new(); + mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */ + mlx_array_free(attn_t); + + int attn_flat_shape[] = {seq, q_dim}; + mlx_array attn_flat = mlx_array_new(); + mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S); + mlx_array_free(attn_sq); + + /* ---- 2i. Output projection + residual ---- */ + mlx_array proj = linear(attn_flat, l->wo, S); + mlx_array_free(attn_flat); + + mlx_array x2 = mlx_array_new(); + mlx_add(&x2, x, proj, S); + mlx_array_free(x); + mlx_array_free(proj); + x = x2; + + /* ---- 2j. Post-attention RMSNorm ---- */ + mlx_array xn2 = mlx_array_new(); + mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S); + + /* ---- 2k. SwiGLU MLP ---- + * gate = x_norm @ gate_proj^T [seq, inter] + * up = x_norm @ up_proj^T [seq, inter] + * mid = silu(gate) * up = (gate * sigmoid(gate)) * up + * out = mid @ down_proj^T [seq, hidden] + */ + mlx_array gate = linear(xn2, l->gate_proj, S); + mlx_array up = linear(xn2, l->up_proj, S); + mlx_array_free(xn2); + + /* SiLU(gate) = gate * sigmoid(gate) */ + mlx_array gate_sig = mlx_array_new(); + mlx_sigmoid(&gate_sig, gate, S); + mlx_array silu_gate = mlx_array_new(); + mlx_multiply(&silu_gate, gate, gate_sig, S); + mlx_array_free(gate); + mlx_array_free(gate_sig); + + /* mid = silu_gate * up */ + mlx_array mid = mlx_array_new(); + mlx_multiply(&mid, silu_gate, up, S); + mlx_array_free(silu_gate); + mlx_array_free(up); + + /* down projection */ + mlx_array ffn = linear(mid, l->down_proj, S); + mlx_array_free(mid); + + /* Residual */ + mlx_array x3 = mlx_array_new(); + mlx_add(&x3, x, ffn, S); + mlx_array_free(x); + mlx_array_free(ffn); + x = x3; + } + + /* ------------------------------------------------------------------ + * 3. Final RMSNorm + * ------------------------------------------------------------------ */ + mlx_array x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S); + mlx_array_free(x); + + /* ------------------------------------------------------------------ + * 4. Mean pooling: mean(x, axis=0) [hidden] + * ------------------------------------------------------------------ */ + mlx_array emb = mlx_array_new(); + mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S); + mlx_array_free(x_normed); + + /* ------------------------------------------------------------------ + * 5. L2 normalization: emb / ||emb||_2 + * ------------------------------------------------------------------ */ + int norm_axes[] = {0}; + mlx_array norm_val = mlx_array_new(); + mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S); + + mlx_array emb_normed = mlx_array_new(); + mlx_divide(&emb_normed, emb, norm_val, S); + mlx_array_free(emb); + mlx_array_free(norm_val); + + /* ------------------------------------------------------------------ + * 6. Evaluate and copy result to CPU + * ------------------------------------------------------------------ */ + mlx_array_eval(emb_normed); + + const float *data = mlx_array_data_float32(emb_normed); + if (!data) { + fprintf(stderr, "mlx: failed to get embedding data\n"); + mlx_array_free(emb_normed); + return NULL; + } + + float *out = (float *)malloc(hidden * sizeof(float)); + if (out) memcpy(out, data, hidden * sizeof(float)); + + mlx_array_free(emb_normed); + return out; +} diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h new file mode 100644 index 0000000..5327fd0 --- /dev/null +++ b/pplx_embed_mlx.h @@ -0,0 +1,35 @@ +/* + * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference + * + * Uses Apple's mlx-c to run the transformer on Metal GPU. + * All operations run on the GPU default stream; the result is copied + * back to CPU as a float* embedding. + */ + +#ifndef PPLX_EMBED_MLX_H +#define PPLX_EMBED_MLX_H + +#include +#include + +/* Opaque MLX model context */ +typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; + +/* + * Load model into MLX arrays from safetensors in model_dir. + * Returns NULL on error. + */ +pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); + +/* + * Free all MLX resources. + */ +void pplx_mlx_free(pplx_mlx_ctx_t *ctx); + +/* + * Compute embedding for token_ids[0..n_tokens-1]. + * Returns malloc'd float[1024] (caller frees). NULL on error. + */ +float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); + +#endif /* PPLX_EMBED_MLX_H */ diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c new file mode 100644 index 0000000..c0d9d70 --- /dev/null +++ b/qwen_asr_kernels.c @@ -0,0 +1,1247 @@ +/* + * qwen_asr_kernels.c - Math kernels for Qwen3-ASR inference + * Adapted from voxtral-realtime project. + */ + +#include "qwen_asr_kernels.h" +#include "qwen_asr_kernels_impl.h" +#include +#include +#include +#include +#include +#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#ifdef __APPLE__ +#include +#else +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* ======================================================================== + * Thread Pool + * ======================================================================== */ + +#define QWEN_MAX_THREADS 16 + +typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); + +static struct { + pthread_t threads[QWEN_MAX_THREADS - 1]; + int tids[QWEN_MAX_THREADS - 1]; + int n_threads; + int shutdown; + + parallel_fn_t fn; + void *arg; + int generation; + + pthread_mutex_t mutex; + pthread_cond_t cond_work; + pthread_cond_t cond_done; + int n_done; +} tp = { + .n_threads = 1, + .shutdown = 0, + .generation = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond_work = PTHREAD_COND_INITIALIZER, + .cond_done = PTHREAD_COND_INITIALIZER, +}; + +static void *worker_loop(void *arg) { + int tid = *(int *)arg; + int my_gen = 0; + + for (;;) { + pthread_mutex_lock(&tp.mutex); + while (tp.generation == my_gen && !tp.shutdown) + pthread_cond_wait(&tp.cond_work, &tp.mutex); + if (tp.shutdown) { + pthread_mutex_unlock(&tp.mutex); + return NULL; + } + my_gen = tp.generation; + parallel_fn_t fn = tp.fn; + void *a = tp.arg; + int nt = tp.n_threads; + pthread_mutex_unlock(&tp.mutex); + + fn(tid, nt, a); + + pthread_mutex_lock(&tp.mutex); + if (++tp.n_done >= tp.n_threads - 1) + pthread_cond_signal(&tp.cond_done); + pthread_mutex_unlock(&tp.mutex); + } +} + +void qwen_set_threads(int n) { + if (n < 1) n = 1; + if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS; + + /* Shutdown existing workers */ + if (tp.n_threads > 1) { + pthread_mutex_lock(&tp.mutex); + tp.shutdown = 1; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + for (int i = 0; i < tp.n_threads - 1; i++) + pthread_join(tp.threads[i], NULL); + tp.shutdown = 0; + tp.generation = 0; + } + + tp.n_threads = n; + if (n <= 1) return; + + for (int i = 0; i < n - 1; i++) { + tp.tids[i] = i + 1; + pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); + } + + if (qwen_verbose >= 2) + fprintf(stderr, "Thread pool: %d threads\n", n); +} + +int qwen_get_num_cpus(void) { +#ifdef __APPLE__ + int n = 0; + size_t len = sizeof(n); + sysctlbyname("hw.ncpu", &n, &len, NULL, 0); + return n > 0 ? n : 1; +#else + int n = (int)sysconf(_SC_NPROCESSORS_ONLN); + return n > 0 ? n : 1; +#endif +} + +/* Dispatch work to all threads; main thread is tid=0 */ +static void parallel_for(parallel_fn_t fn, void *arg) { + if (tp.n_threads <= 1) { + fn(0, 1, arg); + return; + } + + pthread_mutex_lock(&tp.mutex); + tp.fn = fn; + tp.arg = arg; + tp.n_done = 0; + tp.generation++; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + + fn(0, tp.n_threads, arg); + + pthread_mutex_lock(&tp.mutex); + while (tp.n_done < tp.n_threads - 1) + pthread_cond_wait(&tp.cond_done, &tp.mutex); + pthread_mutex_unlock(&tp.mutex); +} + +/* ======================================================================== + * Basic Element-wise Operations + * ======================================================================== */ + +void qwen_add_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) a[i] += b[i]; +} + +void qwen_mul_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) a[i] *= b[i]; +} + +void qwen_scale(float *x, float s, int n) { + for (int i = 0; i < n; i++) x[i] *= s; +} + +void qwen_copy(float *dst, const float *src, int n) { + memcpy(dst, src, n * sizeof(float)); +} + +/* ======================================================================== + * Matrix Operations + * ======================================================================== */ + +void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); +#else + for (int m = 0; m < M; m++) { + for (int n = 0; n < N; n++) { + float sum = 0.0f; + for (int k = 0; k < K; k++) { + sum += A[m * K + k] * B[n * K + k]; + } + C[m * N + n] = sum; + } + } +#endif +} + +void qwen_linear(float *y, const float *x, const float *W, const float *b, + int seq_len, int in_dim, int out_dim) { +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + seq_len, out_dim, in_dim, + 1.0f, x, in_dim, W, in_dim, + 0.0f, y, out_dim); + if (b != NULL) { + for (int s = 0; s < seq_len; s++) { + for (int o = 0; o < out_dim; o++) { + y[s * out_dim + o] += b[o]; + } + } + } +#else + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * in_dim; + float *y_row = y + s * out_dim; + for (int o = 0; o < out_dim; o++) { + const float *w_row = W + o * in_dim; + float sum = (b != NULL) ? b[o] : 0.0f; + for (int i = 0; i < in_dim; i++) { + sum += x_row[i] * w_row[i]; + } + y_row[o] = sum; + } + } +#endif +} + +void qwen_linear_nobias(float *y, const float *x, const float *W, + int seq_len, int in_dim, int out_dim) { + qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim); +} + +/* Convert bf16 buffer to f32 buffer */ +static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { + uint32_t *d = (uint32_t *)(void *)dst; + for (size_t i = 0; i < n; i++) + d[i] = ((uint32_t)src[i]) << 16; +} + +/* Reusable scratch buffer for bf16->f32 conversion */ +static float *bf16_scratch = NULL; +static size_t bf16_scratch_cap = 0; + +static float *bf16_get_scratch(size_t n) { + if (n > bf16_scratch_cap) { + free(bf16_scratch); + bf16_scratch = (float *)malloc(n * sizeof(float)); + bf16_scratch_cap = bf16_scratch ? n : 0; + } + return bf16_scratch; +} + +typedef struct { + const uint16_t *src; + size_t n; + float *dst_f32; +} bf16_cache_entry_t; + +static bf16_cache_entry_t *bf16_cache = NULL; +static int bf16_cache_len = 0; +static int bf16_cache_cap = 0; +static size_t bf16_cache_bytes = 0; +static size_t bf16_cache_limit_bytes = 0; +static int bf16_cache_limit_init = 0; + +static void bf16_cache_init_limit(void) { + if (bf16_cache_limit_init) return; + bf16_cache_limit_init = 1; + + /* Default OFF. Override with QWEN_BF16_CACHE_MB= to enable. */ + unsigned long long mb = 0; + const char *env = getenv("QWEN_BF16_CACHE_MB"); + if (env && env[0] != '\0') { + char *end = NULL; + unsigned long long v = strtoull(env, &end, 10); + if (end != env) mb = v; + } + bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL); + + if (qwen_verbose >= 2) { + fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb); + } +} + +static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) { + bf16_cache_init_limit(); + + for (int i = 0; i < bf16_cache_len; i++) { + if (bf16_cache[i].src == src && bf16_cache[i].n == n) { + return bf16_cache[i].dst_f32; + } + } + + if (bf16_cache_limit_bytes == 0) return NULL; + + size_t bytes = n * sizeof(float); + if (bytes > bf16_cache_limit_bytes) return NULL; + if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL; + + float *dst = (float *)malloc(bytes); + if (!dst) return NULL; + bf16_to_f32_buf(dst, src, n); + + if (bf16_cache_len == bf16_cache_cap) { + int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256; + bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc( + bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t)); + if (!tmp) { + free(dst); + return NULL; + } + bf16_cache = tmp; + bf16_cache_cap = new_cap; + } + + bf16_cache[bf16_cache_len].src = src; + bf16_cache[bf16_cache_len].n = n; + bf16_cache[bf16_cache_len].dst_f32 = dst; + bf16_cache_len++; + bf16_cache_bytes += bytes; + return dst; +} + +static const float *bf16_get_f32_view(const uint16_t *src, size_t n) { + const float *cached = bf16_get_cached_f32(src, n); + if (cached) return cached; + + float *scratch = bf16_get_scratch(n); + if (!scratch) return NULL; + bf16_to_f32_buf(scratch, src, n); + return scratch; +} + +/* + * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias + * Processes 2 output rows at a time to amortize x vector loads. + */ +static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); +} + +/* Threaded matvec: split output rows across threads */ +typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int in_dim; + int out_dim; +} matvec_task_t; + +static void matvec_worker(int tid, int n_threads, void *arg) { + matvec_task_t *t = (matvec_task_t *)arg; + int chunk = (t->out_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->out_dim) end = t->out_dim; + if (start >= end) return; + + bf16_matvec_fused(t->y + start, t->x, + t->W_bf16 + (size_t)start * t->in_dim, + t->bias ? t->bias + start : NULL, + t->in_dim, end - start); +} + +static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + if (tp.n_threads <= 1) { + bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); + return; + } + matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim }; + parallel_for(matvec_worker, &task); +} + +typedef struct { + float *q; + float *k; + float *v; + const float *x; + const uint16_t *Wq_bf16; + const uint16_t *Wk_bf16; + const uint16_t *Wv_bf16; + int in_dim; + int q_dim; + int kv_dim; + int total_dim; +} qkv_matvec_task_t; + +static void qkv_matvec_worker(int tid, int n_threads, void *arg) { + qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; + int chunk = (t->total_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->total_dim) end = t->total_dim; + if (start >= end) return; + + int q_end = t->q_dim; + int k_end = q_end + t->kv_dim; + int v_end = k_end + t->kv_dim; + + if (start < q_end) { + int s = start; + int e = end < q_end ? end : q_end; + if (s < e) { + bf16_matvec_fused(t->q + s, t->x, + t->Wq_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > q_end && start < k_end) { + int s = start > q_end ? start - q_end : 0; + int e_abs = end < k_end ? end : k_end; + int e = e_abs - q_end; + if (s < e) { + bf16_matvec_fused(t->k + s, t->x, + t->Wk_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > k_end && start < v_end) { + int s = start > k_end ? start - k_end : 0; + int e_abs = end < v_end ? end : v_end; + int e = e_abs - k_end; + if (s < e) { + bf16_matvec_fused(t->v + s, t->x, + t->Wv_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } +} + +void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int in_dim, int q_dim, int kv_dim) { + if (tp.n_threads <= 1) { + bf16_matvec_fused(q, x, Wq_bf16, NULL, in_dim, q_dim); + bf16_matvec_fused(k, x, Wk_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(v, x, Wv_bf16, NULL, in_dim, kv_dim); + return; + } + + qkv_matvec_task_t task = { + .q = q, + .k = k, + .v = v, + .x = x, + .Wq_bf16 = Wq_bf16, + .Wk_bf16 = Wk_bf16, + .Wv_bf16 = Wv_bf16, + .in_dim = in_dim, + .q_dim = q_dim, + .kv_dim = kv_dim, + .total_dim = q_dim + 2 * kv_dim, + }; + parallel_for(qkv_matvec_worker, &task); +} + +void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, + int seq_len, int in_dim, int out_dim) { + if (seq_len == 1) { + bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); + return; + } + size_t n = (size_t)out_dim * in_dim; + const float *W_f32 = bf16_get_f32_view(W_bf16, n); + if (!W_f32) return; + qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim); +} + +void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, + const float *b, int seq_len, int in_dim, int out_dim) { + if (seq_len == 1) { + bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim); + return; + } + size_t n = (size_t)out_dim * in_dim; + const float *W_f32 = bf16_get_f32_view(W_bf16, n); + if (!W_f32) return; + qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim); +} + +/* Find argmax over a range of output rows [start, end). + * Uses 2-row processing to amortize x vector loads (same as bf16_matvec_fused). */ +static void argmax_bf16_range(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + qwen_argmax_bf16_range_impl(x, W_bf16, in_dim, start, end, best_out, best_val_out); +} + +typedef struct { + const float *x; + const uint16_t *W_bf16; + int in_dim; + int out_dim; + int best_idx[QWEN_MAX_THREADS]; + float best_val[QWEN_MAX_THREADS]; +} argmax_task_t; + +static void argmax_worker(int tid, int n_threads, void *arg) { + argmax_task_t *t = (argmax_task_t *)arg; + int chunk = (t->out_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->out_dim) end = t->out_dim; + if (start >= end) { + t->best_val[tid] = -1e30f; + t->best_idx[tid] = 0; + return; + } + argmax_bf16_range(t->x, t->W_bf16, t->in_dim, start, end, + &t->best_idx[tid], &t->best_val[tid]); +} + +int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, + int in_dim, int out_dim) { + if (tp.n_threads <= 1) { + int best; + float best_val; + argmax_bf16_range(x, W_bf16, in_dim, 0, out_dim, &best, &best_val); + return best; + } + + argmax_task_t task; + task.x = x; + task.W_bf16 = W_bf16; + task.in_dim = in_dim; + task.out_dim = out_dim; + parallel_for(argmax_worker, &task); + + int best = task.best_idx[0]; + float best_val = task.best_val[0]; + for (int i = 1; i < tp.n_threads; i++) { + if (task.best_val[i] > best_val) { + best_val = task.best_val[i]; + best = task.best_idx[i]; + } + } + return best; +} + +void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, + int M, int K, int N) { + if (M == 1) { + bf16_matvec_threaded(C, A, B_bf16, NULL, K, N); + } else { + size_t n = (size_t)N * K; + const float *B_f32 = bf16_get_f32_view(B_bf16, n); + if (!B_f32) return; + qwen_matmul_t(C, A, B_f32, M, K, N); + } +} + +/* ======================================================================== + * 2D Convolution (im2col + BLAS sgemm) + * ======================================================================== */ + +/* + * im2col: Unroll input patches into a column matrix for GEMM-based conv2d. + * Input: [C_in, H_in, W_in] + * Output columns: [C_in * kH * kW, H_out * W_out] + */ +static void im2col(const float *in, float *cols, + int c_in, int h_in, int w_in, + int kh, int kw, int stride, int padding, + int h_out, int w_out) { + int col_len = h_out * w_out; + for (int ic = 0; ic < c_in; ic++) { + for (int ki = 0; ki < kh; ki++) { + for (int kj = 0; kj < kw; kj++) { + int col_row = (ic * kh + ki) * kw + kj; + float *col_ptr = cols + (size_t)col_row * col_len; + for (int oh = 0; oh < h_out; oh++) { + int ih = oh * stride - padding + ki; + for (int ow = 0; ow < w_out; ow++) { + int iw = ow * stride - padding + kj; + if (ih >= 0 && ih < h_in && iw >= 0 && iw < w_in) { + col_ptr[oh * w_out + ow] = in[ic * h_in * w_in + ih * w_in + iw]; + } else { + col_ptr[oh * w_out + ow] = 0.0f; + } + } + } + } + } + } +} + +void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, + int c_in, int c_out, int h_in, int w_in, + int kh, int kw, int stride, int padding) { + int h_out = (h_in + 2 * padding - kh) / stride + 1; + int w_out = (w_in + 2 * padding - kw) / stride + 1; + int patch_size = c_in * kh * kw; + int spatial_out = h_out * w_out; + + /* im2col: input -> column matrix [patch_size, spatial_out] */ + float *cols = (float *)malloc((size_t)patch_size * spatial_out * sizeof(float)); + im2col(in, cols, c_in, h_in, w_in, kh, kw, stride, padding, h_out, w_out); + + /* GEMM: weight[c_out, patch_size] @ cols[patch_size, spatial_out] = out[c_out, spatial_out] */ +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + c_out, spatial_out, patch_size, + 1.0f, weight, patch_size, cols, spatial_out, + 0.0f, out, spatial_out); +#else + for (int oc = 0; oc < c_out; oc++) { + for (int s = 0; s < spatial_out; s++) { + float sum = 0.0f; + for (int p = 0; p < patch_size; p++) { + sum += weight[oc * patch_size + p] * cols[p * spatial_out + s]; + } + out[oc * spatial_out + s] = sum; + } + } +#endif + + free(cols); + + /* Add bias */ + if (bias) { + for (int oc = 0; oc < c_out; oc++) { + float b = bias[oc]; + float *row = out + oc * spatial_out; + for (int s = 0; s < spatial_out; s++) { + row[s] += b; + } + } + } +} + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, + int seq_len, int hidden, float eps) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * hidden; + float *out_row = out + s * hidden; + + /* Compute mean */ +#if defined(__AVX512F__) + __m512 sumv = _mm512_setzero_ps(); + int i = 0; + for (; i + 16 <= hidden; i += 16) { + sumv = _mm512_add_ps(sumv, _mm512_loadu_ps(x_row + i)); + } + float mean = _mm512_reduce_add_ps(sumv); + for (; i < hidden; i++) mean += x_row[i]; +#elif defined(__AVX2__) + __m256 sumv = _mm256_setzero_ps(); + int i = 0; + for (; i + 8 <= hidden; i += 8) { + sumv = _mm256_add_ps(sumv, _mm256_loadu_ps(x_row + i)); + } + __m128 sum128 = _mm_add_ps(_mm256_castps256_ps128(sumv), _mm256_extractf128_ps(sumv, 1)); + sum128 = _mm_hadd_ps(sum128, sum128); + sum128 = _mm_hadd_ps(sum128, sum128); + float mean = _mm_cvtss_f32(sum128); + for (; i < hidden; i++) mean += x_row[i]; +#else + float mean = 0.0f; + for (int i = 0; i < hidden; i++) mean += x_row[i]; +#endif + mean /= hidden; + + /* Compute variance */ +#if defined(__AVX512F__) && defined(__FMA__) + __m512 meanv = _mm512_set1_ps(mean); + __m512 accv = _mm512_setzero_ps(); + int j = 0; + for (; j + 16 <= hidden; j += 16) { + __m512 v = _mm512_sub_ps(_mm512_loadu_ps(x_row + j), meanv); + accv = _mm512_fmadd_ps(v, v, accv); + } + float var = _mm512_reduce_add_ps(accv); + for (; j < hidden; j++) { + float d = x_row[j] - mean; + var += d * d; + } +#elif defined(__AVX2__) && defined(__FMA__) + __m256 meanv = _mm256_set1_ps(mean); + __m256 accv = _mm256_setzero_ps(); + int j = 0; + for (; j + 8 <= hidden; j += 8) { + __m256 v = _mm256_sub_ps(_mm256_loadu_ps(x_row + j), meanv); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float var = _mm_cvtss_f32(acc128); + for (; j < hidden; j++) { + float d = x_row[j] - mean; + var += d * d; + } +#else + float var = 0.0f; + for (int i = 0; i < hidden; i++) { + float d = x_row[i] - mean; + var += d * d; + } +#endif + var /= hidden; + + float inv_std = 1.0f / sqrtf(var + eps); +#if defined(__AVX512F__) && defined(__FMA__) + __m512 meanv2 = _mm512_set1_ps(mean); + __m512 invv = _mm512_set1_ps(inv_std); + int k = 0; + for (; k + 16 <= hidden; k += 16) { + __m512 vx = _mm512_sub_ps(_mm512_loadu_ps(x_row + k), meanv2); + __m512 vw = _mm512_loadu_ps(weight + k); + __m512 vb = _mm512_loadu_ps(bias + k); + __m512 v = _mm512_mul_ps(_mm512_mul_ps(vx, invv), vw); + v = _mm512_add_ps(v, vb); + _mm512_storeu_ps(out_row + k, v); + } + for (; k < hidden; k++) { + out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; + } +#elif defined(__AVX2__) && defined(__FMA__) + __m256 meanv2 = _mm256_set1_ps(mean); + __m256 invv = _mm256_set1_ps(inv_std); + int k = 0; + for (; k + 8 <= hidden; k += 8) { + __m256 vx = _mm256_sub_ps(_mm256_loadu_ps(x_row + k), meanv2); + __m256 vw = _mm256_loadu_ps(weight + k); + __m256 vb = _mm256_loadu_ps(bias + k); + __m256 v = _mm256_mul_ps(_mm256_mul_ps(vx, invv), vw); + v = _mm256_add_ps(v, vb); + _mm256_storeu_ps(out_row + k, v); + } + for (; k < hidden; k++) { + out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; + } +#else + for (int i = 0; i < hidden; i++) { + out_row[i] = (x_row[i] - mean) * inv_std * weight[i] + bias[i]; + } +#endif + } +} + +void qwen_rms_norm(float *out, const float *x, const float *weight, + int seq_len, int hidden, float eps) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * hidden; + float *out_row = out + s * hidden; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int i = 0; + for (; i + 16 <= hidden; i += 16) { + __m512 v = _mm512_loadu_ps(x_row + i); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int i = 0; + for (; i + 8 <= hidden; i += 8) { + __m256 v = _mm256_loadu_ps(x_row + i); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; +#else + float sum_sq = 0.0f; + for (int i = 0; i < hidden; i++) { + sum_sq += x_row[i] * x_row[i]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= hidden; j += 16) { + __m512 vx = _mm512_loadu_ps(x_row + j); + __m512 vw = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= hidden; j += 8) { + __m256 vx = _mm256_loadu_ps(x_row + j); + __m256 vw = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; +#else + for (int i = 0; i < hidden; i++) { + out_row[i] = x_row[i] * rms_inv * weight[i]; + } +#endif + } +} + +void qwen_rms_norm_per_head(float *x, const float *weight, + int seq_len, int n_heads, int head_dim, float eps) { + /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ + int hidden = n_heads * head_dim; + for (int s = 0; s < seq_len; s++) { + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int d = 0; + for (; d + 16 <= head_dim; d += 16) { + __m512 v = _mm512_loadu_ps(vec + d); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int d = 0; + for (; d + 8 <= head_dim; d += 8) { + __m256 v = _mm256_loadu_ps(vec + d); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; +#else + float sum_sq = 0.0f; + for (int d = 0; d < head_dim; d++) { + sum_sq += vec[d] * vec[d]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= head_dim; j += 16) { + __m512 v = _mm512_loadu_ps(vec + j); + __m512 w = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= head_dim; j += 8) { + __m256 v = _mm256_loadu_ps(vec + j); + __m256 w = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; +#else + for (int d = 0; d < head_dim; d++) { + vec[d] = vec[d] * rms_inv * weight[d]; + } +#endif + } + } +} + +/* ======================================================================== + * Activation Functions + * ======================================================================== */ + +void qwen_silu(float *x, int n) { + for (int i = 0; i < n; i++) { + float val = x[i]; + x[i] = val / (1.0f + expf(-val)); + } +} + +void qwen_gelu(float *x, int n) { + for (int i = 0; i < n; i++) { + float val = x[i]; + float x3 = val * val * val; + float inner = 0.7978845608028654f * (val + 0.044715f * x3); + x[i] = 0.5f * val * (1.0f + tanhf(inner)); + } +} + +typedef struct { + float *out; + const float *gate_up; + int seq_len; + int intermediate; +} swiglu_task_t; + +static void swiglu_worker(int tid, int n_threads, void *arg) { + swiglu_task_t *t = (swiglu_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int s0 = tid * chunk; + int s1 = s0 + chunk; + if (s1 > t->seq_len) s1 = t->seq_len; + if (s0 >= s1) return; + + int inter = t->intermediate; + int alias_inplace = (t->out == t->gate_up); + for (int s = s0; s < s1; s++) { + const float *gu = t->gate_up + (size_t)s * 2 * inter; + float *o = t->out + (size_t)s * inter; + if (!alias_inplace) { +#if defined(__APPLE__) && defined(USE_BLAS) + /* Fast path for prefill: vectorized exp(-g) using Accelerate/vForce. */ + for (int j = 0; j < inter; j++) o[j] = -gu[2 * j]; + int n = inter; + vvexpf(o, o, &n); + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + o[j] = (g / (1.0f + o[j])) * u; + } +#else + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + g = g / (1.0f + expf(-g)); /* SiLU */ + o[j] = g * u; + } +#endif + } else { + /* In-place mode (decode seq=1): keep single-pass scalar to avoid alias hazards. */ + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + g = g / (1.0f + expf(-g)); /* SiLU */ + o[j] = g * u; + } + } + } +} + +void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate) { + swiglu_task_t task = { + .out = out, + .gate_up = gate_up, + .seq_len = seq_len, + .intermediate = intermediate + }; + + if (tp.n_threads > 1 && seq_len >= 2 && intermediate >= 256) { + parallel_for(swiglu_worker, &task); + } else { + swiglu_worker(0, 1, &task); + } +} + +void qwen_softmax(float *x, int rows, int cols) { + for (int r = 0; r < rows; r++) { + float *row = x + r * cols; + float max_val = row[0]; + for (int c = 1; c < cols; c++) { + if (row[c] > max_val) max_val = row[c]; + } + float sum = 0.0f; + for (int c = 0; c < cols; c++) { + row[c] = expf(row[c] - max_val); + sum += row[c]; + } + float inv_sum = 1.0f / sum; + for (int c = 0; c < cols; c++) { + row[c] *= inv_sum; + } + } +} + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +static inline float qwen_dot_f32(const float *a, const float *b, int n) { + return qwen_dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void qwen_vec_scale_inplace(float *dst, float scale, int n) { + qwen_vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void qwen_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + qwen_vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void qwen_vec_scale_add(float *dst, const float *src, float correction, int n) { + qwen_vec_scale_add_impl(dst, src, correction, n); +} + +void qwen_bidirectional_attention(float *out, const float *Q, const float *K, + const float *V, int seq __attribute__((unused)), + int n_heads, int head_dim, float scale, + const int *window_starts, int n_windows) { + int hidden = n_heads * head_dim; + + for (int h = 0; h < n_heads; h++) { + for (int w = 0; w < n_windows; w++) { + int ws = window_starts[w]; + int we = window_starts[w + 1]; + + for (int i = ws; i < we; i++) { + const float *q_row = Q + i * hidden + h * head_dim; + float *o_row = out + i * hidden + h * head_dim; + + /* Online softmax */ + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = ws; j < we; j++) { + const float *k_row = K + j * hidden + h * head_dim; + const float *v_row = V + j * hidden + h * head_dim; + + float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } + } + } +} + +static void qwen_causal_attention_heads(float *out, const float *Q, const float *K, + const float *V, int seq_q, int seq_k, + int n_heads, int n_kv_heads, int head_dim, + float scale, int q_offset, + int head_start, int head_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + + for (int h = head_start; h < head_end; h++) { + int kv_h = h / heads_per_kv; + + for (int i = 0; i < seq_q; i++) { + const float *q_row = Q + i * q_hidden + h * head_dim; + float *o_row = out + i * q_hidden + h * head_dim; + int global_pos = q_offset + i; + int k_end = global_pos + 1; + if (k_end > seq_k) k_end = seq_k; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = 0; j < k_end; j++) { + const float *k_row = K + j * kv_hidden + kv_h * head_dim; + const float *v_row = V + j * kv_hidden + kv_h * head_dim; + + float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } + } +} + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + int seq_q, seq_k; + int n_heads, n_kv_heads; + int head_dim; + float scale; + int q_offset; +} causal_attn_task_t; + +static void causal_attn_worker(int tid, int n_threads, void *arg) { + causal_attn_task_t *t = (causal_attn_task_t *)arg; + int chunk = (t->n_heads + n_threads - 1) / n_threads; + int h0 = tid * chunk; + int h1 = h0 + chunk; + if (h1 > t->n_heads) h1 = t->n_heads; + if (h0 >= h1) return; + + qwen_causal_attention_heads(t->out, t->Q, t->K, t->V, + t->seq_q, t->seq_k, t->n_heads, t->n_kv_heads, + t->head_dim, t->scale, t->q_offset, h0, h1); +} + +void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, + int seq_q, int seq_k, int n_heads, int n_kv_heads, + int head_dim, float scale, int q_offset) { + if (tp.n_threads > 1 && n_heads >= 2 && (seq_q >= 2 || seq_k >= 128)) { + causal_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, + .seq_q = seq_q, .seq_k = seq_k, + .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale, .q_offset = q_offset + }; + parallel_for(causal_attn_worker, &task); + return; + } + + qwen_causal_attention_heads(out, Q, K, V, + seq_q, seq_k, n_heads, n_kv_heads, + head_dim, scale, q_offset, 0, n_heads); +} + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model) { + int half = d_model / 2; + float log_timescale = logf(10000.0f) / (float)(half - 1); + + for (int p = 0; p < n_pos; p++) { + float *row = pe + p * d_model; + for (int d = 0; d < half; d++) { + float inv_timescale = expf(-(float)d * log_timescale); + float angle = (float)p * inv_timescale; + row[d] = sinf(angle); /* first half: sin */ + row[half + d] = cosf(angle); /* second half: cos */ + } + } +} + +void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, + int seq, int head_dim, float theta) { + int half = head_dim / 2; + + for (int s = 0; s < seq; s++) { + float pos = (float)positions[s]; + for (int d = 0; d < half; d++) { + float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = pos * freq; + float c = cosf(angle); + float sn = sinf(angle); + /* Duplicate for full head_dim */ + cos_out[s * head_dim + d] = c; + cos_out[s * head_dim + half + d] = c; + sin_out[s * head_dim + d] = sn; + sin_out[s * head_dim + half + d] = sn; + } + } +} + +void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, + int seq, int n_heads, int head_dim) { + /* + * NeoX split-half style: + * x1 = x[..., :half], x2 = x[..., half:] + * rotated = cat(-x2, x1) + * result = x * cos + rotated * sin + */ + int half = head_dim / 2; + int hidden = n_heads * head_dim; + + for (int s = 0; s < seq; s++) { + const float *c = cos_vals + s * head_dim; + const float *sn = sin_vals + s * head_dim; + + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + int d = 0; + for (; d + 16 <= half; d += 16) { + __m512 x1 = _mm512_loadu_ps(vec + d); + __m512 x2 = _mm512_loadu_ps(vec + half + d); + /* RoPE cache duplicates cos/sin across halves. */ + __m512 cc = _mm512_loadu_ps(c + d); + __m512 ss = _mm512_loadu_ps(sn + d); + __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); + __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); + _mm512_storeu_ps(vec + d, new1); + _mm512_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__AVX2__) && defined(__FMA__) + int d = 0; + for (; d + 8 <= half; d += 8) { + __m256 x1 = _mm256_loadu_ps(vec + d); + __m256 x2 = _mm256_loadu_ps(vec + half + d); + __m256 cc = _mm256_loadu_ps(c + d); + __m256 ss = _mm256_loadu_ps(sn + d); + __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); + __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); + _mm256_storeu_ps(vec + d, new1); + _mm256_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#else + for (int d = 0; d < half; d++) { + float x1 = vec[d]; /* first half */ + float x2 = vec[half + d]; /* second half */ + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#endif + } + } +} diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h new file mode 100644 index 0000000..c59aeff --- /dev/null +++ b/qwen_asr_kernels.h @@ -0,0 +1,173 @@ +/* + * qwen_asr_kernels.h - Math kernels for Qwen3-ASR inference + * + * Low-level math operations. All operate on float32 tensors in row-major order. + * Adapted from voxtral-realtime project. + */ + +#ifndef QWEN_ASR_KERNELS_H +#define QWEN_ASR_KERNELS_H + +#include +#include + +/* ======================================================================== + * Basic Operations + * ======================================================================== */ + +void qwen_add_inplace(float *a, const float *b, int n); +void qwen_mul_inplace(float *a, const float *b, int n); +void qwen_scale(float *x, float s, int n); +void qwen_copy(float *dst, const float *src, int n); + +/* ======================================================================== + * Matrix Operations + * ======================================================================== */ + +/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ +void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); + +/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ +void qwen_linear(float *y, const float *x, const float *W, const float *b, + int seq_len, int in_dim, int out_dim); + +void qwen_linear_nobias(float *y, const float *x, const float *W, + int seq_len, int in_dim, int out_dim); + +/* bf16 weight variants */ +void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, + const float *b, int seq_len, int in_dim, int out_dim); + +void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, + int seq_len, int in_dim, int out_dim); + +/* seq=1 decoder fast path: compute Q/K/V matvecs with one threaded dispatch */ +void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int in_dim, int q_dim, int kv_dim); + +void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, + int M, int K, int N); + +/* ======================================================================== + * 2D Convolution (for audio encoder conv stem) + * ======================================================================== */ + +/* + * 2D Convolution: out = conv2d(in, weight, bias) + * in: [C_in, H, W] + * weight: [C_out, C_in, kH, kW] + * bias: [C_out] (can be NULL) + * out: [C_out, H_out, W_out] + * H_out = (H + 2*padding - kH) / stride + 1 + * W_out = (W + 2*padding - kW) / stride + 1 + */ +void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, + int c_in, int c_out, int h_in, int w_in, + int kh, int kw, int stride, int padding); + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +/* LayerNorm with bias: out = (x - mean) / sqrt(var + eps) * weight + bias */ +void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, + int seq_len, int hidden, float eps); + +/* RMS Normalization: out = x / rms(x) * weight */ +void qwen_rms_norm(float *out, const float *x, const float *weight, + int seq_len, int hidden, float eps); + +/* Per-head RMS Normalization for Q/K norms in decoder + * x: [seq, n_heads, head_dim], weight: [head_dim] + * Normalizes each head independently */ +void qwen_rms_norm_per_head(float *x, const float *weight, + int seq_len, int n_heads, int head_dim, float eps); + +/* ======================================================================== + * Activation Functions + * ======================================================================== */ + +void qwen_silu(float *x, int n); +void qwen_gelu(float *x, int n); +void qwen_softmax(float *x, int rows, int cols); +/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */ +void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate); + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +/* + * Bidirectional windowed attention (encoder). + * Q, K, V: [seq, n_heads * head_dim] + * out: [seq, n_heads * head_dim] + * window_starts: array of window start positions + * window_starts[n_windows] = seq (sentinel) + * All heads have same dimensions (no GQA in encoder). + */ +void qwen_bidirectional_attention(float *out, const float *Q, const float *K, + const float *V, int seq, int n_heads, + int head_dim, float scale, + const int *window_starts, int n_windows); + +/* + * Causal attention with GQA (decoder). + * Q: [seq_q, n_heads * head_dim] + * K: [seq_k, n_kv_heads * head_dim] + * V: [seq_k, n_kv_heads * head_dim] + * q_offset: global position of first query (for causal mask) + */ +void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, + int seq_q, int seq_k, int n_heads, int n_kv_heads, + int head_dim, float scale, int q_offset); + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +/* + * Sinusoidal position embeddings (encoder). + * pe: output [n_pos, d_model] + * First half = sin, second half = cos. + */ +void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model); + +/* + * NeoX-style RoPE: compute cos/sin for positions. + * cos_out, sin_out: [seq, head_dim] + * cos[d] and cos[half+d] are the same (duplicated for full head_dim). + */ +void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, + int seq, int head_dim, float theta); + +/* + * Apply NeoX-style RoPE to Q or K. + * x: [seq, n_heads * head_dim] (in-place) + * cos_vals, sin_vals: [seq, head_dim] + */ +void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, + int seq, int n_heads, int head_dim); + +/* Streaming argmax: finds argmax(W_bf16 @ x) without materializing full logits. + * Returns the index of the row with highest dot product. */ +int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, + int in_dim, int out_dim); + +/* ======================================================================== + * Threading + * ======================================================================== */ + +/* Set number of threads for parallel operations (default: 1). + * Creates a persistent thread pool. Call before inference. */ +void qwen_set_threads(int n); + +/* Get number of available CPU cores */ +int qwen_get_num_cpus(void); + +/* Global verbose flag */ +extern int qwen_verbose; + +#endif /* QWEN_ASR_KERNELS_H */ diff --git a/qwen_asr_kernels_avx.c b/qwen_asr_kernels_avx.c new file mode 100644 index 0000000..3fa2194 --- /dev/null +++ b/qwen_asr_kernels_avx.c @@ -0,0 +1,502 @@ +/* + * qwen_asr_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) + * + * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. + * Uses AVX-512F+BW for 16-wide bf16 matvec/argmax (dominant for seq_len==1), and + * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). + * + * The bf16 matvec processes 4 output rows simultaneously to reduce instruction + * overhead and improve out-of-order execution on memory-bound workloads. + */ + +#include "qwen_asr_kernels_impl.h" + +#if defined(__AVX2__) && defined(__FMA__) + +#include +#include + +/* ===================================================================== + * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise + * ===================================================================== */ + +#if defined(__AVX512F__) && defined(__AVX512BW__) + +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + /* Process 4 output rows at a time */ + for (; o + 3 < out_dim; o += 4) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + const uint16_t *w2 = w1 + in_dim; + const uint16_t *w3 = w2 + in_dim; + + __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); + __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); + __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps(); + __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + + /* Row 0 */ + __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + + /* Row 1 */ + __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + + /* Row 2 */ + __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k)); + __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16)); + a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4); + a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5); + + /* Row 3 */ + __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k)); + __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16)); + a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6); + a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7); + } + + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4); + a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6); + } + + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); + float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f); + float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f); + + for (; k < in_dim; k++) { + float xk = x[k]; + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; + float v0,v1,v2,v3; + memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); + s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; + } + y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3; + } + + /* Handle 2-row remainder */ + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); + __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); + __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + } + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + } + + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); + + for (; k < in_dim; k++) { + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); + s0+=v0*x[k]; s1+=v1*x[k]; + } + y[o]=s0; y[o+1]=s1; + } + + /* Single remaining row */ + for (; o < out_dim; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m512 acc = _mm512_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); + acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), + _mm512_loadu_ps(x + k), acc); + } + float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum += wv*x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + int o = start; + + /* Process 4 rows at a time */ + for (; o + 3 < end; o += 4) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + const uint16_t *w2 = w1 + in_dim; + const uint16_t *w3 = w2 + in_dim; + + __m512 a0=_mm512_setzero_ps(), a1=_mm512_setzero_ps(); + __m512 a2=_mm512_setzero_ps(), a3=_mm512_setzero_ps(); + __m512 a4=_mm512_setzero_ps(), a5=_mm512_setzero_ps(); + __m512 a6=_mm512_setzero_ps(), a7=_mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + __m256i r0a=_mm256_loadu_si256((const __m256i*)(w0+k)); + __m256i r0b=_mm256_loadu_si256((const __m256i*)(w0+k+16)); + a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a),16)),xv0,a0); + a1=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b),16)),xv1,a1); + __m256i r1a=_mm256_loadu_si256((const __m256i*)(w1+k)); + __m256i r1b=_mm256_loadu_si256((const __m256i*)(w1+k+16)); + a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a),16)),xv0,a2); + a3=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b),16)),xv1,a3); + __m256i r2a=_mm256_loadu_si256((const __m256i*)(w2+k)); + __m256i r2b=_mm256_loadu_si256((const __m256i*)(w2+k+16)); + a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a),16)),xv0,a4); + a5=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b),16)),xv1,a5); + __m256i r3a=_mm256_loadu_si256((const __m256i*)(w3+k)); + __m256i r3b=_mm256_loadu_si256((const __m256i*)(w3+k+16)); + a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a),16)),xv0,a6); + a7=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b),16)),xv1,a7); + } + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0=_mm256_loadu_si256((const __m256i*)(w0+k)); + __m256i r1=_mm256_loadu_si256((const __m256i*)(w1+k)); + __m256i r2=_mm256_loadu_si256((const __m256i*)(w2+k)); + __m256i r3=_mm256_loadu_si256((const __m256i*)(w3+k)); + a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0),16)),xv,a0); + a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1),16)),xv,a2); + a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2),16)),xv,a4); + a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3),16)),xv,a6); + } + + float s0=_mm512_reduce_add_ps(_mm512_add_ps(a0,a1)); + float s1=_mm512_reduce_add_ps(_mm512_add_ps(a2,a3)); + float s2=_mm512_reduce_add_ps(_mm512_add_ps(a4,a5)); + float s3=_mm512_reduce_add_ps(_mm512_add_ps(a6,a7)); + + for (; k < in_dim; k++) { + float xk=x[k]; + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; + float v0,v1,v2,v3; + memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); + s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; + } + + if (s0>best_val){best_val=s0;best=o;} + if (s1>best_val){best_val=s1;best=o+1;} + if (s2>best_val){best_val=s2;best=o+2;} + if (s3>best_val){best_val=s3;best=o+3;} + } + + /* Remaining 1-3 rows */ + for (; o < end; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m512 acc = _mm512_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); + acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), + _mm512_loadu_ps(x + k), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum += wv*x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + + *best_out = best; + *best_val_out = best_val; +} + +#else /* AVX2 only (or AVX-512F without BW) */ + +/* Helper: bf16 to f32 for 8 elements */ +static inline __m256 bf16x8_to_f32(__m128i raw) { + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16)); +} + +static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { + __m256i raw = _mm256_loadu_si256((const __m256i *)src); + *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw)); + *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); +} + +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 xlo = _mm256_loadu_ps(x + k); + __m256 xhi = _mm256_loadu_ps(x + k + 8); + __m256 wlo, whi; + bf16x16_to_f32(w0 + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, xlo, a0); + a1 = _mm256_fmadd_ps(whi, xhi, a1); + bf16x16_to_f32(w1 + k, &wlo, &whi); + a2 = _mm256_fmadd_ps(wlo, xlo, a2); + a3 = _mm256_fmadd_ps(whi, xhi, a3); + } + a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3); + __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); + r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0); + float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f); + float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f); + for (; k < in_dim; k++) { + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); + s0+=v0*x[k]; s1+=v1*x[k]; + } + y[o]=s0; y[o+1]=s1; + } + for (; o < out_dim; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 wlo, whi; + bf16x16_to_f32(w + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); + a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); + } + a0 = _mm256_add_ps(a0, a1); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum+=wv*x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + + for (int o = start; o < end; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 wlo, whi; + bf16x16_to_f32(w + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); + a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); + } + a0 = _mm256_add_ps(a0, a1); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum+=wv*x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + *best_out = best; + *best_val_out = best_val; +} + +#endif /* AVX-512F+BW vs AVX2 for bf16 */ + +/* ===================================================================== + * f32 attention helpers - AVX2+FMA, with AVX-512F when available + * (operates on L1-resident head vectors) + * ===================================================================== */ + +float qwen_dot_f32_avx(const float *a, const float *b, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); + } + __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); + for (; i + 16 <= n; i += 16) { + acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +#else + int i = 0; + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3); + } + acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + for (; i + 8 <= n; i += 8) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); + } + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +#endif +} + +void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 s = _mm512_set1_ps(scale); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); + _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); + _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + } + for (; i < n; i++) dst[i] *= scale; +#else + int i = 0; + __m256 s = _mm256_set1_ps(scale); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); + _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s)); + _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s)); + _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s)); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); + } + for (; i < n; i++) dst[i] *= scale; +#endif +} + +void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 a = _mm512_set1_ps(alpha); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + _mm512_storeu_ps(dst + i + 16, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +#else + int i = 0; + __m256 a = _mm256_set1_ps(alpha); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); + _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8))); + _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16))); + _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +#endif +} + +void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 c = _mm512_set1_ps(correction); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + _mm512_storeu_ps(dst + i + 16, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +#else + int i = 0; + __m256 c = _mm256_set1_ps(correction); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); + _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8))); + _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16))); + _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +#endif +} + +#endif /* __AVX2__ && __FMA__ */ diff --git a/qwen_asr_kernels_generic.c b/qwen_asr_kernels_generic.c new file mode 100644 index 0000000..d322d8e --- /dev/null +++ b/qwen_asr_kernels_generic.c @@ -0,0 +1,65 @@ +/* + * qwen_asr_kernels_generic.c - architecture-generic hot kernels + */ + +#include "qwen_asr_kernels_impl.h" + +#include + +void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + for (int o = 0; o < out_dim; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = bias ? bias[o] : 0.0f; + for (int k = 0; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + + for (int o = start; o < end; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = 0.0f; + for (int k = 0; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + if (sum > best_val) { + best_val = sum; + best = o; + } + } + + *best_out = best; + *best_val_out = best_val; +} + +float qwen_dot_f32_generic(const float *a, const float *b, int n) { + float sum = 0.0f; + for (int i = 0; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) { + for (int i = 0; i < n; i++) dst[i] *= scale; +} + +void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { + for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; +} + +void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { + for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; +} diff --git a/qwen_asr_kernels_impl.h b/qwen_asr_kernels_impl.h new file mode 100644 index 0000000..31b9cbb --- /dev/null +++ b/qwen_asr_kernels_impl.h @@ -0,0 +1,65 @@ +/* + * qwen_asr_kernels_impl.h - internal architecture dispatch for hot kernels + */ + +#ifndef QWEN_ASR_KERNELS_IMPL_H +#define QWEN_ASR_KERNELS_IMPL_H + +#include + +void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_generic(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON +void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_neon(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); + +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_neon +#define qwen_dot_f32_impl qwen_dot_f32_neon +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon +#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon + +#elif defined(__AVX2__) && defined(__FMA__) +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_avx(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); + +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_avx +#define qwen_dot_f32_impl qwen_dot_f32_avx +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx +#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx + +#else +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_generic +#define qwen_dot_f32_impl qwen_dot_f32_generic +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic +#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic +#endif + +#endif /* QWEN_ASR_KERNELS_IMPL_H */ diff --git a/qwen_asr_kernels_neon.c b/qwen_asr_kernels_neon.c new file mode 100644 index 0000000..90d8ed9 --- /dev/null +++ b/qwen_asr_kernels_neon.c @@ -0,0 +1,274 @@ +/* + * qwen_asr_kernels_neon.c - ARM NEON hot kernels + */ + +#include "qwen_asr_kernels_impl.h" + +#ifdef __ARM_NEON + +#include +#include + +void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */ + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; + float s0 = bias ? bias[o] : 0.0f; + float s1 = bias ? bias[o + 1] : 0.0f; + + float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); + float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); + float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); + float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + float32x4_t x2 = vld1q_f32(x + k + 8); + float32x4_t x3 = vld1q_f32(x + k + 12); + float32x4_t x4 = vld1q_f32(x + k + 16); + float32x4_t x5 = vld1q_f32(x + k + 20); + float32x4_t x6 = vld1q_f32(x + k + 24); + float32x4_t x7 = vld1q_f32(x + k + 28); + + uint16x8_t r0a = vld1q_u16(w0 + k); + uint16x8_t r0b = vld1q_u16(w0 + k + 8); + uint16x8_t r0c = vld1q_u16(w0 + k + 16); + uint16x8_t r0d = vld1q_u16(w0 + k + 24); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); + + uint16x8_t r1a = vld1q_u16(w1 + k); + uint16x8_t r1b = vld1q_u16(w1 + k + 8); + uint16x8_t r1c = vld1q_u16(w1 + k + 16); + uint16x8_t r1d = vld1q_u16(w1 + k + 24); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); + } + for (; k + 8 <= in_dim; k += 8) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + uint16x8_t r0 = vld1q_u16(w0 + k); + uint16x8_t r1 = vld1q_u16(w1 + k); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0), 16)), x1); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1), 16)), x1); + } + s0 += vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); + s1 += vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); + + for (; k < in_dim; k++) { + uint32_t bits0 = ((uint32_t)w0[k]) << 16; + uint32_t bits1 = ((uint32_t)w1[k]) << 16; + float wv0, wv1; + memcpy(&wv0, &bits0, sizeof(float)); + memcpy(&wv1, &bits1, sizeof(float)); + s0 += wv0 * x[k]; + s1 += wv1 * x[k]; + } + y[o] = s0; + y[o + 1] = s1; + } + + /* Handle remaining odd row */ + for (; o < out_dim; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = bias ? bias[o] : 0.0f; + int k = 0; + + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; k + 8 <= in_dim; k += 8) { + uint16x8_t bf = vld1q_u16(w_row + k); + acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), + vld1q_f32(x + k)); + acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), + vld1q_f32(x + k + 4)); + } + sum += vaddvq_f32(vaddq_f32(acc0, acc1)); + + for (; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + int o = start; + + /* Process 2 rows at a time, 32 elements/iter, 8 accumulators per row */ + for (; o + 1 < end; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; + float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); + float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); + float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); + float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + float32x4_t x2 = vld1q_f32(x + k + 8); + float32x4_t x3 = vld1q_f32(x + k + 12); + float32x4_t x4 = vld1q_f32(x + k + 16); + float32x4_t x5 = vld1q_f32(x + k + 20); + float32x4_t x6 = vld1q_f32(x + k + 24); + float32x4_t x7 = vld1q_f32(x + k + 28); + + uint16x8_t r0a = vld1q_u16(w0 + k); + uint16x8_t r0b = vld1q_u16(w0 + k + 8); + uint16x8_t r0c = vld1q_u16(w0 + k + 16); + uint16x8_t r0d = vld1q_u16(w0 + k + 24); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); + + uint16x8_t r1a = vld1q_u16(w1 + k); + uint16x8_t r1b = vld1q_u16(w1 + k + 8); + uint16x8_t r1c = vld1q_u16(w1 + k + 16); + uint16x8_t r1d = vld1q_u16(w1 + k + 24); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); + } + + float s0 = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); + float s1 = vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); + + for (; k < in_dim; k++) { + uint32_t bits0 = ((uint32_t)w0[k]) << 16; + uint32_t bits1 = ((uint32_t)w1[k]) << 16; + float wv0, wv1; + memcpy(&wv0, &bits0, sizeof(float)); + memcpy(&wv1, &bits1, sizeof(float)); + s0 += wv0 * x[k]; + s1 += wv1 * x[k]; + } + + if (s0 > best_val) { best_val = s0; best = o; } + if (s1 > best_val) { best_val = s1; best = o + 1; } + } + + for (; o < end; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = 0.0f; + int k = 0; + + float32x4_t acc0 = vdupq_n_f32(0.0f), acc1 = vdupq_n_f32(0.0f); + for (; k + 8 <= in_dim; k += 8) { + uint16x8_t bf = vld1q_u16(w_row + k); + acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), + vld1q_f32(x + k)); + acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), + vld1q_f32(x + k + 4)); + } + sum += vaddvq_f32(vaddq_f32(acc0, acc1)); + + for (; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + + *best_out = best; + *best_val_out = best_val; +} + +float qwen_dot_f32_neon(const float *a, const float *b, int n) { + int i = 0; + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; i + 8 <= n; i += 8) { + float32x4_t a0 = vld1q_f32(a + i); + float32x4_t b0 = vld1q_f32(b + i); + float32x4_t a1 = vld1q_f32(a + i + 4); + float32x4_t b1 = vld1q_f32(b + i + 4); + acc0 = vfmaq_f32(acc0, a0, b0); + acc1 = vfmaq_f32(acc1, a1, b1); + } + float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { + int i = 0; + float32x4_t s = vdupq_n_f32(scale); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); + vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); + } + for (; i < n; i++) dst[i] *= scale; +} + +void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { + int i = 0; + float32x4_t a = vdupq_n_f32(alpha); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); + vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +} + +void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { + int i = 0; + float32x4_t c = vdupq_n_f32(correction); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); + vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +} + +#endif /* __ARM_NEON */ diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c new file mode 100644 index 0000000..0bdf272 --- /dev/null +++ b/qwen_asr_safetensors.c @@ -0,0 +1,394 @@ +/* + * qwen_asr_safetensors.c - Safetensors reader with multi-shard support + * Adapted from voxtral-realtime project. + */ + +#include "qwen_asr_safetensors.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* ======================================================================== + * Minimal JSON parser for safetensors header + * ======================================================================== */ + +static void skip_whitespace(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; +} + +static int parse_string(const char **p, char *out, size_t max_len) { + skip_whitespace(p); + if (**p != '"') return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') out[i++] = '\n'; + else if (**p == 't') out[i++] = '\t'; + else if (**p == '"') out[i++] = '"'; + else if (**p == '\\') out[i++] = '\\'; + else out[i++] = **p; + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') return -1; + (*p)++; + return 0; +} + +static int64_t parse_int(const char **p) { + skip_whitespace(p); + int64_t val = 0; + int neg = 0; + if (**p == '-') { neg = 1; (*p)++; } + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + return neg ? -val : val; +} + +static safetensor_dtype_t parse_dtype(const char *s) { + if (strcmp(s, "F32") == 0) return DTYPE_F32; + if (strcmp(s, "F16") == 0) return DTYPE_F16; + if (strcmp(s, "BF16") == 0) return DTYPE_BF16; + if (strcmp(s, "I32") == 0) return DTYPE_I32; + if (strcmp(s, "I64") == 0) return DTYPE_I64; + if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL; + return DTYPE_UNKNOWN; +} + +static int parse_tensor_entry(const char **p, safetensor_t *t) { + skip_whitespace(p); + if (**p != '{') return -1; + (*p)++; + + t->dtype = DTYPE_UNKNOWN; + t->ndim = 0; + t->data_offset = 0; + t->data_size = 0; + + while (**p && **p != '}') { + skip_whitespace(p); + if (**p == ',') { (*p)++; continue; } + + char key[64]; + if (parse_string(p, key, sizeof(key)) != 0) return -1; + skip_whitespace(p); + if (**p != ':') return -1; + (*p)++; + skip_whitespace(p); + + if (strcmp(key, "dtype") == 0) { + char dtype_str[32]; + if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1; + t->dtype = parse_dtype(dtype_str); + } else if (strcmp(key, "shape") == 0) { + if (**p != '[') return -1; + (*p)++; + t->ndim = 0; + while (**p && **p != ']' && t->ndim < 8) { + skip_whitespace(p); + if (**p == ',') { (*p)++; continue; } + t->shape[t->ndim++] = parse_int(p); + } + if (**p == ']') (*p)++; + } else if (strcmp(key, "data_offsets") == 0) { + if (**p != '[') return -1; + (*p)++; + skip_whitespace(p); + size_t start = (size_t)parse_int(p); + skip_whitespace(p); + if (**p == ',') (*p)++; + skip_whitespace(p); + size_t end = (size_t)parse_int(p); + t->data_offset = start; + t->data_size = end - start; + skip_whitespace(p); + if (**p == ']') (*p)++; + } else { + /* Skip unknown value */ + if (**p == '"') { + (*p)++; + while (**p && **p != '"') { + if (**p == '\\') (*p)++; + if (**p) (*p)++; + } + if (**p == '"') (*p)++; + } else if (**p == '[') { + int depth = 1; (*p)++; + while (**p && depth > 0) { + if (**p == '[') depth++; + else if (**p == ']') depth--; + (*p)++; + } + } else if (**p == '{') { + int depth = 1; (*p)++; + while (**p && depth > 0) { + if (**p == '{') depth++; + else if (**p == '}') depth--; + (*p)++; + } + } else { + while (**p && **p != ',' && **p != '}') (*p)++; + } + } + } + if (**p == '}') (*p)++; + return 0; +} + +static int parse_header(safetensors_file_t *sf) { + const char *p = sf->header_json; + skip_whitespace(&p); + if (*p != '{') return -1; + p++; + + sf->num_tensors = 0; + + while (*p && *p != '}' && sf->num_tensors < SAFETENSORS_MAX_TENSORS) { + skip_whitespace(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char name[256]; + if (parse_string(&p, name, sizeof(name)) != 0) return -1; + skip_whitespace(&p); + if (*p != ':') return -1; + p++; + + if (strcmp(name, "__metadata__") == 0) { + skip_whitespace(&p); + if (*p == '{') { + int depth = 1; p++; + while (*p && depth > 0) { + if (*p == '{') depth++; + else if (*p == '}') depth--; + p++; + } + } + continue; + } + + safetensor_t *t = &sf->tensors[sf->num_tensors]; + snprintf(t->name, sizeof(t->name), "%s", name); + if (parse_tensor_entry(&p, t) != 0) return -1; + sf->num_tensors++; + } + return 0; +} + +/* ======================================================================== + * Single file operations + * ======================================================================== */ + +safetensors_file_t *safetensors_open(const char *path) { + int fd = open(path, O_RDONLY); + if (fd < 0) return NULL; + + struct stat st; + if (fstat(fd, &st) < 0) { close(fd); return NULL; } + + size_t file_size = (size_t)st.st_size; + if (file_size < 8) { close(fd); return NULL; } + + void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (data == MAP_FAILED) return NULL; + + uint64_t header_size = 0; + memcpy(&header_size, data, 8); + if (header_size > file_size - 8) { munmap(data, file_size); return NULL; } + + safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); + if (!sf) { munmap(data, file_size); return NULL; } + + sf->path = strdup(path); + sf->data = data; + sf->file_size = file_size; + sf->header_size = (size_t)header_size; + + sf->header_json = malloc(header_size + 1); + if (!sf->header_json) { safetensors_close(sf); return NULL; } + memcpy(sf->header_json, (char *)data + 8, header_size); + sf->header_json[header_size] = '\0'; + + if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } + + return sf; +} + +void safetensors_close(safetensors_file_t *sf) { + if (!sf) return; + if (sf->data) munmap(sf->data, sf->file_size); + free(sf->path); + free(sf->header_json); + free(sf); +} + +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { + return (const char *)sf->data + 8 + sf->header_size + t->data_offset; +} + +int64_t safetensor_numel(const safetensor_t *t) { + int64_t n = 1; + for (int i = 0; i < t->ndim; i++) n *= t->shape[i]; + return n; +} + +static float bf16_to_f32(uint16_t bf16) { + uint32_t f32 = ((uint32_t)bf16) << 16; + float result; + memcpy(&result, &f32, sizeof(float)); + return result; +} + +float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { + int64_t n = safetensor_numel(t); + if (n <= 0) return NULL; + + float *out = malloc(n * sizeof(float)); + if (!out) return NULL; + + const void *data = safetensors_data(sf, t); + + switch (t->dtype) { + case DTYPE_F32: + memcpy(out, data, n * sizeof(float)); + break; + case DTYPE_BF16: { + const uint16_t *src = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]); + break; + } + default: + free(out); + return NULL; + } + return out; +} + +int safetensor_is_bf16(const safetensor_t *t) { + return t && t->dtype == DTYPE_BF16; +} + +uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t) { + if (!sf || !t || t->dtype != DTYPE_BF16) return NULL; + return (uint16_t *)safetensors_data(sf, t); +} + +void safetensor_print(const safetensor_t *t) { + const char *dtype_names[] = {"F32", "F16", "BF16", "I32", "I64", "BOOL"}; + const char *dtype_name = t->dtype >= 0 && t->dtype <= 5 ? + dtype_names[t->dtype] : "UNKNOWN"; + printf("%s: dtype=%s, shape=[", t->name, dtype_name); + for (int i = 0; i < t->ndim; i++) { + printf("%ld%s", (long)t->shape[i], i < t->ndim - 1 ? ", " : ""); + } + printf("]\n"); +} + +void safetensors_print_all(const safetensors_file_t *sf) { + printf("File: %s (%d tensors)\n", sf->path, sf->num_tensors); + for (int i = 0; i < sf->num_tensors; i++) safetensor_print(&sf->tensors[i]); +} + +/* ======================================================================== + * Multi-shard operations + * ======================================================================== */ + +multi_safetensors_t *multi_safetensors_open(const char *model_dir) { + multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); + if (!ms) return NULL; + + char path[4096]; + + /* Try single file first */ + snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); + safetensors_file_t *sf = safetensors_open(path); + if (sf) { + ms->shards[0] = sf; + ms->num_shards = 1; + return ms; + } + + /* Try multi-shard: model-00001-of-NNNNN.safetensors */ + for (int i = 1; i <= SAFETENSORS_MAX_SHARDS; i++) { + snprintf(path, sizeof(path), "%s/model-%05d-of-%05d.safetensors", + model_dir, i, i); /* placeholder - need to detect count */ + /* We don't know the total count yet, try common patterns */ + break; + } + + /* Scan directory for shard files */ + DIR *dir = opendir(model_dir); + if (!dir) { free(ms); return NULL; } + + struct dirent *entry; + char shard_names[SAFETENSORS_MAX_SHARDS][256]; + int n_shards = 0; + + while ((entry = readdir(dir)) != NULL && n_shards < SAFETENSORS_MAX_SHARDS) { + if (strncmp(entry->d_name, "model-", 6) == 0 && + strstr(entry->d_name, ".safetensors") != NULL) { + snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), + "%s", entry->d_name); + n_shards++; + } + } + closedir(dir); + + if (n_shards == 0) { + fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); + free(ms); + return NULL; + } + + /* Sort shard names to ensure consistent ordering */ + qsort(shard_names, n_shards, 256, (int(*)(const void*,const void*))strcmp); + + /* Open each shard */ + for (int i = 0; i < n_shards; i++) { + snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); + ms->shards[i] = safetensors_open(path); + if (!ms->shards[i]) { + fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); + multi_safetensors_close(ms); + return NULL; + } + } + ms->num_shards = n_shards; + return ms; +} + +void multi_safetensors_close(multi_safetensors_t *ms) { + if (!ms) return; + for (int i = 0; i < ms->num_shards; i++) { + safetensors_close(ms->shards[i]); + } + free(ms); +} + +const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, + const char *name, + safetensors_file_t **out_sf) { + for (int s = 0; s < ms->num_shards; s++) { + safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) { + if (out_sf) *out_sf = sf; + return &sf->tensors[i]; + } + } + } + return NULL; +} diff --git a/qwen_asr_safetensors.h b/qwen_asr_safetensors.h new file mode 100644 index 0000000..4cb09f7 --- /dev/null +++ b/qwen_asr_safetensors.h @@ -0,0 +1,77 @@ +/* + * qwen_asr_safetensors.h - Safetensors file format reader (multi-shard support) + * Adapted from voxtral-realtime project. + */ + +#ifndef QWEN_ASR_SAFETENSORS_H +#define QWEN_ASR_SAFETENSORS_H + +#include +#include + +#define SAFETENSORS_MAX_TENSORS 1024 +#define SAFETENSORS_MAX_SHARDS 8 + +typedef enum { + DTYPE_F32 = 0, + DTYPE_F16 = 1, + DTYPE_BF16 = 2, + DTYPE_I32 = 3, + DTYPE_I64 = 4, + DTYPE_BOOL = 5, + DTYPE_UNKNOWN = -1 +} safetensor_dtype_t; + +typedef struct { + char name[256]; + safetensor_dtype_t dtype; + int ndim; + int64_t shape[8]; + size_t data_offset; + size_t data_size; +} safetensor_t; + +typedef struct { + char *path; + void *data; + size_t file_size; + size_t header_size; + char *header_json; + int num_tensors; + safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; +} safetensors_file_t; + +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ +typedef struct { + safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; + int num_shards; +} multi_safetensors_t; + +/* Open a single safetensors file (memory-mapped) */ +safetensors_file_t *safetensors_open(const char *path); +void safetensors_close(safetensors_file_t *sf); + +/* Open model from directory (auto-detects single file or multi-shard) */ +multi_safetensors_t *multi_safetensors_open(const char *model_dir); +void multi_safetensors_close(multi_safetensors_t *ms); + +/* Find a tensor by name across all shards */ +const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, + const char *name, + safetensors_file_t **out_sf); + +/* Get raw pointer to tensor data (within mmap'd region) */ +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); + +/* Get tensor data as float32 (allocates, caller must free) */ +float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); + +/* Get direct pointer to bf16 data in mmap'd region (no copy) */ +uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t); + +int safetensor_is_bf16(const safetensor_t *t); +int64_t safetensor_numel(const safetensor_t *t); +void safetensor_print(const safetensor_t *t); +void safetensors_print_all(const safetensors_file_t *sf); + +#endif /* QWEN_ASR_SAFETENSORS_H */ diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c new file mode 100644 index 0000000..7bbc5db --- /dev/null +++ b/qwen_asr_tokenizer.c @@ -0,0 +1,650 @@ +/* + * qwen_asr_tokenizer.c - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * + * Supports: + * - decode token IDs to UTF-8 text (for ASR output assembly) + * - encode UTF-8 text to token IDs using vocab.json + merges.txt + */ + +#include "qwen_asr_tokenizer.h" +#include "qwen_asr_kernels.h" +#include +#include +#include +#include +#include + +/* ======================================================================== + * GPT-2 Bytes-to-Unicode Mapping + * ======================================================================== */ + +static int gpt2_byte_to_unicode[256]; +static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ +static int gpt2_mapping_initialized = 0; + +static void init_gpt2_mapping(void) { + if (gpt2_mapping_initialized) return; + + memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); + + int n = 0; + for (int b = 0; b < 256; b++) { + int is_normal = 0; + if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */ + if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */ + if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */ + + if (is_normal) gpt2_byte_to_unicode[b] = b; + else gpt2_byte_to_unicode[b] = 256 + n++; + } + + for (int b = 0; b < 256; b++) { + int cp = gpt2_byte_to_unicode[b]; + if (cp < 512) gpt2_unicode_to_byte[cp] = b; + } + + gpt2_mapping_initialized = 1; +} + +/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */ +static int utf8_encode_cp(int cp, char out[4]) { + if (cp < 0x80) { + out[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; +} + +/* + * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text. + * Returns allocated string (caller must free). + */ +static char *decode_gpt2_token(const char *token_str) { + init_gpt2_mapping(); + + size_t len = strlen(token_str); + unsigned char *bytes = (unsigned char *)malloc(len + 1); + if (!bytes) return NULL; + + int byte_count = 0; + const unsigned char *p = (const unsigned char *)token_str; + const unsigned char *end = p + len; + + while (p < end) { + int cp; + int nbytes; + + if ((*p & 0x80) == 0) { + cp = *p; + nbytes = 1; + } else if ((*p & 0xE0) == 0xC0) { + cp = (*p & 0x1F) << 6; + if (p + 1 < end) cp |= (p[1] & 0x3F); + nbytes = 2; + } else if ((*p & 0xF0) == 0xE0) { + cp = (*p & 0x0F) << 12; + if (p + 1 < end) cp |= (p[1] & 0x3F) << 6; + if (p + 2 < end) cp |= (p[2] & 0x3F); + nbytes = 3; + } else if ((*p & 0xF8) == 0xF0) { + cp = (*p & 0x07) << 18; + if (p + 1 < end) cp |= (p[1] & 0x3F) << 12; + if (p + 2 < end) cp |= (p[2] & 0x3F) << 6; + if (p + 3 < end) cp |= (p[3] & 0x3F); + nbytes = 4; + } else { + cp = *p; + nbytes = 1; + } + p += nbytes; + + if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) { + bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp]; + } else { + bytes[byte_count++] = '?'; + } + } + bytes[byte_count] = '\0'; + return (char *)bytes; +} + +/* ======================================================================== + * Simple JSON parser for vocab.json + * ======================================================================== */ + +static void skip_ws(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; +} + +static int parse_json_string(const char **p, char *out, size_t max_len) { + skip_ws(p); + if (**p != '"') return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') out[i++] = '\n'; + else if (**p == 't') out[i++] = '\t'; + else if (**p == '"') out[i++] = '"'; + else if (**p == '\\') out[i++] = '\\'; + else if (**p == '/') out[i++] = '/'; + else if (**p == 'u') { + (*p)++; + unsigned int cp = 0; + for (int j = 0; j < 4 && **p; j++, (*p)++) { + cp <<= 4; + char c = **p; + if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0'); + else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10); + } + if (cp < 0x80 && i + 1 < max_len) { + out[i++] = (char)cp; + } else if (cp < 0x800 && i + 2 < max_len) { + out[i++] = (char)(0xC0 | (cp >> 6)); + out[i++] = (char)(0x80 | (cp & 0x3F)); + } else if (i + 3 < max_len) { + out[i++] = (char)(0xE0 | (cp >> 12)); + out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[i++] = (char)(0x80 | (cp & 0x3F)); + } + continue; + } else { + out[i++] = **p; + } + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') return -1; + (*p)++; + return 0; +} + +static int64_t parse_json_int(const char **p) { + skip_ws(p); + int neg = 0; + if (**p == '-') { neg = 1; (*p)++; } + int64_t val = 0; + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + return neg ? -val : val; +} + +/* ======================================================================== + * String->int hash map (open addressing) + * ======================================================================== */ + +typedef struct { + char *key; /* NULL means empty slot */ + int value; +} str_int_entry_t; + +static uint64_t fnv1a_hash(const char *s) { + uint64_t h = 1469598103934665603ULL; + while (*s) { + h ^= (unsigned char)*s++; + h *= 1099511628211ULL; + } + return h; +} + +static int next_pow2(int x) { + int p = 1; + while (p < x) p <<= 1; + return p; +} + +static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { + if (!map || cap <= 0 || !key) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) { + map[idx].key = key; + map[idx].value = value; + return 0; + } + if (strcmp(map[idx].key, key) == 0) { + map[idx].value = value; + return 0; + } + } + return -1; +} + +static int map_get(const str_int_entry_t *map, int cap, const char *key) { + if (!map || cap <= 0 || !key) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) return -1; + if (strcmp(map[idx].key, key) == 0) return map[idx].value; + } + return -1; +} + +/* ======================================================================== + * BPE helpers + * ======================================================================== */ + +static int utf8_char_len(unsigned char c) { + if ((c & 0x80) == 0) return 1; + if ((c & 0xE0) == 0xC0) return 2; + if ((c & 0xF0) == 0xE0) return 3; + if ((c & 0xF8) == 0xF0) return 4; + return 1; +} + +static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { + *out_syms = NULL; + *out_n = 0; + if (!s || !*s) return 0; + + int n = 0; + for (const unsigned char *p = (const unsigned char *)s; *p; ) { + int l = utf8_char_len(*p); + p += l; + n++; + } + if (n <= 0) return 0; + + char **syms = (char **)calloc((size_t)n, sizeof(char *)); + if (!syms) return -1; + + int i = 0; + for (const unsigned char *p = (const unsigned char *)s; *p; ) { + int l = utf8_char_len(*p); + syms[i] = (char *)malloc((size_t)l + 1); + if (!syms[i]) { + for (int j = 0; j < i; j++) free(syms[j]); + free(syms); + return -1; + } + memcpy(syms[i], p, (size_t)l); + syms[i][l] = '\0'; + p += l; + i++; + } + + *out_syms = syms; + *out_n = n; + return 0; +} + +static char *str_concat2(const char *a, const char *b) { + size_t la = strlen(a), lb = strlen(b); + char *r = (char *)malloc(la + lb + 1); + if (!r) return NULL; + memcpy(r, a, la); + memcpy(r + la, b, lb); + r[la + lb] = '\0'; + return r; +} + +static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { + if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; + + size_t la = strlen(a), lb = strlen(b); + char *pair = (char *)malloc(la + 1 + lb + 1); + if (!pair) return INT_MAX; + memcpy(pair, a, la); + pair[la] = ' '; + memcpy(pair + la + 1, b, lb); + pair[la + 1 + lb] = '\0'; + + int rank = map_get((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, pair); + free(pair); + return rank >= 0 ? rank : INT_MAX; +} + +static int append_id(int **arr, int *n, int *cap, int id) { + if (*n >= *cap) { + int new_cap = (*cap == 0) ? 16 : (*cap * 2); + int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); + if (!tmp) return -1; + *arr = tmp; + *cap = new_cap; + } + (*arr)[(*n)++] = id; + return 0; +} + +/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */ +static char *text_to_bpe_unicode(const char *text) { + init_gpt2_mapping(); + size_t len = strlen(text); + /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ + char *out = (char *)malloc(len * 3 + 1); + if (!out) return NULL; + + size_t w = 0; + for (size_t i = 0; i < len; i++) { + int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; + char tmp[4]; + int n = utf8_encode_cp(cp, tmp); + for (int j = 0; j < n; j++) out[w++] = tmp[j]; + } + out[w] = '\0'; + return out; +} + +/* Encode one mapped BPE unicode string to token IDs. */ +static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { + *out_ids = NULL; + *out_n = 0; + if (!mapped || !*mapped) return 0; + + char **syms = NULL; + int n_syms = 0; + if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1; + if (n_syms <= 0) { + free(syms); + return 0; + } + + while (n_syms > 1) { + int best_rank = INT_MAX; + int best_i = -1; + for (int i = 0; i < n_syms - 1; i++) { + int r = merge_rank(tok, syms[i], syms[i + 1]); + if (r < best_rank) { + best_rank = r; + best_i = i; + } + } + if (best_i < 0 || best_rank == INT_MAX) break; + + char *merged = str_concat2(syms[best_i], syms[best_i + 1]); + if (!merged) { + for (int i = 0; i < n_syms; i++) free(syms[i]); + free(syms); + return -1; + } + free(syms[best_i]); + free(syms[best_i + 1]); + syms[best_i] = merged; + for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1]; + n_syms--; + } + + int *ids = NULL; + int n_ids = 0, cap = 0; + for (int i = 0; i < n_syms; i++) { + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); + if (id < 0) { + /* Should not happen with valid vocab + merges + byte-level mapping. */ + for (int k = 0; k < n_syms; k++) free(syms[k]); + free(syms); + free(ids); + return -1; + } + if (append_id(&ids, &n_ids, &cap, id) != 0) { + for (int k = 0; k < n_syms; k++) free(syms[k]); + free(syms); + free(ids); + return -1; + } + } + + for (int i = 0; i < n_syms; i++) free(syms[i]); + free(syms); + + *out_ids = ids; + *out_n = n_ids; + return 0; +} + +static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { + const char *slash = strrchr(vocab_path, '/'); + if (!slash) { + if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1; + return 0; + } + size_t dir_len = (size_t)(slash - vocab_path); + if (dir_len + 12 >= out_cap) return -1; + memcpy(out_path, vocab_path, dir_len); + out_path[dir_len] = '\0'; + snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); + return 0; +} + +static void trim_newline(char *line) { + size_t n = strlen(line); + while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0'; +} + +static int parse_merge_pair(char *line, char **a, char **b) { + *a = NULL; + *b = NULL; + trim_newline(line); + if (line[0] == '\0') return -1; + if (line[0] == '#') return -1; + + char *sp = strchr(line, ' '); + if (!sp) return -1; + *sp = '\0'; + char *p2 = sp + 1; + while (*p2 == ' ') p2++; + if (*line == '\0' || *p2 == '\0') return -1; + *a = line; + *b = p2; + return 0; +} + +static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { + FILE *f = fopen(merges_path, "rb"); + if (!f) return -1; + + int n_pairs = 0; + char line[8192]; + while (fgets(line, sizeof(line), f)) { + char *a = NULL, *b = NULL; + if (parse_merge_pair(line, &a, &b) == 0) n_pairs++; + } + if (n_pairs <= 0) { + fclose(f); + return -1; + } + + tok->merge_map_cap = next_pow2(n_pairs * 2); + tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t)); + if (!tok->merge_map) { + fclose(f); + tok->merge_map_cap = 0; + return -1; + } + + rewind(f); + int rank = 0; + while (fgets(line, sizeof(line), f)) { + char *a = NULL, *b = NULL; + if (parse_merge_pair(line, &a, &b) != 0) continue; + + size_t la = strlen(a), lb = strlen(b); + char *key = (char *)malloc(la + 1 + lb + 1); + if (!key) continue; + memcpy(key, a, la); + key[la] = ' '; + memcpy(key + la + 1, b, lb); + key[la + 1 + lb] = '\0'; + + if (map_insert((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, key, rank) != 0) { + free(key); + } + rank++; + } + fclose(f); + return 0; +} + +/* ======================================================================== + * Public API + * ======================================================================== */ + +qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { + FILE *f = fopen(vocab_json_path, "rb"); + if (!f) { + fprintf(stderr, "qwen_tokenizer_load: cannot open %s\n", vocab_json_path); + return NULL; + } + + fseek(f, 0, SEEK_END); + long file_size = ftell(f); + fseek(f, 0, SEEK_SET); + + char *json = (char *)malloc((size_t)file_size + 1); + if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) { + fclose(f); + free(json); + return NULL; + } + fclose(f); + json[file_size] = '\0'; + + int max_id = 0; + const char *p = json; + skip_ws(&p); + if (*p != '{') { + free(json); + return NULL; + } + p++; + + const char *saved = p; + while (*p && *p != '}') { + skip_ws(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char key[4096]; + if (parse_json_string(&p, key, sizeof(key)) != 0) break; + skip_ws(&p); + if (*p != ':') break; + p++; + int64_t id = parse_json_int(&p); + if (id > max_id) max_id = (int)id; + } + + int vocab_size = max_id + 1; + qwen_tokenizer_t *tok = (qwen_tokenizer_t *)calloc(1, sizeof(qwen_tokenizer_t)); + if (!tok) { + free(json); + return NULL; + } + tok->vocab_size = vocab_size; + tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); + tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); + if (!tok->id_to_text || !tok->id_to_bpe) { + qwen_tokenizer_free(tok); + free(json); + return NULL; + } + + p = saved; + while (*p && *p != '}') { + skip_ws(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char key[4096]; + if (parse_json_string(&p, key, sizeof(key)) != 0) break; + skip_ws(&p); + if (*p != ':') break; + p++; + int64_t id = parse_json_int(&p); + + if (id >= 0 && id < vocab_size) { + free(tok->id_to_bpe[id]); + free(tok->id_to_text[id]); + tok->id_to_bpe[id] = strdup(key); + tok->id_to_text[id] = decode_gpt2_token(key); + } + } + free(json); + + int n_vocab_entries = 0; + for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++; + tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); + tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); + if (!tok->vocab_map) { + qwen_tokenizer_free(tok); + return NULL; + } + for (int i = 0; i < vocab_size; i++) { + if (tok->id_to_bpe[i]) { + map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i); + } + } + + char merges_path[1024]; + if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { + if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { + fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", + merges_path); + } + } + + return tok; +} + +const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { + if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; + return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; +} + +int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !text || text[0] == '\0') return NULL; + + char *mapped = text_to_bpe_unicode(text); + if (!mapped) return NULL; + + int *ids = NULL; + int n_ids = 0; + if (encode_bpe_word(tok, mapped, &ids, &n_ids) != 0) { + free(mapped); + free(ids); + return NULL; + } + free(mapped); + + if (out_n_tokens) *out_n_tokens = n_ids; + return ids; +} + +void qwen_tokenizer_free(qwen_tokenizer_t *tok) { + if (!tok) return; + + if (tok->id_to_text) { + for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]); + free(tok->id_to_text); + } + if (tok->id_to_bpe) { + for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]); + free(tok->id_to_bpe); + } + + if (tok->merge_map) { + str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; + for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key); + free(tok->merge_map); + } + free(tok->vocab_map); + free(tok); +} diff --git a/qwen_asr_tokenizer.h b/qwen_asr_tokenizer.h new file mode 100644 index 0000000..0cea192 --- /dev/null +++ b/qwen_asr_tokenizer.h @@ -0,0 +1,38 @@ +/* + * qwen_asr_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * + * Supports: + * - decode token ID -> UTF-8 text + * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) + */ + +#ifndef QWEN_ASR_TOKENIZER_H +#define QWEN_ASR_TOKENIZER_H + +typedef struct { + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + int vocab_size; + + /* Internal hash maps (opaque to callers) */ + void *vocab_map; + int vocab_map_cap; + void *merge_map; + int merge_map_cap; +} qwen_tokenizer_t; + +/* Load tokenizer from vocab.json in model directory */ +qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); + +/* Decode a single token ID to text. Returns pointer to internal string. */ +const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); + +/* Encode UTF-8 text into token IDs using BPE. + * Returns malloc'd array of token IDs and sets *out_n_tokens. + * Returns NULL on error (and sets *out_n_tokens to 0). */ +int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); + +/* Free tokenizer */ +void qwen_tokenizer_free(qwen_tokenizer_t *tok); + +#endif /* QWEN_ASR_TOKENIZER_H */ -- cgit v1.3.1