aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-03-28 21:27:58 +0100
committerOnur Gündüz <ogunduz@gmail.com>2026-03-28 23:10:03 +0100
commitc47e900b94d18cca1bf600e58f10dbcc2e34c1f7 (patch)
treebf7a4247c528841abe71bbbd408eda3ff71b1152
parent34e9eefde7870edcc318209738cabb17c71b50ed (diff)
downloadffwd-c47e900b94d18cca1bf600e58f10dbcc2e34c1f7.tar.gz
ffwd-c47e900b94d18cca1bf600e58f10dbcc2e34c1f7.zip
Refactor and enhance pplx_embed and pplx_embed_mlx for improved model support
- Updated `pplx_embed.h` to support new embedding models: `pplx-embed-v1-4b` and `pplx-embed-context-v1-4b`. - Simplified model configuration structure and constants for better maintainability. - Enhanced comments for clarity on model architecture and configuration. - Modified `pplx_embed_mlx.c` to streamline loading of model weights and configuration from `config.json`. - Improved memory management and error handling in the MLX backend. - Updated embedding computation to support dynamic hidden sizes based on model configuration. - Added `pplx_mlx_config` function to retrieve model configuration details. - Updated README.md - Added .gitignore
-rw-r--r--.gitignore2
-rw-r--r--Makefile13
-rw-r--r--README.md88
-rw-r--r--main.c479
-rw-r--r--pplx_embed.c459
-rw-r--r--pplx_embed.h147
-rw-r--r--pplx_embed_mlx.c478
-rw-r--r--pplx_embed_mlx.h15
-rw-r--r--qwen_asr_kernels.c5
-rw-r--r--qwen_asr_safetensors.c5
-rw-r--r--qwen_asr_tokenizer.c8
11 files changed, 830 insertions, 869 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6a85c96
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.o
+pplx_embed
diff --git a/Makefile b/Makefile
index ee6b813..bb3815e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,4 @@
-# pplx_embed - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
-# Makefile
-
-CC = gcc
+CC = gcc
CFLAGS_BASE = -Wall -Wextra -O3 -march=native -ffast-math
LDFLAGS = -lm -lpthread
@@ -27,7 +24,7 @@ TARGET = pplx_embed
all: help
help:
- @echo "pplx_embed - Pure C inference for pplx-embed-v1-0.6b"
+ @echo "pplx_embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models"
@echo ""
@echo "Build targets:"
@echo " make blas Build with BLAS acceleration (CPU)"
@@ -35,7 +32,7 @@ help:
@echo " make debug Debug build with AddressSanitizer"
@echo " make clean Remove build artifacts"
@echo ""
- @echo "Usage after build:"
+ @echo "Usage:"
@echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\""
@echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)"
@@ -52,8 +49,6 @@ endif
blas:
$(MAKE) clean
$(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
- @echo ""
- @echo "Built: $(TARGET) (CPU/BLAS)"
# =============================================================================
# MLX build (Apple Silicon GPU - uses mlx-c pure C API)
@@ -75,8 +70,6 @@ endif
mlx:
$(MAKE) clean
$(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
- @echo ""
- @echo "Built: $(TARGET) (CPU/BLAS + MLX GPU, use --mlx flag)"
# =============================================================================
# Debug build
diff --git a/README.md b/README.md
index 613ce47..bbc205a 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,87 @@
-# pplx-embed-v1
+# pplx-embed
+
+Pure C inference for [pplx-embed](https://research.perplexity.ai/articles/pplx-embed-state-of-the-art-embedding-models-for-web-scale-retrieval) embedding models by [Perplexity AI](https://www.perplexity.ai/). Supports safetensors and hardware-accelerated execution (BLAS/mlx-c) for generating float32 vectors.
+
+Based on math kernels from [antirez](https://github.com/antirez)'s [qwen-asr](https://github.com/antirez/qwen-asr).
+
+## Supported models
+
+All [pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed-v1-68126b2e7cc987cd820ebbbd) models are supported.
+
+| Model | Dims | Params | Size |
+|---|---|---|---|
+| [pplx-embed-v1-0.6b](https://huggingface.co/perplexity-ai/pplx-embed-v1-0.6b) | 1024 | 0.6B | ~2.5 GB |
+| [pplx-embed-v1-4b](https://huggingface.co/perplexity-ai/pplx-embed-v1-4b) | 2560 | 4B | ~16 GB |
+| [pplx-embed-context-v1-0.6B](https://huggingface.co/perplexity-ai/pplx-embed-context-v1-0.6B) | 1024 | 0.6B | ~2.5 GB |
+| [pplx-embed-context-v1-4B](https://huggingface.co/perplexity-ai/pplx-embed-context-v1-4B) | 2560 | 4B | ~16 GB |
+
+## Quickstart
+
+```bash
+make mlx # Apple Silicon (fastest)
+# or: make blas # CPU with BLAS (any platform)
+
+./pplx_embed -d /path/to/model "query: what is the capital of France?"
+# prints 1024 (or 2560) floats to stdout
+
+./pplx_embed -d /path/to/model \
+ "query: what is the capital of France?" \
+ "document: Paris is the capital of France." \
+ "document: Berlin is the capital of Germany."
+# prints cosine similarity matrix
+```
+
+### Daemon mode
+
+Start the process once and feed texts through stdin. The model stays loaded in memory.
+
+```bash
+# One embedding per line, output as JSON
+./pplx_embed -d /path/to/model --daemon --mlx <<'EOF'
+query: what is the capital of France?
+document: Paris is the capital of France.
+document: Berlin is the capital of Germany.
+document: Istanbul is the capital of Germany.
+EOF
+```
+
+Output (one JSON object per line):
+
+```json
+{"embedding":[0.0231,-0.0412,...], "dim":1024, "tokens":9, "ms":15.2}
+{"embedding":[0.0187,-0.0339,...], "dim":1024, "tokens":9, "ms":14.8}
+...
+```
+
+## Building
+
+```bash
+make blas # CPU with BLAS (Accelerate on macOS, OpenBLAS on Linux)
+make mlx # Apple Silicon GPU via mlx-c (recommended on Apple Silicon)
+make debug # Debug build with AddressSanitizer
+```
+
+For `make mlx`, install MLX first: `brew install mlx mlx-c`
+
+For `make blas` on Linux: `sudo apt install libopenblas-dev`
+
+## Usage
+
+```
+./pplx_embed -d <model_dir> [options] [text...]
+
+Options:
+ -d <dir> Model directory (required)
+ --mlx Use Apple MLX GPU backend
+ --daemon Read texts from stdin, write JSON embeddings to stdout
+ -t <n> CPU thread count (default: all cores)
+ -e Print raw embedding vectors (with multiple texts)
+ -v Verbose (-vv for debug)
+```
+
+**Single text**: prints the embedding vector as space-separated floats.
+
+**Multiple texts**: prints a cosine similarity matrix.
+
+**Daemon mode** (`--daemon`): reads one text per line from stdin, writes one JSON object per line to stdout. EOF or empty line exits.
+
diff --git a/main.c b/main.c
index cc2e492..9236a47 100644
--- a/main.c
+++ b/main.c
@@ -1,25 +1,4 @@
-/*
- * main.c - pplx_embed command-line tool
- *
- * Usage:
- * pplx_embed -d <model_dir> [options] <text> [<text2> ...]
- * pplx_embed -d <model_dir> [options] (read lines from stdin)
- *
- * Options:
- * -d <dir> Model directory (required)
- * -t <n> Number of threads (default: all CPU cores)
- * -e Print raw embedding vector(s) instead of similarity matrix
- * -v Verbose output (use -vv for debug)
- *
- * Output:
- * 1 text : prints the 1024-dim embedding as space-separated floats
- * 2+ texts: prints cosine-similarity matrix between every pair,
- * then optionally the individual embeddings if -e is given
- *
- * Example:
- * ./pplx_embed -d /path/to/model "hello world" "hi there"
- * ./pplx_embed -d /path/to/model -e "query: what is AI?"
- */
+/* main.c - pplx_embed command-line tool */
#include "pplx_embed.h"
#include "qwen_asr_kernels.h"
@@ -35,10 +14,6 @@
#include <math.h>
#include <time.h>
-/* ========================================================================
- * Helpers
- * ======================================================================== */
-
static double now_ms(void)
{
struct timespec ts;
@@ -46,318 +21,324 @@ static double now_ms(void)
return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
}
-static void print_embedding(const float *emb, int dim)
-{
- for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(' ');
- printf("%.8f", (double)emb[i]);
- }
- putchar('\n');
-}
-
static void print_usage(const char *prog)
{
fprintf(stderr,
"Usage: %s -d <model_dir> [options] [text...]\n"
"\n"
"Options:\n"
- " -d <dir> Model directory (model.safetensors + vocab.json + merges.txt)\n"
- " -t <n> Number of threads (default: all CPU cores, CPU backend)\n"
- " --mlx Use Apple MLX GPU backend (Apple Silicon only)\n"
- " -e Always print raw embedding vector(s)\n"
- " -v Verbose (repeat for more: -vv)\n"
- " -h Show this help\n"
+ " -d <dir> Model directory (required)\n"
+ " --mlx Use Apple MLX GPU backend\n"
+ " --daemon Read lines from stdin, write JSON embeddings to stdout\n"
+ " -t <n> CPU threads (default: all cores)\n"
+ " -e Print raw embeddings (with multiple texts)\n"
+ " -v Verbose (-vv for debug)\n"
+ " -h Show this help\n"
"\n"
- "With 1 text: prints the 1024-dim embedding vector\n"
- "With 2+ texts: prints cosine similarity between each pair\n"
- " (add -e to also print the individual embeddings)\n"
+ "Modes:\n"
+ " 1 text arg Print embedding as space-separated floats\n"
+ " 2+ text args Print cosine similarity matrix\n"
+ " no args Batch: read all stdin lines, then similarity matrix\n"
+ " --daemon Streaming: read stdin lines, write JSON per line\n"
"\n"
- "If no text arguments are given, reads one text per line from stdin.\n"
- "\n"
- "Example:\n"
- " %s -d ./model \"query: what is AI?\" \"document: AI is artificial intelligence\"\n"
- " %s -d ./model --mlx \"query: what is AI?\"\n",
+ "Examples:\n"
+ " %s -d ./model \"query: what is AI?\"\n"
+ " %s -d ./model --mlx --daemon < texts.txt\n",
prog, prog, prog);
}
/* ========================================================================
- * Process one text: tokenize -> embed, returns malloc'd float[dim]
+ * Embed one text: tokenize > forward > return float[dim]
* ======================================================================== */
-static float *process_text(pplx_ctx_t *ctx,
+typedef struct {
+ pplx_ctx_t *ctx;
#ifdef USE_MLX
- pplx_mlx_ctx_t *mlx_ctx,
+ pplx_mlx_ctx_t *mlx_ctx;
#endif
- qwen_tokenizer_t *tok,
- const char *text,
- int print_tokens)
+ qwen_tokenizer_t *tok;
+ int dim;
+} engine_t;
+
+typedef struct {
+ float *emb; /* malloc'd float[dim], caller frees */
+ int n_tokens;
+ double ms;
+} embed_result_t;
+
+static embed_result_t embed_text(engine_t *e, const char *text)
{
- /* Tokenize */
- int n_tokens = 0;
- int *token_ids = qwen_tokenizer_encode(tok, text, &n_tokens);
- if (!token_ids || n_tokens == 0) {
- fprintf(stderr, "pplx_embed: tokenization failed for: %s\n", text);
- free(token_ids);
- return NULL;
+ embed_result_t r = {0};
+
+ int n = 0;
+ int *ids = qwen_tokenizer_encode(e->tok, text, &n);
+ if (!ids || n == 0) {
+ fprintf(stderr, "tokenization failed: %s\n", text);
+ free(ids);
+ return r;
}
- if (print_tokens || pplx_verbose >= 1) {
- fprintf(stderr, "tokens (%d): ", n_tokens);
- for (int i = 0; i < n_tokens && i < 20; i++)
- fprintf(stderr, "%d ", token_ids[i]);
- if (n_tokens > 20) fprintf(stderr, "...");
+ if (pplx_verbose >= 1) {
+ fprintf(stderr, "tokens (%d): ", n);
+ for (int i = 0; i < n && i < 20; i++) fprintf(stderr, "%d ", ids[i]);
+ if (n > 20) fprintf(stderr, "...");
fprintf(stderr, "\n");
}
- /* Embed */
- double t0 = now_ms();
- float *emb = NULL;
+ double t0 = now_ms();
#ifdef USE_MLX
- if (mlx_ctx)
- emb = pplx_mlx_embed(mlx_ctx, token_ids, n_tokens);
+ if (e->mlx_ctx)
+ r.emb = pplx_mlx_embed(e->mlx_ctx, ids, n);
else
#endif
- emb = pplx_embed(ctx, token_ids, n_tokens);
- double dt = now_ms() - t0;
- free(token_ids);
-
- if (!emb) {
- fprintf(stderr, "pplx_embed: forward pass failed\n");
- return NULL;
- }
+ r.emb = pplx_embed(e->ctx, ids, n);
+ r.ms = now_ms() - t0;
+ r.n_tokens = n;
+ free(ids);
+ if (!r.emb) fprintf(stderr, "forward pass failed\n");
if (pplx_verbose >= 1)
- fprintf(stderr, "embed: %d tokens in %.1f ms\n", n_tokens, dt);
+ fprintf(stderr, "embed: %d tokens in %.1f ms\n", r.n_tokens, r.ms);
- return emb;
+ return r;
}
/* ========================================================================
- * Main
+ * Output helpers
* ======================================================================== */
-int main(int argc, char *argv[])
+static void print_embedding_raw(const float *emb, int dim)
{
- const char *model_dir = NULL;
- int n_threads = 0; /* 0 = auto */
- int print_embs = 0;
- int verbose = 0;
- int use_mlx = 0;
-
- /* ---- Parse flags ---- */
- int arg_start = 1;
- while (arg_start < argc && argv[arg_start][0] == '-') {
- const char *flag = argv[arg_start];
- if (strcmp(flag, "-d") == 0) {
- if (arg_start + 1 >= argc) {
- fprintf(stderr, "pplx_embed: -d requires an argument\n");
- return 1;
- }
- model_dir = argv[++arg_start];
- } else if (strcmp(flag, "-t") == 0) {
- if (arg_start + 1 >= argc) {
- fprintf(stderr, "pplx_embed: -t requires an argument\n");
- return 1;
- }
- n_threads = atoi(argv[++arg_start]);
- } else if (strcmp(flag, "-e") == 0) {
- print_embs = 1;
- } else if (strcmp(flag, "--mlx") == 0) {
-#ifdef USE_MLX
- use_mlx = 1;
-#else
- fprintf(stderr, "pplx_embed: --mlx not available (build with: make mlx)\n");
- return 1;
-#endif
- } else if (strcmp(flag, "-v") == 0) {
- verbose++;
- } else if (strcmp(flag, "-vv") == 0) {
- verbose = 2;
- } else if (strcmp(flag, "-h") == 0 || strcmp(flag, "--help") == 0) {
- print_usage(argv[0]);
- return 0;
- } else {
- /* Stop flag parsing on first non-flag argument */
- break;
- }
- arg_start++;
+ for (int i = 0; i < dim; i++) {
+ if (i > 0) putchar(' ');
+ printf("%.8f", (double)emb[i]);
}
+ putchar('\n');
+}
- if (!model_dir || model_dir[0] == '\0') {
- fprintf(stderr, "pplx_embed: model directory required (-d <dir>)\n"
- " Hint: if using MODEL=.. ./pplx_embed -d \"$MODEL\",\n"
- " make sure MODEL is exported or use: MODEL=.. && ./pplx_embed -d \"$MODEL\"\n");
- print_usage(argv[0]);
- return 1;
+static void print_embedding_json(const float *emb, int dim, int n_tokens, double ms)
+{
+ printf("{\"embedding\":[");
+ for (int i = 0; i < dim; i++) {
+ if (i > 0) putchar(',');
+ printf("%.8f", (double)emb[i]);
}
+ printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f}\n", dim, n_tokens, ms);
+ fflush(stdout);
+}
- /* ---- Set verbosity ---- */
- pplx_verbose = verbose;
- qwen_verbose = verbose; /* kernels share the same flag name */
-
- /* ---- Set thread count (CPU backend only) ---- */
- if (!use_mlx) {
- if (n_threads <= 0)
- n_threads = qwen_get_num_cpus();
- qwen_set_threads(n_threads);
- if (verbose >= 1)
- fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else {
- if (verbose >= 1)
- fprintf(stderr, "Using MLX GPU backend\n");
- }
+/* ========================================================================
+ * Daemon mode: read lines from stdin, write JSON to stdout
+ * ======================================================================== */
- /* ---- Load tokenizer ---- */
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+static int run_daemon(engine_t *e)
+{
+ char line[65536];
- double t0_load = now_ms();
- qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
- if (!tok) {
- fprintf(stderr, "pplx_embed: failed to load tokenizer from %s\n", vocab_path);
- return 1;
- }
- if (verbose >= 1)
- fprintf(stderr, "Tokenizer loaded in %.0f ms\n", now_ms() - t0_load);
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "daemon: ready, reading from stdin\n");
- /* ---- Load model ---- */
- pplx_ctx_t *ctx = NULL;
-#ifdef USE_MLX
- pplx_mlx_ctx_t *mlx_ctx = NULL;
-#endif
+ while (fgets(line, sizeof(line), stdin)) {
+ /* Strip newline */
+ size_t len = strlen(line);
+ while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r'))
+ line[--len] = '\0';
+ if (len == 0) continue;
- double t0_model = now_ms();
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "daemon: \"%.*s%s\"\n",
+ (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
-#ifdef USE_MLX
- if (use_mlx) {
- mlx_ctx = pplx_mlx_load(model_dir);
- if (!mlx_ctx) {
- fprintf(stderr, "pplx_embed: failed to load MLX model from %s\n", model_dir);
- qwen_tokenizer_free(tok);
- return 1;
- }
- } else
-#endif
- {
- ctx = pplx_load(model_dir);
- if (!ctx) {
- fprintf(stderr, "pplx_embed: failed to load model from %s\n", model_dir);
- qwen_tokenizer_free(tok);
- return 1;
+ embed_result_t r = embed_text(e, line);
+ if (r.emb) {
+ print_embedding_json(r.emb, e->dim, r.n_tokens, r.ms);
+ free(r.emb);
+ } else {
+ /* Output an error line so the consumer stays in sync */
+ printf("{\"error\":\"embedding failed\"}\n");
+ fflush(stdout);
}
}
- if (verbose >= 1)
- fprintf(stderr, "Model loaded in %.0f ms%s\n", now_ms() - t0_model,
- use_mlx ? " (MLX)" : "");
- int dim = PPLX_HIDDEN_SIZE;
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "daemon: stdin EOF\n");
- /* ---- Gather input texts ---- */
- /* texts[] and embeddings[] are grown dynamically */
- int n_texts = 0;
- int texts_cap = 0;
- char **texts = NULL;
- float **embeddings = NULL;
+ return 0;
+}
- /* Helper to append a text */
-#define APPEND_TEXT(s) do { \
- if (n_texts == texts_cap) { \
- texts_cap = texts_cap ? texts_cap * 2 : 8; \
- texts = realloc(texts, texts_cap * sizeof(char *)); \
- embeddings = realloc(embeddings, texts_cap * sizeof(float *)); \
- if (!texts || !embeddings) { \
- fprintf(stderr, "pplx_embed: out of memory\n"); return 1; \
- } \
- } \
- texts[n_texts] = strdup(s); \
- embeddings[n_texts] = NULL; \
- n_texts++; \
+/* ========================================================================
+ * Batch mode: embed args or stdin lines, then print similarity or vectors
+ * ======================================================================== */
+
+static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int print_embs)
+{
+ int n_texts = 0, cap = 0;
+ char **texts = NULL;
+ float **embs = NULL;
+
+#define APPEND(s) do { \
+ if (n_texts == cap) { \
+ cap = cap ? cap * 2 : 8; \
+ texts = realloc(texts, cap * sizeof(char *)); \
+ embs = realloc(embs, cap * sizeof(float *)); \
+ if (!texts || !embs) { fprintf(stderr, "OOM\n"); return 1; } \
+ } \
+ texts[n_texts] = strdup(s); \
+ embs[n_texts] = NULL; \
+ n_texts++; \
} while (0)
if (arg_start < argc) {
- /* Text arguments on command line */
- for (int i = arg_start; i < argc; i++)
- APPEND_TEXT(argv[i]);
+ for (int i = arg_start; i < argc; i++) APPEND(argv[i]);
} else {
- /* Read lines from stdin */
char line[65536];
while (fgets(line, sizeof(line), stdin)) {
- /* Strip trailing newline */
size_t l = strlen(line);
while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
line[--l] = '\0';
if (l == 0) continue;
- APPEND_TEXT(line);
+ APPEND(line);
}
}
+#undef APPEND
if (n_texts == 0) {
- fprintf(stderr, "pplx_embed: no input texts\n");
- pplx_free(ctx);
- qwen_tokenizer_free(tok);
+ fprintf(stderr, "no input texts\n");
+ free(texts); free(embs);
return 1;
}
- /* ---- Embed all texts ---- */
+ int dim = e->dim;
+
for (int i = 0; i < n_texts; i++) {
- if (verbose >= 1)
+ if (pplx_verbose >= 1)
fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
-
- embeddings[i] = process_text(ctx,
-#ifdef USE_MLX
- mlx_ctx,
-#endif
- tok, texts[i],
- /*print_tokens=*/ verbose >= 2);
- if (!embeddings[i]) {
- /* Error already printed */
- goto cleanup;
- }
+ embed_result_t r = embed_text(e, texts[i]);
+ embs[i] = r.emb;
+ if (!embs[i]) goto done;
}
- /* ---- Output ---- */
if (n_texts == 1) {
- /* Single text: always print the embedding */
- print_embedding(embeddings[0], dim);
+ print_embedding_raw(embs[0], dim);
} else {
- /* Multiple texts: print cosine similarity matrix */
printf("Cosine similarity matrix (%d texts):\n", n_texts);
-
- /* Header */
printf("%-4s", "");
for (int j = 0; j < n_texts; j++) printf(" [%d] ", j);
printf("\n");
-
for (int i = 0; i < n_texts; i++) {
printf("[%d] ", i);
- for (int j = 0; j < n_texts; j++) {
- float sim = pplx_cosine_similarity(embeddings[i], embeddings[j], dim);
- printf(" %.4f", (double)sim);
- }
+ for (int j = 0; j < n_texts; j++)
+ printf(" %.4f", (double)pplx_cosine_similarity(embs[i], embs[j], dim));
printf(" \"%s\"\n", texts[i]);
}
-
- /* Print raw embeddings if requested */
if (print_embs) {
printf("\nEmbeddings:\n");
for (int i = 0; i < n_texts; i++) {
printf("[%d] ", i);
- print_embedding(embeddings[i], dim);
+ print_embedding_raw(embs[i], dim);
}
}
}
-cleanup:
- for (int i = 0; i < n_texts; i++) {
- free(texts[i]);
- free(embeddings[i]);
+done:
+ for (int i = 0; i < n_texts; i++) { free(texts[i]); free(embs[i]); }
+ free(texts); free(embs);
+ return 0;
+}
+
+/* ========================================================================
+ * Main
+ * ======================================================================== */
+
+int main(int argc, char *argv[])
+{
+ const char *model_dir = NULL;
+ int n_threads = 0;
+ int print_embs = 0;
+ int verbose = 0;
+ int use_mlx = 0;
+ int daemon = 0;
+
+ int arg_start = 1;
+ while (arg_start < argc && argv[arg_start][0] == '-') {
+ const char *f = argv[arg_start];
+ if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
+ else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); }
+ else if (!strcmp(f, "-e")) { print_embs = 1; }
+ else if (!strcmp(f, "-v")) { verbose++; }
+ else if (!strcmp(f, "-vv")) { verbose = 2; }
+ else if (!strcmp(f, "--daemon")){ daemon = 1; }
+ else if (!strcmp(f, "--mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+#else
+ fprintf(stderr, "--mlx not available (build with: make mlx)\n"); return 1;
+#endif
+ }
+ else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); return 0; }
+ else break;
+ arg_start++;
+ }
+
+ if (!model_dir || !model_dir[0]) {
+ fprintf(stderr, "model directory required (-d <dir>)\n");
+ print_usage(argv[0]);
+ return 1;
+ }
+
+ pplx_verbose = verbose;
+ qwen_verbose = verbose;
+
+ /* Threads (CPU backend) */
+ if (!use_mlx) {
+ if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ qwen_set_threads(n_threads);
+ if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ } else {
+ if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ }
+
+ /* Tokenizer */
+ char vocab_path[1024];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+ double t0 = now_ms();
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; }
+ if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
+
+ /* Model */
+ engine_t e = {0};
+ e.tok = tok;
+ t0 = now_ms();
+
+#ifdef USE_MLX
+ if (use_mlx) {
+ e.mlx_ctx = pplx_mlx_load(model_dir);
+ if (!e.mlx_ctx) { fprintf(stderr, "failed to load model\n"); return 1; }
+ e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size;
+ } else
+#endif
+ {
+ e.ctx = pplx_load(model_dir);
+ if (!e.ctx) { fprintf(stderr, "failed to load model\n"); return 1; }
+ e.dim = e.ctx->config.hidden_size;
}
- free(texts);
- free(embeddings);
- if (ctx) pplx_free(ctx);
+ if (verbose >= 1)
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
+ e.dim, now_ms() - t0, use_mlx ? " (MLX)" : "");
+
+ /* Run */
+ int rc;
+ if (daemon)
+ rc = run_daemon(&e);
+ else
+ rc = run_batch(&e, argc, argv, arg_start, print_embs);
+
+ /* Cleanup */
+ if (e.ctx) pplx_free(e.ctx);
#ifdef USE_MLX
- if (mlx_ctx) pplx_mlx_free(mlx_ctx);
+ if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
qwen_tokenizer_free(tok);
- return 0;
+ return rc;
}
diff --git a/pplx_embed.c b/pplx_embed.c
index 2220760..dcd1acd 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -1,21 +1,3 @@
-/*
- * pplx_embed.c - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
- *
- * Transformer forward pass:
- * embed -> 28x[RMSNorm -> QKV (no bias) -> Q/K RMSNorm -> RoPE
- * -> Bidirectional GQA attention -> O proj -> residual
- * -> RMSNorm -> SwiGLU MLP -> residual]
- * -> final RMSNorm -> mean pool -> L2 normalize
- *
- * Key implementation choices:
- * - All weights are F32 (mmap'd direct, no copy)
- * - Bidirectional attention: reuse qwen_causal_attention with
- * q_offset = (seq_len - 1), which removes the causal mask
- * (every query sees all seq_len keys)
- * - GQA ratio 2:1 (16 Q-heads, 8 KV-heads) handled by causal_attention
- * - RoPE cache grown lazily, positions [0 .. seq_len-1]
- */
-
#include "pplx_embed.h"
#include "qwen_asr_kernels.h"
#include "qwen_asr_safetensors.h"
@@ -26,32 +8,128 @@
#include <math.h>
/* ========================================================================
- * Global verbose flags
- * qwen_verbose is declared extern in qwen_asr_kernels.h and qwen_asr_tokenizer.c;
- * we own its definition here since we don't link qwen_asr.c.
+ * Globals
* ======================================================================== */
int pplx_verbose = 0;
-int qwen_verbose = 0; /* required by qwen_asr_kernels.c and qwen_asr_tokenizer.c */
+int qwen_verbose = 0;
/* ========================================================================
- * Internal helpers: weight loading
+ * Minimal JSON helpers for config.json parsing
+ * ======================================================================== */
+
+static const char *skip_ws(const char *p)
+{
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+ return p;
+}
+
+/* Find "key": <value> in a flat JSON object. Returns pointer to value start. */
+static const char *json_find_key(const char *json, const char *key)
+{
+ size_t klen = strlen(key);
+ const char *p = json;
+ while ((p = strstr(p, key)) != NULL) {
+ /* Check it's actually a quoted key: "key" */
+ if (p > json && *(p - 1) == '"') {
+ const char *after = p + klen;
+ if (*after == '"') {
+ after = skip_ws(after + 1);
+ if (*after == ':') return skip_ws(after + 1);
+ }
+ }
+ p += klen;
+ }
+ return NULL;
+}
+
+static int json_get_int(const char *json, const char *key, int fallback)
+{
+ const char *v = json_find_key(json, key);
+ if (!v) return fallback;
+ return atoi(v);
+}
+
+static double json_get_double(const char *json, const char *key, double fallback)
+{
+ const char *v = json_find_key(json, key);
+ if (!v) return fallback;
+ return atof(v);
+}
+
+static int parse_config(pplx_config_t *cfg, const char *model_dir)
+{
+ char path[1024];
+ snprintf(path, sizeof(path), "%s/config.json", model_dir);
+
+ FILE *f = fopen(path, "rb");
+ if (!f) {
+ fprintf(stderr, "pplx_load: cannot open %s\n", path);
+ return -1;
+ }
+
+ fseek(f, 0, SEEK_END);
+ long sz = ftell(f);
+ fseek(f, 0, SEEK_SET);
+ char *buf = (char *)malloc(sz + 1);
+ if (!buf || fread(buf, 1, sz, f) != (size_t)sz) {
+ fclose(f); free(buf); return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+
+ cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
+ cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
+ cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
+ cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
+ cfg->head_dim = json_get_int(buf, "head_dim", PPLX_HEAD_DIM);
+ cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0);
+ cfg->vocab_size = json_get_int(buf, "vocab_size", PPLX_VOCAB_SIZE);
+ cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
+ cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
+
+ cfg->q_dim = cfg->n_heads * cfg->head_dim;
+ cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
+
+ free(buf);
+
+ /* Sanity checks */
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0) {
+ fprintf(stderr, "pplx_load: invalid config in %s "
+ "(hidden=%d, layers=%d, heads=%d)\n",
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads);
+ return -1;
+ }
+ if (cfg->n_layers > PPLX_MAX_LAYERS) {
+ fprintf(stderr, "pplx_load: too many layers (%d > %d)\n",
+ cfg->n_layers, PPLX_MAX_LAYERS);
+ return -1;
+ }
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, "
+ "inter=%d, head_dim=%d\n",
+ cfg->hidden_size, cfg->n_layers,
+ cfg->n_heads, cfg->n_kv_heads,
+ cfg->intermediate_size, cfg->head_dim);
+
+ return 0;
+}
+
+/* ========================================================================
+ * Weight loading (direct mmap pointers into safetensors)
* ======================================================================== */
-/*
- * Return a direct pointer into the mmap'd safetensors region for an F32 tensor.
- * No allocation, no copy; caller must NOT free the returned pointer.
- */
static const float *load_f32_direct(multi_safetensors_t *ms, const char *name)
{
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
- fprintf(stderr, "pplx_embed: tensor not found: %s\n", name);
+ fprintf(stderr, "pplx: tensor not found: %s\n", name);
return NULL;
}
if (t->dtype != DTYPE_F32) {
- fprintf(stderr, "pplx_embed: expected F32 for %s, got dtype=%d\n",
+ fprintf(stderr, "pplx: expected F32 for %s, got dtype=%d\n",
name, (int)t->dtype);
return NULL;
}
@@ -59,96 +137,81 @@ static const float *load_f32_direct(multi_safetensors_t *ms, const char *name)
}
/* ========================================================================
- * Internal helpers: working buffers
+ * Working buffer management
* ======================================================================== */
static int ensure_buffers(pplx_ctx_t *ctx, int seq)
{
if (seq <= ctx->buf_seq_cap) return 0;
- int new_cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64;
- while (new_cap < seq) new_cap *= 2;
+ int cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64;
+ while (cap < seq) cap *= 2;
- int hidden = ctx->config.hidden_size;
- int q_dim = ctx->config.q_dim;
- int kv_dim = ctx->config.kv_dim;
- int inter = ctx->config.intermediate_size;
+ const pplx_config_t *c = &ctx->config;
-#define PPLX_REALLOC(ptr, n) do { \
- void *_tmp = realloc((ptr), (size_t)(n) * sizeof(float)); \
- if (!_tmp) return -1; \
- (ptr) = (float *)_tmp; \
+#define R(ptr, n) do { \
+ void *_t = realloc((ptr), (size_t)(n) * sizeof(float)); \
+ if (!_t) return -1; \
+ (ptr) = (float *)_t; \
} while (0)
- PPLX_REALLOC(ctx->x, new_cap * hidden);
- PPLX_REALLOC(ctx->x_norm, new_cap * hidden);
- PPLX_REALLOC(ctx->q, new_cap * q_dim);
- PPLX_REALLOC(ctx->k, new_cap * kv_dim);
- PPLX_REALLOC(ctx->v, new_cap * kv_dim);
- PPLX_REALLOC(ctx->attn_out, new_cap * q_dim);
- PPLX_REALLOC(ctx->proj_out, new_cap * hidden);
- PPLX_REALLOC(ctx->ffn_gate, new_cap * inter);
- PPLX_REALLOC(ctx->ffn_up, new_cap * inter);
- PPLX_REALLOC(ctx->ffn_out, new_cap * hidden);
+ R(ctx->x, cap * c->hidden_size);
+ R(ctx->x_norm, cap * c->hidden_size);
+ R(ctx->q, cap * c->q_dim);
+ R(ctx->k, cap * c->kv_dim);
+ R(ctx->v, cap * c->kv_dim);
+ R(ctx->attn_out, cap * c->q_dim);
+ R(ctx->proj_out, cap * c->hidden_size);
+ R(ctx->ffn_gate, cap * c->intermediate_size);
+ R(ctx->ffn_up, cap * c->intermediate_size);
+ R(ctx->ffn_out, cap * c->hidden_size);
-#undef PPLX_REALLOC
+#undef R
- ctx->buf_seq_cap = new_cap;
+ ctx->buf_seq_cap = cap;
return 0;
}
/* ========================================================================
- * Internal helpers: RoPE cache
- *
- * Stores cos/sin for positions [0 .. rope_cache_cap-1].
- * Layout: [pos, head_dim] - values are duplicated across the two halves
- * so that qwen_apply_rope_neox can load contiguous cos[d] and cos[half+d]
- * from a single row: cos_row[d] == cos_row[half+d].
+ * RoPE cache
* ======================================================================== */
static int ensure_rope_cache(pplx_ctx_t *ctx, int n_pos)
{
if (n_pos <= ctx->rope_cache_cap) return 0;
- int new_cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512;
- while (new_cap < n_pos) new_cap *= 2;
+ int cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512;
+ while (cap < n_pos) cap *= 2;
int head_dim = ctx->config.head_dim;
int half = head_dim / 2;
float theta = ctx->config.rope_theta;
- size_t n = (size_t)new_cap * head_dim;
- float *new_cos = (float *)realloc(ctx->rope_cos, n * sizeof(float));
- float *new_sin = (float *)realloc(ctx->rope_sin, n * sizeof(float));
- if (!new_cos || !new_sin) {
- /* If realloc failed, partial state is still valid up to old cap */
- if (new_cos) ctx->rope_cos = new_cos;
- if (new_sin) ctx->rope_sin = new_sin;
+ size_t n = (size_t)cap * head_dim;
+ float *nc = (float *)realloc(ctx->rope_cos, n * sizeof(float));
+ float *ns = (float *)realloc(ctx->rope_sin, n * sizeof(float));
+ if (!nc || !ns) {
+ if (nc) ctx->rope_cos = nc;
+ if (ns) ctx->rope_sin = ns;
return -1;
}
- ctx->rope_cos = new_cos;
- ctx->rope_sin = new_sin;
+ ctx->rope_cos = nc;
+ ctx->rope_sin = ns;
- /* Compute only the newly needed positions */
- for (int pos = ctx->rope_cache_cap; pos < new_cap; pos++) {
+ for (int pos = ctx->rope_cache_cap; pos < cap; pos++) {
float fp = (float)pos;
- float *c = ctx->rope_cos + (size_t)pos * head_dim;
- float *s = ctx->rope_sin + (size_t)pos * head_dim;
+ float *c = ctx->rope_cos + (size_t)pos * head_dim;
+ float *s = ctx->rope_sin + (size_t)pos * head_dim;
for (int d = 0; d < half; d++) {
- /* inv_freq[d] = 1 / (theta ^ (2d / head_dim)) */
- float inv_freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
- float angle = fp * inv_freq;
- float cv = cosf(angle);
- float sv = sinf(angle);
- /* Duplicate across both halves for the NeoX rotation kernel */
- c[d] = cv;
- c[half + d] = cv;
- s[d] = sv;
- s[half + d] = sv;
+ float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
+ float angle = fp * inv;
+ float cv = cosf(angle), sv = sinf(angle);
+ c[d] = cv; c[half + d] = cv;
+ s[d] = sv; s[half + d] = sv;
}
}
- ctx->rope_cache_cap = new_cap;
+ ctx->rope_cache_cap = cap;
return 0;
}
@@ -161,21 +224,15 @@ pplx_ctx_t *pplx_load(const char *model_dir)
pplx_ctx_t *ctx = (pplx_ctx_t *)calloc(1, sizeof(pplx_ctx_t));
if (!ctx) return NULL;
- /* ---- Hardcoded config (mirrors config.json) ---- */
- pplx_config_t *cfg = &ctx->config;
- cfg->hidden_size = PPLX_HIDDEN_SIZE;
- cfg->n_layers = PPLX_NUM_LAYERS;
- cfg->n_heads = PPLX_NUM_HEADS;
- cfg->n_kv_heads = PPLX_NUM_KV_HEADS;
- cfg->head_dim = PPLX_HEAD_DIM;
- cfg->q_dim = PPLX_Q_DIM;
- cfg->kv_dim = PPLX_KV_DIM;
- cfg->intermediate_size = PPLX_INTERMEDIATE_SIZE;
- cfg->vocab_size = PPLX_VOCAB_SIZE;
- cfg->rms_norm_eps = 1e-6f;
- cfg->rope_theta = 1000000.0f;
+ /* Parse config.json */
+ if (parse_config(&ctx->config, model_dir) != 0) {
+ free(ctx);
+ return NULL;
+ }
+
+ const pplx_config_t *cfg = &ctx->config;
- /* ---- Open safetensors ---- */
+ /* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
if (!ms) {
fprintf(stderr, "pplx_load: failed to open safetensors in %s\n", model_dir);
@@ -187,12 +244,16 @@ pplx_ctx_t *pplx_load(const char *model_dir)
if (pplx_verbose >= 1)
fprintf(stderr, "pplx_load: loading weights from %s\n", model_dir);
- /* ---- Token embeddings ---- */
- pplx_weights_t *w = &ctx->weights;
+ /* Token embeddings */
+ pplx_weights_t *w = &ctx->weights;
w->embed_tokens = load_f32_direct(ms, "embed_tokens.weight");
if (!w->embed_tokens) goto fail;
- /* ---- Per-layer weights ---- */
+ /* Allocate layer array */
+ w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t));
+ if (!w->layers) goto fail;
+
+ /* Per-layer weights */
char name[256];
for (int i = 0; i < cfg->n_layers; i++) {
pplx_layer_t *l = &w->layers[i];
@@ -221,12 +282,13 @@ pplx_ctx_t *pplx_load(const char *model_dir)
fprintf(stderr, " layer %d loaded\n", i);
}
- /* ---- Final RMSNorm ---- */
+ /* Final RMSNorm */
w->norm = load_f32_direct(ms, "norm.weight");
if (!w->norm) goto fail;
if (pplx_verbose >= 1)
- fprintf(stderr, "pplx_load: all weights loaded\n");
+ fprintf(stderr, "pplx_load: %d layers loaded (%d-dim embeddings)\n",
+ cfg->n_layers, cfg->hidden_size);
return ctx;
@@ -244,34 +306,27 @@ void pplx_free(pplx_ctx_t *ctx)
if (!ctx) return;
if (ctx->safetensors)
multi_safetensors_close((multi_safetensors_t *)ctx->safetensors);
-
- free(ctx->x);
- free(ctx->x_norm);
- free(ctx->q);
- free(ctx->k);
- free(ctx->v);
- free(ctx->attn_out);
- free(ctx->proj_out);
- free(ctx->ffn_gate);
- free(ctx->ffn_up);
- free(ctx->ffn_out);
- free(ctx->rope_cos);
- free(ctx->rope_sin);
+ free(ctx->weights.layers);
+ free(ctx->x); free(ctx->x_norm);
+ free(ctx->q); free(ctx->k); free(ctx->v);
+ free(ctx->attn_out); free(ctx->proj_out);
+ free(ctx->ffn_gate); free(ctx->ffn_up); free(ctx->ffn_out);
+ free(ctx->rope_cos); free(ctx->rope_sin);
free(ctx);
}
/* ========================================================================
- * Forward pass
+ * Forward pass (returns full token-level embeddings)
* ======================================================================== */
-float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
{
if (!ctx || !token_ids || n_tokens <= 0) return NULL;
const pplx_config_t *cfg = &ctx->config;
const pplx_weights_t *w = &ctx->weights;
- int hidden = cfg->hidden_size;
+ int hidden = cfg->hidden_size;
int n_heads = cfg->n_heads;
int n_kv_heads = cfg->n_kv_heads;
int head_dim = cfg->head_dim;
@@ -281,39 +336,28 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
float eps = cfg->rms_norm_eps;
int seq = n_tokens;
- /* Allocate/grow working buffers */
- if (ensure_buffers(ctx, seq) != 0) {
- fprintf(stderr, "pplx_embed: buffer allocation failed\n");
- return NULL;
- }
+ if (ensure_buffers(ctx, seq) != 0) return NULL;
+ if (ensure_rope_cache(ctx, seq) != 0) return NULL;
- /* Pre-compute RoPE tables for positions [0, seq-1] */
- if (ensure_rope_cache(ctx, seq) != 0) {
- fprintf(stderr, "pplx_embed: RoPE cache allocation failed\n");
- return NULL;
- }
- const float *rope_cos = ctx->rope_cos; /* [seq_cap, head_dim] */
+ const float *rope_cos = ctx->rope_cos;
const float *rope_sin = ctx->rope_sin;
float *x = ctx->x;
float *x_norm = ctx->x_norm;
- float *q = ctx->q;
- float *k = ctx->k;
- float *v = ctx->v;
+ float *q_buf = ctx->q;
+ float *k_buf = ctx->k;
+ float *v_buf = ctx->v;
float *attn_out = ctx->attn_out;
float *proj_out = ctx->proj_out;
float *ffn_gate = ctx->ffn_gate;
float *ffn_up = ctx->ffn_up;
float *ffn_out = ctx->ffn_out;
- /* ------------------------------------------------------------------
- * Step 1: Token embedding lookup
- * x[i, :] = embed_tokens[token_ids[i], :]
- * ------------------------------------------------------------------ */
+ /* 1. Token embedding lookup */
for (int i = 0; i < seq; i++) {
int id = token_ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "pplx_embed: invalid token id %d at position %d\n", id, i);
+ fprintf(stderr, "pplx: invalid token id %d at position %d\n", id, i);
return NULL;
}
memcpy(x + i * hidden,
@@ -323,120 +367,97 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
float scale = 1.0f / sqrtf((float)head_dim);
- /* ------------------------------------------------------------------
- * Steps 2..29: 28 transformer layers
- * ------------------------------------------------------------------ */
+ /* 2. Transformer layers */
for (int layer = 0; layer < cfg->n_layers; layer++) {
const pplx_layer_t *l = &w->layers[layer];
- /* ---- 2a. Input RMSNorm ---- */
+ /* Input RMSNorm */
qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps);
- /* ---- 2b. QKV projections (no bias) ----
- * q: [seq, q_dim] = x_norm @ wq^T (wq is [q_dim, hidden])
- * k: [seq, kv_dim] = x_norm @ wk^T
- * v: [seq, kv_dim] = x_norm @ wv^T
- */
- qwen_linear_nobias(q, x_norm, l->wq, seq, hidden, q_dim);
- qwen_linear_nobias(k, x_norm, l->wk, seq, hidden, kv_dim);
- qwen_linear_nobias(v, x_norm, l->wv, seq, hidden, kv_dim);
+ /* QKV projections */
+ qwen_linear_nobias(q_buf, x_norm, l->wq, seq, hidden, q_dim);
+ qwen_linear_nobias(k_buf, x_norm, l->wk, seq, hidden, kv_dim);
+ qwen_linear_nobias(v_buf, x_norm, l->wv, seq, hidden, kv_dim);
- /* ---- 2c. Per-head Q/K RMSNorm (Qwen3 feature) ----
- * Normalize each head's [head_dim] segment independently.
- */
- qwen_rms_norm_per_head(q, l->q_norm, seq, n_heads, head_dim, eps);
- qwen_rms_norm_per_head(k, l->k_norm, seq, n_kv_heads, head_dim, eps);
+ /* Per-head Q/K RMSNorm */
+ qwen_rms_norm_per_head(q_buf, l->q_norm, seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(k_buf, l->k_norm, seq, n_kv_heads, head_dim, eps);
- /* ---- 2d. Apply NeoX-style RoPE to Q and K ----
- * rope_cos/sin: [seq_cap, head_dim], positions 0..seq-1 cached
- */
- qwen_apply_rope_neox(q, rope_cos, rope_sin, seq, n_heads, head_dim);
- qwen_apply_rope_neox(k, rope_cos, rope_sin, seq, n_kv_heads, head_dim);
+ /* RoPE */
+ qwen_apply_rope_neox(q_buf, rope_cos, rope_sin, seq, n_heads, head_dim);
+ qwen_apply_rope_neox(k_buf, rope_cos, rope_sin, seq, n_kv_heads, head_dim);
- /* ---- 2e. Bidirectional GQA attention ----
- *
- * We reuse qwen_causal_attention with q_offset = (seq - 1).
- *
- * Inside the kernel for query position i:
- * global_pos = q_offset + i = (seq-1) + i
- * k_end = min(global_pos + 1, seq_k)
- * = min(seq + i, seq) = seq for all i >= 0
- *
- * So every query attends to ALL seq key positions - full attention,
- * no causal mask, while still benefiting from the GQA + threading
- * in qwen_causal_attention.
- */
- qwen_causal_attention(attn_out, q, k, v,
- seq, /* seq_q */
- seq, /* seq_k */
- n_heads, n_kv_heads, head_dim,
- scale,
- seq - 1); /* q_offset - removes causal mask */
+ /* Bidirectional GQA attention (q_offset = seq-1 removes causal mask) */
+ qwen_causal_attention(attn_out, q_buf, k_buf, v_buf,
+ seq, seq, n_heads, n_kv_heads,
+ head_dim, scale, seq - 1);
- /* ---- 2f. Output projection + residual ----
- * proj_out = attn_out @ wo^T (wo is [hidden, q_dim])
- * x += proj_out
- */
+ /* Output projection + residual */
qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, seq * hidden);
- /* ---- 2g. Post-attention RMSNorm ---- */
+ /* Post-attention RMSNorm */
qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps);
- /* ---- 2h. SwiGLU MLP ----
- * gate = gate_proj(x_norm) [seq, intermediate]
- * up = up_proj(x_norm) [seq, intermediate]
- * mid = SiLU(gate) * up [seq, intermediate]
- * out = down_proj(mid) [seq, hidden]
- */
+ /* SwiGLU MLP */
qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter);
qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter);
-
- qwen_silu(ffn_gate, seq * inter); /* SiLU(gate) in-place */
- qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); /* gate *= up */
-
+ qwen_silu(ffn_gate, seq * inter);
+ qwen_mul_inplace(ffn_gate, ffn_up, seq * inter);
qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden);
qwen_add_inplace(x, ffn_out, seq * hidden);
}
- /* ------------------------------------------------------------------
- * Step 3: Final RMSNorm (in-place on x)
- * ------------------------------------------------------------------ */
+ /* 3. Final RMSNorm (in-place) */
qwen_rms_norm(x, x, w->norm, seq, hidden, eps);
- /* ------------------------------------------------------------------
- * Step 4: Mean pooling across all token positions
- * embedding[d] = mean over i in [0, seq) of x[i, d]
- * ------------------------------------------------------------------ */
- float *embedding = (float *)calloc(hidden, sizeof(float));
- if (!embedding) return NULL;
+ /* Copy result out (caller frees) */
+ size_t out_size = (size_t)seq * hidden * sizeof(float);
+ float *out = (float *)malloc(out_size);
+ if (out) memcpy(out, x, out_size);
+ return out;
+}
+
+/* ========================================================================
+ * Embed (forward + mean pool + L2 norm)
+ * ======================================================================== */
+
+float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ float *all = pplx_forward(ctx, token_ids, n_tokens);
+ if (!all) return NULL;
+
+ int hidden = ctx->config.hidden_size;
+ int seq = n_tokens;
+
+ /* Mean pool */
+ float *emb = (float *)calloc(hidden, sizeof(float));
+ if (!emb) { free(all); return NULL; }
for (int i = 0; i < seq; i++) {
- const float *row = x + i * hidden;
- for (int d = 0; d < hidden; d++) {
- embedding[d] += row[d];
- }
+ const float *row = all + i * hidden;
+ for (int d = 0; d < hidden; d++)
+ emb[d] += row[d];
}
+ free(all);
+
float inv_seq = 1.0f / (float)seq;
- for (int d = 0; d < hidden; d++) embedding[d] *= inv_seq;
+ for (int d = 0; d < hidden; d++) emb[d] *= inv_seq;
- /* ------------------------------------------------------------------
- * Step 5: L2 normalization
- * embedding /= ||embedding||_2
- * ------------------------------------------------------------------ */
+ /* L2 normalize */
float norm_sq = 0.0f;
- for (int d = 0; d < hidden; d++) norm_sq += embedding[d] * embedding[d];
- float norm_val = sqrtf(norm_sq);
- if (norm_val > 1e-8f) {
- float inv = 1.0f / norm_val;
- for (int d = 0; d < hidden; d++) embedding[d] *= inv;
+ for (int d = 0; d < hidden; d++) norm_sq += emb[d] * emb[d];
+ float nv = sqrtf(norm_sq);
+ if (nv > 1e-8f) {
+ float inv = 1.0f / nv;
+ for (int d = 0; d < hidden; d++) emb[d] *= inv;
}
- return embedding;
+ return emb;
}
/* ========================================================================
- * Cosine similarity (dot product of two L2-normalized vectors)
+ * Cosine similarity
* ======================================================================== */
float pplx_cosine_similarity(const float *a, const float *b, int dim)
diff --git a/pplx_embed.h b/pplx_embed.h
index 69b67c3..67b193e 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -1,23 +1,7 @@
/*
- * pplx_embed.h - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b
+ * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models.
*
- * Architecture: Qwen3-family transformer with BIDIRECTIONAL (non-causal)
- * attention. Based on antirez's qwen-asr kernels. Produces dense float32
- * embedding vectors via mean-pooling + L2-normalization.
- *
- * Model spec:
- * hidden_size = 1024
- * num_hidden_layers = 28
- * num_attention_heads = 16 (query heads)
- * num_key_value_heads = 8 (GQA: 2 query heads share 1 KV head)
- * head_dim = 128
- * intermediate_size = 3072
- * vocab_size = 151936
- * rms_norm_eps = 1e-6
- * rope_theta = 1000000.0 (1e6)
- * weights dtype = F32 (all weights float32)
- * tie_word_embeddings = true
- * use_bidirectional_attention = true
+ * Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B).
*/
#ifndef PPLX_EMBED_H
@@ -27,32 +11,26 @@
#include <stdint.h>
/* ========================================================================
- * Constants
+ * Shared constants (identical across all models)
* ======================================================================== */
-#define PPLX_HIDDEN_SIZE 1024
-#define PPLX_NUM_LAYERS 28
-#define PPLX_NUM_HEADS 16
-#define PPLX_NUM_KV_HEADS 8
-#define PPLX_HEAD_DIM 128
-#define PPLX_Q_DIM (PPLX_NUM_HEADS * PPLX_HEAD_DIM) /* 2048 */
-#define PPLX_KV_DIM (PPLX_NUM_KV_HEADS * PPLX_HEAD_DIM) /* 1024 */
-#define PPLX_INTERMEDIATE_SIZE 3072
-#define PPLX_VOCAB_SIZE 151936
+#define PPLX_VOCAB_SIZE 151936
+#define PPLX_HEAD_DIM 128
+#define PPLX_MAX_LAYERS 64 /* upper bound for stack arrays */
/* ========================================================================
- * Model Configuration
+ * Model Configuration (populated from config.json at load time)
* ======================================================================== */
typedef struct {
- int hidden_size; /* 1024 */
- int n_layers; /* 28 */
- int n_heads; /* 16 */
- int n_kv_heads; /* 8 */
- int head_dim; /* 128 */
- int q_dim; /* n_heads * head_dim = 2048 */
- int kv_dim; /* n_kv_heads * head_dim = 1024 */
- int intermediate_size; /* 3072 */
+ int hidden_size; /* 0.6B: 1024 4B: 2560 */
+ int n_layers; /* 0.6B: 28 4B: 36 */
+ int n_heads; /* 0.6B: 16 4B: 32 */
+ int n_kv_heads; /* 0.6B: 8 4B: 8 */
+ int head_dim; /* always 128 */
+ int q_dim; /* n_heads * head_dim */
+ int kv_dim; /* n_kv_heads * head_dim */
+ int intermediate_size; /* 0.6B: 3072 4B: 9728 */
int vocab_size; /* 151936 */
float rms_norm_eps; /* 1e-6 */
float rope_theta; /* 1e6 */
@@ -60,37 +38,17 @@ typedef struct {
/* ========================================================================
* Per-Layer Weights (all F32, direct mmap pointers - read-only)
- *
- * Tensor shapes (row-major):
- * wq [q_dim, hidden] = [2048, 1024]
- * wk [kv_dim, hidden] = [1024, 1024]
- * wv [kv_dim, hidden] = [1024, 1024]
- * wo [hidden, q_dim] = [1024, 2048]
- * q_norm [head_dim] = [128]
- * k_norm [head_dim] = [128]
- * input_norm [hidden] = [1024]
- * post_attn_norm [hidden] = [1024]
- * gate_proj [intermediate, hidden] = [3072, 1024]
- * up_proj [intermediate, hidden] = [3072, 1024]
- * down_proj [hidden, intermediate] = [1024, 3072]
* ======================================================================== */
typedef struct {
- /* Self-attention projections (no bias) */
const float *wq; /* [q_dim, hidden] */
const float *wk; /* [kv_dim, hidden] */
const float *wv; /* [kv_dim, hidden] */
const float *wo; /* [hidden, q_dim] */
-
- /* Per-head Q/K RMSNorm weights (Qwen3 feature) */
const float *q_norm; /* [head_dim] */
const float *k_norm; /* [head_dim] */
-
- /* Layer norms (no bias - RMSNorm only) */
- const float *input_norm; /* [hidden] pre-attention */
- const float *post_attn_norm;/* [hidden] pre-MLP */
-
- /* SwiGLU MLP (no bias) */
+ const float *input_norm; /* [hidden] */
+ const float *post_attn_norm;/* [hidden] */
const float *gate_proj; /* [intermediate, hidden] */
const float *up_proj; /* [intermediate, hidden] */
const float *down_proj; /* [hidden, intermediate] */
@@ -101,14 +59,9 @@ typedef struct {
* ======================================================================== */
typedef struct {
- /* Token embedding table (tied with output head, but we don't need lm_head) */
- const float *embed_tokens; /* [vocab_size, hidden] */
-
- /* Transformer layers */
- pplx_layer_t layers[PPLX_NUM_LAYERS];
-
- /* Final RMSNorm */
- const float *norm; /* [hidden] */
+ const float *embed_tokens; /* [vocab_size, hidden] */
+ pplx_layer_t *layers; /* [n_layers] heap-allocated */
+ const float *norm; /* [hidden] */
} pplx_weights_t;
/* ========================================================================
@@ -118,27 +71,25 @@ typedef struct {
typedef struct {
pplx_config_t config;
pplx_weights_t weights;
-
- /* Open safetensors handle (keeps mmap alive) */
- void *safetensors; /* multi_safetensors_t* */
+ void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
/* Working buffers, grown lazily to buf_seq_cap */
int buf_seq_cap;
- float *x; /* [seq, hidden] main activations */
- float *x_norm; /* [seq, hidden] after RMSNorm */
- float *q; /* [seq, q_dim] query vectors */
- float *k; /* [seq, kv_dim] key vectors */
- float *v; /* [seq, kv_dim] value vectors */
- float *attn_out; /* [seq, q_dim] attention output */
- float *proj_out; /* [seq, hidden] after o_proj */
- float *ffn_gate; /* [seq, intermediate] gate branch */
- float *ffn_up; /* [seq, intermediate] up branch */
- float *ffn_out; /* [seq, hidden] after down_proj */
+ float *x; /* [seq, hidden] */
+ float *x_norm; /* [seq, hidden] */
+ float *q; /* [seq, q_dim] */
+ float *k; /* [seq, kv_dim] */
+ float *v; /* [seq, kv_dim] */
+ float *attn_out; /* [seq, q_dim] */
+ float *proj_out; /* [seq, hidden] */
+ float *ffn_gate; /* [seq, intermediate] */
+ float *ffn_up; /* [seq, intermediate] */
+ float *ffn_out; /* [seq, hidden] */
/* RoPE cosine/sine cache [n_pos, head_dim], grown lazily */
float *rope_cos;
float *rope_sin;
- int rope_cache_cap; /* number of cached positions */
+ int rope_cache_cap;
} pplx_ctx_t;
/* ========================================================================
@@ -146,11 +97,12 @@ typedef struct {
* ======================================================================== */
/*
- * Load the model from a directory containing:
- * model.safetensors - weight tensors (F32)
- * vocab.json - BPE vocabulary
- * merges.txt - BPE merge rules
+ * Load model from directory containing:
+ * config.json - model hyperparameters
+ * model.safetensors (or sharded) - F32 weights
+ * vocab.json + merges.txt - BPE tokenizer
*
+ * Reads config.json to determine model size (0.6B vs 4B).
* Returns NULL on error.
*/
pplx_ctx_t *pplx_load(const char *model_dir);
@@ -159,28 +111,35 @@ pplx_ctx_t *pplx_load(const char *model_dir);
void pplx_free(pplx_ctx_t *ctx);
/*
- * Compute the embedding for a sequence of token IDs.
+ * Compute embedding for a token sequence.
*
- * Performs:
* 1. Token embedding lookup
- * 2. 28 × bidirectional transformer layers (GQA, RoPE, SwiGLU)
+ * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU)
* 3. Final RMSNorm
- * 4. Mean pooling across all token positions
+ * 4. Mean pooling over all positions
* 5. L2 normalization
*
- * Returns a malloc'd float[hidden_size] (caller must free).
- * Returns NULL on allocation error or empty input.
+ * Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
/*
- * Cosine similarity between two L2-normalized embedding vectors.
- * (This is just the dot product since both are unit vectors.)
+ * Run the transformer forward pass WITHOUT pooling.
+ * Returns the full [n_tokens * hidden_size] output after final RMSNorm.
+ * Caller frees the returned buffer. NULL on error.
+ *
+ * Useful for contextual (late-chunking) models where you need
+ * per-token embeddings before splitting by separator positions.
+ */
+float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens);
+
+/*
+ * Cosine similarity between two L2-normalized vectors.
*/
float pplx_cosine_similarity(const float *a, const float *b, int dim);
/* Verbose level: 0=quiet, 1=info, 2=debug */
extern int pplx_verbose;
-extern int qwen_verbose; /* same flag, used internally by kernel/tokenizer */
+extern int qwen_verbose;
#endif /* PPLX_EMBED_H */
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index d6637cd..3c82db1 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -1,21 +1,6 @@
-/*
- * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference
- *
- * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference.
- * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm.
- *
- * Key operations used:
- * mlx_take - embedding lookup
- * mlx_matmul - linear projections (no bias)
- * mlx_fast_rms_norm - RMSNorm (fused Metal kernel)
- * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel)
- * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel)
- * sigmoid * gate, multiply - SwiGLU activation
- */
-
#include "pplx_embed_mlx.h"
-#include "pplx_embed.h" /* config constants */
-#include "qwen_asr_safetensors.h" /* safetensors reader */
+#include "pplx_embed.h"
+#include "qwen_asr_safetensors.h"
#include <mlx/c/mlx.h>
@@ -25,30 +10,22 @@
#include <math.h>
/* ========================================================================
- * Per-layer weight arrays (MLX)
+ * Per-layer MLX weight arrays
* ======================================================================== */
typedef struct {
- mlx_array wq; /* [q_dim, hidden] */
- mlx_array wk; /* [kv_dim, hidden] */
- mlx_array wv; /* [kv_dim, hidden] */
- mlx_array wo; /* [hidden, q_dim] */
- mlx_array q_norm; /* [head_dim] */
- mlx_array k_norm; /* [head_dim] */
- mlx_array input_norm; /* [hidden] */
- mlx_array post_attn_norm; /* [hidden] */
- mlx_array gate_proj; /* [intermediate, hidden] */
- mlx_array up_proj; /* [intermediate, hidden] */
- mlx_array down_proj; /* [hidden, intermediate] */
+ mlx_array wq, wk, wv, wo;
+ mlx_array q_norm, k_norm;
+ mlx_array input_norm, post_attn_norm;
+ mlx_array gate_proj, up_proj, down_proj;
} mlx_layer_t;
struct pplx_mlx_ctx {
- mlx_array embed_tokens; /* [vocab, hidden] */
- mlx_layer_t layers[PPLX_NUM_LAYERS];
- mlx_array norm; /* [hidden] */
+ pplx_config_t config;
+ mlx_array embed_tokens;
+ mlx_layer_t *layers; /* heap-allocated [n_layers] */
+ mlx_array norm;
mlx_stream stream;
-
- /* Safetensors kept alive for mmap */
multi_safetensors_t *ms;
};
@@ -56,13 +33,8 @@ struct pplx_mlx_ctx {
* Helpers
* ======================================================================== */
-/* Default GPU stream */
-static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; }
+static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-/*
- * Load an F32 tensor from safetensors into an mlx_array.
- * The data is copied into MLX (which will move it to GPU as needed).
- */
static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
const int *shape, int ndim)
{
@@ -76,82 +48,156 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
fprintf(stderr, "mlx: expected F32 for %s\n", name);
return (mlx_array){0};
}
- const void *data = safetensors_data(sf, t);
- return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32);
+ return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32);
}
-/* Quick check: is arr valid? */
-static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+/* y = x @ W^T */
+static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
+{
+ mlx_array Wt = mlx_array_new();
+ mlx_transpose(&Wt, W, s);
+ mlx_array res = mlx_array_new();
+ mlx_matmul(&res, x, Wt, s);
+ mlx_array_free(Wt);
+ return res;
+}
+
+const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx)
+{
+ return ctx ? &ctx->config : NULL;
+}
/* ========================================================================
- * Load
+ * Load (reuses the same parse_config logic via pplx_load then copies)
* ======================================================================== */
+/* We duplicate a minimal config parser here so MLX can load standalone.
+ * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */
+
+static const char *skip_ws_m(const char *p)
+{
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+ return p;
+}
+
+static const char *json_find_m(const char *json, const char *key)
+{
+ size_t klen = strlen(key);
+ const char *p = json;
+ while ((p = strstr(p, key)) != NULL) {
+ if (p > json && *(p - 1) == '"') {
+ const char *a = p + klen;
+ if (*a == '"') {
+ a = skip_ws_m(a + 1);
+ if (*a == ':') return skip_ws_m(a + 1);
+ }
+ }
+ p += klen;
+ }
+ return NULL;
+}
+
+static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir)
+{
+ char path[1024];
+ snprintf(path, sizeof(path), "%s/config.json", model_dir);
+ FILE *f = fopen(path, "rb");
+ if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; }
+ fseek(f, 0, SEEK_END);
+ long sz = ftell(f);
+ fseek(f, 0, SEEK_SET);
+ char *buf = (char *)malloc(sz + 1);
+ if (!buf || fread(buf, 1, sz, f) != (size_t)sz) { fclose(f); free(buf); return -1; }
+ fclose(f);
+ buf[sz] = '\0';
+
+#define GI(k, fb) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : 0; } while(0)
+#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0)
+ GI("hidden_size", hidden_size);
+ GI("num_hidden_layers", n_layers);
+ GI("num_attention_heads", n_heads);
+ GI("num_key_value_heads", n_kv_heads);
+ GI("intermediate_size", intermediate_size);
+ cfg->head_dim = PPLX_HEAD_DIM;
+ cfg->vocab_size = PPLX_VOCAB_SIZE;
+ GF("rms_norm_eps", rms_norm_eps, 1e-6f);
+ GF("rope_theta", rope_theta, 1000000.0f);
+#undef GI
+#undef GF
+
+ cfg->q_dim = cfg->n_heads * cfg->head_dim;
+ cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
+ free(buf);
+
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0) {
+ fprintf(stderr, "mlx: bad config (hidden=%d layers=%d)\n",
+ cfg->hidden_size, cfg->n_layers);
+ return -1;
+ }
+ return 0;
+}
+
pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
{
pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
if (!ctx) return NULL;
+ if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
+
+ const pplx_config_t *c = &ctx->config;
ctx->stream = mlx_default_gpu_stream_new();
- /* Open safetensors */
ctx->ms = multi_safetensors_open(model_dir);
if (!ctx->ms) {
fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx);
- return NULL;
+ free(ctx); return NULL;
}
- multi_safetensors_t *ms = ctx->ms;
+ int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
+ int inter = c->intermediate_size, hd = c->head_dim;
- /* Embedding table: [vocab_size, hidden] */
- int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE};
- ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2);
+ /* Embedding table */
+ int emb_shape[] = {c->vocab_size, h};
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2);
if (!arr_ok(ctx->embed_tokens)) goto fail;
- /* Per-layer weights */
+ /* Layers */
+ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
+ if (!ctx->layers) goto fail;
+
char name[256];
- for (int i = 0; i < PPLX_NUM_LAYERS; i++) {
+ for (int i = 0; i < c->n_layers; i++) {
mlx_layer_t *l = &ctx->layers[i];
- int hidden = PPLX_HIDDEN_SIZE;
- int q_dim = PPLX_Q_DIM;
- int kv_dim = PPLX_KV_DIM;
- int inter = PPLX_INTERMEDIATE_SIZE;
- int hdim = PPLX_HEAD_DIM;
-#define LOAD(field, fmt, ...) do { \
+#define LD(fld, fmt, ...) do { \
snprintf(name, sizeof(name), fmt, i); \
int sh[] = {__VA_ARGS__}; \
- int nd = sizeof(sh) / sizeof(sh[0]); \
- l->field = load_tensor(ms, name, sh, nd); \
- if (!arr_ok(l->field)) goto fail; \
+ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
+ if (!arr_ok(l->fld)) goto fail; \
} while (0)
- LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden);
- LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden);
- LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden);
- LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim);
- LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim);
- LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim);
- LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden);
- LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden);
- LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden);
- LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden);
- LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter);
+ LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
+ LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
+ LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
+ LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
+ LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
+ LD(input_norm, "layers.%d.input_layernorm.weight", h);
+ LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h);
+ LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
+ LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
+ LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
+#undef LD
-#undef LOAD
-
- if (pplx_verbose >= 2)
- fprintf(stderr, " mlx layer %d loaded\n", i);
+ if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
}
- /* Final norm */
- int norm_shape[] = {PPLX_HIDDEN_SIZE};
- ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1);
+ int ns[] = {h};
+ ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
if (!arr_ok(ctx->norm)) goto fail;
if (pplx_verbose >= 1)
- fprintf(stderr, "mlx: all weights loaded\n");
+ fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h);
return ctx;
@@ -164,18 +210,13 @@ fail:
* Free
* ======================================================================== */
-static void free_layer(mlx_layer_t *l)
+static void free_mlx_layer(mlx_layer_t *l)
{
- mlx_array_free(l->wq);
- mlx_array_free(l->wk);
- mlx_array_free(l->wv);
- mlx_array_free(l->wo);
- mlx_array_free(l->q_norm);
- mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm);
- mlx_array_free(l->post_attn_norm);
- mlx_array_free(l->gate_proj);
- mlx_array_free(l->up_proj);
+ mlx_array_free(l->wq); mlx_array_free(l->wk);
+ mlx_array_free(l->wv); mlx_array_free(l->wo);
+ mlx_array_free(l->q_norm); mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm);
+ mlx_array_free(l->gate_proj); mlx_array_free(l->up_proj);
mlx_array_free(l->down_proj);
}
@@ -183,8 +224,11 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
{
if (!ctx) return;
mlx_array_free(ctx->embed_tokens);
- for (int i = 0; i < PPLX_NUM_LAYERS; i++)
- free_layer(&ctx->layers[i]);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_mlx_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
mlx_array_free(ctx->norm);
mlx_stream_free(ctx->stream);
if (ctx->ms) multi_safetensors_close(ctx->ms);
@@ -192,274 +236,168 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
}
/* ========================================================================
- * Forward pass (all on GPU stream, lazy-evaluated)
- *
- * Notation: S = gpu_stream(ctx)
- *
- * Linear without bias: y = x @ W^T
- * Since mlx_matmul(x, W^T) requires transposing W, and our weights
- * are stored as [out, in], we do: y = x @ W^T via
- * Wt = transpose(W) [in, out]
- * y = matmul(x, Wt) [seq, out]
- *
- * Alternatively: y = matmul(x, transpose(W))
- * All MLX ops are lazy (graph-built) so transpose is free.
+ * Forward pass
* ======================================================================== */
-/* Convenience: transpose a 2D matrix */
-static mlx_array t2d(mlx_array a, mlx_stream s)
-{
- mlx_array res = mlx_array_new();
- mlx_transpose(&res, a, s);
- return res;
-}
-
-/* y = x @ W^T (linear, no bias) */
-static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
-{
- mlx_array Wt = t2d(W, s);
- mlx_array res = mlx_array_new();
- mlx_matmul(&res, x, Wt, s);
- mlx_array_free(Wt);
- return res;
-}
-
float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
{
if (!ctx || !token_ids || n_tokens <= 0) return NULL;
- mlx_stream S = gpu_stream(ctx);
- int hidden = PPLX_HIDDEN_SIZE;
- int n_heads = PPLX_NUM_HEADS;
- int n_kv_heads = PPLX_NUM_KV_HEADS;
- int head_dim = PPLX_HEAD_DIM;
- int q_dim = PPLX_Q_DIM;
- int seq = n_tokens;
- float scale = 1.0f / sqrtf((float)head_dim);
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
- (void)n_heads; (void)n_kv_heads; /* used only in shape literals */
+ int hidden = c->hidden_size;
+ int n_heads = c->n_heads;
+ int n_kv_heads = c->n_kv_heads;
+ int head_dim = c->head_dim;
+ int q_dim = c->q_dim;
+ int seq = n_tokens;
+ float scale = 1.0f / sqrtf((float)head_dim);
- /* ------------------------------------------------------------------
- * 1. Token embedding lookup: x = embed_tokens[token_ids]
- * ------------------------------------------------------------------ */
+ /* 1. Embedding lookup */
int ids_shape[] = {seq};
mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32);
mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S);
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
mlx_array_free(ids);
- /* x: [seq, hidden] */
- /* ------------------------------------------------------------------
- * 2. Transformer layers (28x)
- * ------------------------------------------------------------------ */
- for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) {
+ /* 2. Transformer layers */
+ for (int layer = 0; layer < c->n_layers; layer++) {
mlx_layer_t *l = &ctx->layers[layer];
- /* ---- 2a. Input RMSNorm ---- */
+ /* Input RMSNorm */
mlx_array xn = mlx_array_new();
- mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
- /* ---- 2b. QKV projections (no bias) ----
- * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim]
- */
+ /* QKV projections */
mlx_array q_flat = linear(xn, l->wq, S);
mlx_array k_flat = linear(xn, l->wk, S);
mlx_array v_flat = linear(xn, l->wv, S);
mlx_array_free(xn);
- /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */
+ /* Reshape to [seq, n_heads, head_dim] */
int q_shape[] = {seq, n_heads, head_dim};
int k_shape[] = {seq, n_kv_heads, head_dim};
-
mlx_array q = mlx_array_new();
mlx_array k = mlx_array_new();
mlx_array v = mlx_array_new();
mlx_reshape(&q, q_flat, q_shape, 3, S);
mlx_reshape(&k, k_flat, k_shape, 3, S);
mlx_reshape(&v, v_flat, k_shape, 3, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
+ mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
- /* ---- 2d. Per-head Q/K RMSNorm ----
- * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim)
- */
+ /* Per-head Q/K RMSNorm */
mlx_array qn = mlx_array_new();
mlx_array kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S);
- mlx_array_free(q);
- mlx_array_free(k);
+ mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
+ mlx_array_free(q); mlx_array_free(k);
- /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ----
- * MLX rope expects (..., T, D) where T is second-to-last.
- * [seq, n_heads, head_dim]
- * - expand_dims(0) [1, seq, n_heads, head_dim]
- * - transpose [0,2,1,3] [1, n_heads, seq, head_dim]
- */
+ /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */
int perm[] = {0, 2, 1, 3};
-
- mlx_array qe = mlx_array_new();
- mlx_array ke = mlx_array_new();
- mlx_array ve = mlx_array_new();
+ mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new();
mlx_expand_dims(&qe, qn, 0, S);
mlx_expand_dims(&ke, kn, 0, S);
mlx_expand_dims(&ve, v, 0, S);
- mlx_array_free(qn);
- mlx_array_free(kn);
+ mlx_array_free(qn); mlx_array_free(kn);
- mlx_array qt = mlx_array_new();
- mlx_array kt = mlx_array_new();
- mlx_array vt = mlx_array_new();
+ mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
mlx_transpose_axes(&qt, qe, perm, 4, S);
mlx_transpose_axes(&kt, ke, perm, 4, S);
mlx_transpose_axes(&vt, ve, perm, 4, S);
- mlx_array_free(qe);
- mlx_array_free(ke);
- mlx_array_free(ve);
+ mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve);
mlx_array_free(v);
- /* ---- 2f. RoPE (NeoX-style: traditional=false) ----
- * qt: [1, n_heads, seq, head_dim]
- * kt: [1, n_kv_heads, seq, head_dim]
- * MLX rope: T = second-to-last dim = seq
- */
- mlx_optional_float base = {.value = 1000000.0f, .has_value = true};
- mlx_array null_freqs = (mlx_array){0};
+ /* RoPE (NeoX: traditional=false) */
+ mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
+ mlx_array null_arr = (mlx_array){0};
+ mlx_array qr = mlx_array_new(), kr = mlx_array_new();
+ mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_array_free(qt); mlx_array_free(kt);
- mlx_array qr = mlx_array_new();
- mlx_array kr = mlx_array_new();
- mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false,
- base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
- mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false,
- base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
-
- /* ---- 2g. Bidirectional GQA Attention ----
- * qr: [1, n_heads, seq, head_dim]
- * kr: [1, n_kv_heads, seq, head_dim]
- * vt: [1, n_kv_heads, seq, head_dim]
- * mask_mode="" > no causal mask > full bidirectional
- * GQA: SDPA broadcasts KV heads to match Q heads automatically.
- */
+ /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */
mlx_array attn = mlx_array_new();
- mlx_array null_mask = (mlx_array){0};
- mlx_array null_sinks = (mlx_array){0};
mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale,
- /*mask_mode=*/"",
- null_mask, null_sinks, S);
- mlx_array_free(qr);
- mlx_array_free(kr);
- mlx_array_free(vt);
- /* attn: [1, n_heads, seq, head_dim] */
+ &attn, qr, kr, vt, scale, "", null_arr, null_arr, S);
+ mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
- /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */
+ /* Transpose back and reshape to [seq, q_dim] */
mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */
+ mlx_transpose_axes(&attn_t, attn, perm, 4, S);
mlx_array_free(attn);
mlx_array attn_sq = mlx_array_new();
- mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */
+ mlx_squeeze_axis(&attn_sq, attn_t, 0, S);
mlx_array_free(attn_t);
- int attn_flat_shape[] = {seq, q_dim};
+ int flat_shape[] = {seq, q_dim};
mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S);
+ mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S);
mlx_array_free(attn_sq);
- /* ---- 2i. Output projection + residual ---- */
+ /* Output projection + residual */
mlx_array proj = linear(attn_flat, l->wo, S);
mlx_array_free(attn_flat);
-
mlx_array x2 = mlx_array_new();
mlx_add(&x2, x, proj, S);
- mlx_array_free(x);
- mlx_array_free(proj);
+ mlx_array_free(x); mlx_array_free(proj);
x = x2;
- /* ---- 2j. Post-attention RMSNorm ---- */
+ /* Post-attention RMSNorm */
mlx_array xn2 = mlx_array_new();
- mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
- /* ---- 2k. SwiGLU MLP ----
- * gate = x_norm @ gate_proj^T [seq, inter]
- * up = x_norm @ up_proj^T [seq, inter]
- * mid = silu(gate) * up = (gate * sigmoid(gate)) * up
- * out = mid @ down_proj^T [seq, hidden]
- */
+ /* SwiGLU MLP */
mlx_array gate = linear(xn2, l->gate_proj, S);
mlx_array up = linear(xn2, l->up_proj, S);
mlx_array_free(xn2);
- /* SiLU(gate) = gate * sigmoid(gate) */
mlx_array gate_sig = mlx_array_new();
mlx_sigmoid(&gate_sig, gate, S);
mlx_array silu_gate = mlx_array_new();
mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate);
- mlx_array_free(gate_sig);
+ mlx_array_free(gate); mlx_array_free(gate_sig);
- /* mid = silu_gate * up */
mlx_array mid = mlx_array_new();
mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate);
- mlx_array_free(up);
+ mlx_array_free(silu_gate); mlx_array_free(up);
- /* down projection */
mlx_array ffn = linear(mid, l->down_proj, S);
mlx_array_free(mid);
- /* Residual */
mlx_array x3 = mlx_array_new();
mlx_add(&x3, x, ffn, S);
- mlx_array_free(x);
- mlx_array_free(ffn);
+ mlx_array_free(x); mlx_array_free(ffn);
x = x3;
}
- /* ------------------------------------------------------------------
- * 3. Final RMSNorm
- * ------------------------------------------------------------------ */
+ /* 3. Final RMSNorm */
mlx_array x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S);
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
mlx_array_free(x);
- /* ------------------------------------------------------------------
- * 4. Mean pooling: mean(x, axis=0) [hidden]
- * ------------------------------------------------------------------ */
+ /* 4. Mean pool over seq dim */
mlx_array emb = mlx_array_new();
- mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S);
+ mlx_mean_axis(&emb, x_normed, 0, false, S);
mlx_array_free(x_normed);
- /* ------------------------------------------------------------------
- * 5. L2 normalization: emb / ||emb||_2
- * ------------------------------------------------------------------ */
+ /* 5. L2 normalize */
int norm_axes[] = {0};
- mlx_array norm_val = mlx_array_new();
- mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S);
-
- mlx_array emb_normed = mlx_array_new();
- mlx_divide(&emb_normed, emb, norm_val, S);
- mlx_array_free(emb);
- mlx_array_free(norm_val);
+ mlx_array nv = mlx_array_new();
+ mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S);
+ mlx_array emb_n = mlx_array_new();
+ mlx_divide(&emb_n, emb, nv, S);
+ mlx_array_free(emb); mlx_array_free(nv);
- /* ------------------------------------------------------------------
- * 6. Evaluate and copy result to CPU
- * ------------------------------------------------------------------ */
- mlx_array_eval(emb_normed);
-
- const float *data = mlx_array_data_float32(emb_normed);
- if (!data) {
- fprintf(stderr, "mlx: failed to get embedding data\n");
- mlx_array_free(emb_normed);
- return NULL;
+ /* 6. Eval and copy to CPU */
+ mlx_array_eval(emb_n);
+ const float *data = mlx_array_data_float32(emb_n);
+ float *out = NULL;
+ if (data) {
+ out = (float *)malloc(hidden * sizeof(float));
+ if (out) memcpy(out, data, hidden * sizeof(float));
}
-
- float *out = (float *)malloc(hidden * sizeof(float));
- if (out) memcpy(out, data, hidden * sizeof(float));
-
- mlx_array_free(emb_normed);
+ mlx_array_free(emb_n);
return out;
}
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
index 5327fd0..a1544ef 100644
--- a/pplx_embed_mlx.h
+++ b/pplx_embed_mlx.h
@@ -1,9 +1,7 @@
/*
* pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference
*
- * Uses Apple's mlx-c to run the transformer on Metal GPU.
- * All operations run on the GPU default stream; the result is copied
- * back to CPU as a float* embedding.
+ * Uses Apple's mlx-c to run the transformer on Metal.
*/
#ifndef PPLX_EMBED_MLX_H
@@ -11,25 +9,26 @@
#include <stddef.h>
#include <stdint.h>
+#include "pplx_embed.h"
/* Opaque MLX model context */
typedef struct pplx_mlx_ctx pplx_mlx_ctx_t;
/*
- * Load model into MLX arrays from safetensors in model_dir.
+ * Load model into MLX arrays from safetensors + config.json.
* Returns NULL on error.
*/
pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir);
-/*
- * Free all MLX resources.
- */
void pplx_mlx_free(pplx_mlx_ctx_t *ctx);
/*
* Compute embedding for token_ids[0..n_tokens-1].
- * Returns malloc'd float[1024] (caller frees). NULL on error.
+ * Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
+/* Get the config (so main.c can read hidden_size etc.) */
+const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx);
+
#endif /* PPLX_EMBED_MLX_H */
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index c0d9d70..0ea3cc6 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1,8 +1,3 @@
-/*
- * qwen_asr_kernels.c - Math kernels for Qwen3-ASR inference
- * Adapted from voxtral-realtime project.
- */
-
#include "qwen_asr_kernels.h"
#include "qwen_asr_kernels_impl.h"
#include <math.h>
diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c
index 0bdf272..7afe899 100644
--- a/qwen_asr_safetensors.c
+++ b/qwen_asr_safetensors.c
@@ -1,8 +1,3 @@
-/*
- * qwen_asr_safetensors.c - Safetensors reader with multi-shard support
- * Adapted from voxtral-realtime project.
- */
-
#include "qwen_asr_safetensors.h"
#include <stdio.h>
#include <stdlib.h>
diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c
index 7bbc5db..0e6df38 100644
--- a/qwen_asr_tokenizer.c
+++ b/qwen_asr_tokenizer.c
@@ -1,11 +1,3 @@
-/*
- * qwen_asr_tokenizer.c - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
- *
- * Supports:
- * - decode token IDs to UTF-8 text (for ASR output assembly)
- * - encode UTF-8 text to token IDs using vocab.json + merges.txt
- */
-
#include "qwen_asr_tokenizer.h"
#include "qwen_asr_kernels.h"
#include <limits.h>