diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Makefile | 13 | ||||
| -rw-r--r-- | README.md | 88 | ||||
| -rw-r--r-- | main.c | 479 | ||||
| -rw-r--r-- | pplx_embed.c | 459 | ||||
| -rw-r--r-- | pplx_embed.h | 147 | ||||
| -rw-r--r-- | pplx_embed_mlx.c | 478 | ||||
| -rw-r--r-- | pplx_embed_mlx.h | 15 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 5 | ||||
| -rw-r--r-- | qwen_asr_safetensors.c | 5 | ||||
| -rw-r--r-- | qwen_asr_tokenizer.c | 8 |
11 files changed, 830 insertions, 869 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a85c96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.o +pplx_embed @@ -1,7 +1,4 @@ -# pplx_embed - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b -# Makefile - -CC = gcc +CC = gcc CFLAGS_BASE = -Wall -Wextra -O3 -march=native -ffast-math LDFLAGS = -lm -lpthread @@ -27,7 +24,7 @@ TARGET = pplx_embed all: help help: - @echo "pplx_embed - Pure C inference for pplx-embed-v1-0.6b" + @echo "pplx_embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models" @echo "" @echo "Build targets:" @echo " make blas Build with BLAS acceleration (CPU)" @@ -35,7 +32,7 @@ help: @echo " make debug Debug build with AddressSanitizer" @echo " make clean Remove build artifacts" @echo "" - @echo "Usage after build:" + @echo "Usage:" @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\"" @echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)" @@ -52,8 +49,6 @@ endif blas: $(MAKE) clean $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" - @echo "" - @echo "Built: $(TARGET) (CPU/BLAS)" # ============================================================================= # MLX build (Apple Silicon GPU - uses mlx-c pure C API) @@ -75,8 +70,6 @@ endif mlx: $(MAKE) clean $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" - @echo "" - @echo "Built: $(TARGET) (CPU/BLAS + MLX GPU, use --mlx flag)" # ============================================================================= # Debug build @@ -1 +1,87 @@ -# pplx-embed-v1 +# pplx-embed + +Pure C inference for [pplx-embed](https://research.perplexity.ai/articles/pplx-embed-state-of-the-art-embedding-models-for-web-scale-retrieval) embedding models by [Perplexity AI](https://www.perplexity.ai/). Supports safetensors and hardware-accelerated execution (BLAS/mlx-c) for generating float32 vectors. + +Based on math kernels from [antirez](https://github.com/antirez)'s [qwen-asr](https://github.com/antirez/qwen-asr). + +## Supported models + +All [pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed-v1-68126b2e7cc987cd820ebbbd) models are supported. + +| Model | Dims | Params | Size | +|---|---|---|---| +| [pplx-embed-v1-0.6b](https://huggingface.co/perplexity-ai/pplx-embed-v1-0.6b) | 1024 | 0.6B | ~2.5 GB | +| [pplx-embed-v1-4b](https://huggingface.co/perplexity-ai/pplx-embed-v1-4b) | 2560 | 4B | ~16 GB | +| [pplx-embed-context-v1-0.6B](https://huggingface.co/perplexity-ai/pplx-embed-context-v1-0.6B) | 1024 | 0.6B | ~2.5 GB | +| [pplx-embed-context-v1-4B](https://huggingface.co/perplexity-ai/pplx-embed-context-v1-4B) | 2560 | 4B | ~16 GB | + +## Quickstart + +```bash +make mlx # Apple Silicon (fastest) +# or: make blas # CPU with BLAS (any platform) + +./pplx_embed -d /path/to/model "query: what is the capital of France?" +# prints 1024 (or 2560) floats to stdout + +./pplx_embed -d /path/to/model \ + "query: what is the capital of France?" \ + "document: Paris is the capital of France." \ + "document: Berlin is the capital of Germany." +# prints cosine similarity matrix +``` + +### Daemon mode + +Start the process once and feed texts through stdin. The model stays loaded in memory. + +```bash +# One embedding per line, output as JSON +./pplx_embed -d /path/to/model --daemon --mlx <<'EOF' +query: what is the capital of France? +document: Paris is the capital of France. +document: Berlin is the capital of Germany. +document: Istanbul is the capital of Germany. +EOF +``` + +Output (one JSON object per line): + +```json +{"embedding":[0.0231,-0.0412,...], "dim":1024, "tokens":9, "ms":15.2} +{"embedding":[0.0187,-0.0339,...], "dim":1024, "tokens":9, "ms":14.8} +... +``` + +## Building + +```bash +make blas # CPU with BLAS (Accelerate on macOS, OpenBLAS on Linux) +make mlx # Apple Silicon GPU via mlx-c (recommended on Apple Silicon) +make debug # Debug build with AddressSanitizer +``` + +For `make mlx`, install MLX first: `brew install mlx mlx-c` + +For `make blas` on Linux: `sudo apt install libopenblas-dev` + +## Usage + +``` +./pplx_embed -d <model_dir> [options] [text...] + +Options: + -d <dir> Model directory (required) + --mlx Use Apple MLX GPU backend + --daemon Read texts from stdin, write JSON embeddings to stdout + -t <n> CPU thread count (default: all cores) + -e Print raw embedding vectors (with multiple texts) + -v Verbose (-vv for debug) +``` + +**Single text**: prints the embedding vector as space-separated floats. + +**Multiple texts**: prints a cosine similarity matrix. + +**Daemon mode** (`--daemon`): reads one text per line from stdin, writes one JSON object per line to stdout. EOF or empty line exits. + @@ -1,25 +1,4 @@ -/* - * main.c - pplx_embed command-line tool - * - * Usage: - * pplx_embed -d <model_dir> [options] <text> [<text2> ...] - * pplx_embed -d <model_dir> [options] (read lines from stdin) - * - * Options: - * -d <dir> Model directory (required) - * -t <n> Number of threads (default: all CPU cores) - * -e Print raw embedding vector(s) instead of similarity matrix - * -v Verbose output (use -vv for debug) - * - * Output: - * 1 text : prints the 1024-dim embedding as space-separated floats - * 2+ texts: prints cosine-similarity matrix between every pair, - * then optionally the individual embeddings if -e is given - * - * Example: - * ./pplx_embed -d /path/to/model "hello world" "hi there" - * ./pplx_embed -d /path/to/model -e "query: what is AI?" - */ +/* main.c - pplx_embed command-line tool */ #include "pplx_embed.h" #include "qwen_asr_kernels.h" @@ -35,10 +14,6 @@ #include <math.h> #include <time.h> -/* ======================================================================== - * Helpers - * ======================================================================== */ - static double now_ms(void) { struct timespec ts; @@ -46,318 +21,324 @@ static double now_ms(void) return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; } -static void print_embedding(const float *emb, int dim) -{ - for (int i = 0; i < dim; i++) { - if (i > 0) putchar(' '); - printf("%.8f", (double)emb[i]); - } - putchar('\n'); -} - static void print_usage(const char *prog) { fprintf(stderr, "Usage: %s -d <model_dir> [options] [text...]\n" "\n" "Options:\n" - " -d <dir> Model directory (model.safetensors + vocab.json + merges.txt)\n" - " -t <n> Number of threads (default: all CPU cores, CPU backend)\n" - " --mlx Use Apple MLX GPU backend (Apple Silicon only)\n" - " -e Always print raw embedding vector(s)\n" - " -v Verbose (repeat for more: -vv)\n" - " -h Show this help\n" + " -d <dir> Model directory (required)\n" + " --mlx Use Apple MLX GPU backend\n" + " --daemon Read lines from stdin, write JSON embeddings to stdout\n" + " -t <n> CPU threads (default: all cores)\n" + " -e Print raw embeddings (with multiple texts)\n" + " -v Verbose (-vv for debug)\n" + " -h Show this help\n" "\n" - "With 1 text: prints the 1024-dim embedding vector\n" - "With 2+ texts: prints cosine similarity between each pair\n" - " (add -e to also print the individual embeddings)\n" + "Modes:\n" + " 1 text arg Print embedding as space-separated floats\n" + " 2+ text args Print cosine similarity matrix\n" + " no args Batch: read all stdin lines, then similarity matrix\n" + " --daemon Streaming: read stdin lines, write JSON per line\n" "\n" - "If no text arguments are given, reads one text per line from stdin.\n" - "\n" - "Example:\n" - " %s -d ./model \"query: what is AI?\" \"document: AI is artificial intelligence\"\n" - " %s -d ./model --mlx \"query: what is AI?\"\n", + "Examples:\n" + " %s -d ./model \"query: what is AI?\"\n" + " %s -d ./model --mlx --daemon < texts.txt\n", prog, prog, prog); } /* ======================================================================== - * Process one text: tokenize -> embed, returns malloc'd float[dim] + * Embed one text: tokenize > forward > return float[dim] * ======================================================================== */ -static float *process_text(pplx_ctx_t *ctx, +typedef struct { + pplx_ctx_t *ctx; #ifdef USE_MLX - pplx_mlx_ctx_t *mlx_ctx, + pplx_mlx_ctx_t *mlx_ctx; #endif - qwen_tokenizer_t *tok, - const char *text, - int print_tokens) + qwen_tokenizer_t *tok; + int dim; +} engine_t; + +typedef struct { + float *emb; /* malloc'd float[dim], caller frees */ + int n_tokens; + double ms; +} embed_result_t; + +static embed_result_t embed_text(engine_t *e, const char *text) { - /* Tokenize */ - int n_tokens = 0; - int *token_ids = qwen_tokenizer_encode(tok, text, &n_tokens); - if (!token_ids || n_tokens == 0) { - fprintf(stderr, "pplx_embed: tokenization failed for: %s\n", text); - free(token_ids); - return NULL; + embed_result_t r = {0}; + + int n = 0; + int *ids = qwen_tokenizer_encode(e->tok, text, &n); + if (!ids || n == 0) { + fprintf(stderr, "tokenization failed: %s\n", text); + free(ids); + return r; } - if (print_tokens || pplx_verbose >= 1) { - fprintf(stderr, "tokens (%d): ", n_tokens); - for (int i = 0; i < n_tokens && i < 20; i++) - fprintf(stderr, "%d ", token_ids[i]); - if (n_tokens > 20) fprintf(stderr, "..."); + if (pplx_verbose >= 1) { + fprintf(stderr, "tokens (%d): ", n); + for (int i = 0; i < n && i < 20; i++) fprintf(stderr, "%d ", ids[i]); + if (n > 20) fprintf(stderr, "..."); fprintf(stderr, "\n"); } - /* Embed */ - double t0 = now_ms(); - float *emb = NULL; + double t0 = now_ms(); #ifdef USE_MLX - if (mlx_ctx) - emb = pplx_mlx_embed(mlx_ctx, token_ids, n_tokens); + if (e->mlx_ctx) + r.emb = pplx_mlx_embed(e->mlx_ctx, ids, n); else #endif - emb = pplx_embed(ctx, token_ids, n_tokens); - double dt = now_ms() - t0; - free(token_ids); - - if (!emb) { - fprintf(stderr, "pplx_embed: forward pass failed\n"); - return NULL; - } + r.emb = pplx_embed(e->ctx, ids, n); + r.ms = now_ms() - t0; + r.n_tokens = n; + free(ids); + if (!r.emb) fprintf(stderr, "forward pass failed\n"); if (pplx_verbose >= 1) - fprintf(stderr, "embed: %d tokens in %.1f ms\n", n_tokens, dt); + fprintf(stderr, "embed: %d tokens in %.1f ms\n", r.n_tokens, r.ms); - return emb; + return r; } /* ======================================================================== - * Main + * Output helpers * ======================================================================== */ -int main(int argc, char *argv[]) +static void print_embedding_raw(const float *emb, int dim) { - const char *model_dir = NULL; - int n_threads = 0; /* 0 = auto */ - int print_embs = 0; - int verbose = 0; - int use_mlx = 0; - - /* ---- Parse flags ---- */ - int arg_start = 1; - while (arg_start < argc && argv[arg_start][0] == '-') { - const char *flag = argv[arg_start]; - if (strcmp(flag, "-d") == 0) { - if (arg_start + 1 >= argc) { - fprintf(stderr, "pplx_embed: -d requires an argument\n"); - return 1; - } - model_dir = argv[++arg_start]; - } else if (strcmp(flag, "-t") == 0) { - if (arg_start + 1 >= argc) { - fprintf(stderr, "pplx_embed: -t requires an argument\n"); - return 1; - } - n_threads = atoi(argv[++arg_start]); - } else if (strcmp(flag, "-e") == 0) { - print_embs = 1; - } else if (strcmp(flag, "--mlx") == 0) { -#ifdef USE_MLX - use_mlx = 1; -#else - fprintf(stderr, "pplx_embed: --mlx not available (build with: make mlx)\n"); - return 1; -#endif - } else if (strcmp(flag, "-v") == 0) { - verbose++; - } else if (strcmp(flag, "-vv") == 0) { - verbose = 2; - } else if (strcmp(flag, "-h") == 0 || strcmp(flag, "--help") == 0) { - print_usage(argv[0]); - return 0; - } else { - /* Stop flag parsing on first non-flag argument */ - break; - } - arg_start++; + for (int i = 0; i < dim; i++) { + if (i > 0) putchar(' '); + printf("%.8f", (double)emb[i]); } + putchar('\n'); +} - if (!model_dir || model_dir[0] == '\0') { - fprintf(stderr, "pplx_embed: model directory required (-d <dir>)\n" - " Hint: if using MODEL=.. ./pplx_embed -d \"$MODEL\",\n" - " make sure MODEL is exported or use: MODEL=.. && ./pplx_embed -d \"$MODEL\"\n"); - print_usage(argv[0]); - return 1; +static void print_embedding_json(const float *emb, int dim, int n_tokens, double ms) +{ + printf("{\"embedding\":["); + for (int i = 0; i < dim; i++) { + if (i > 0) putchar(','); + printf("%.8f", (double)emb[i]); } + printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f}\n", dim, n_tokens, ms); + fflush(stdout); +} - /* ---- Set verbosity ---- */ - pplx_verbose = verbose; - qwen_verbose = verbose; /* kernels share the same flag name */ - - /* ---- Set thread count (CPU backend only) ---- */ - if (!use_mlx) { - if (n_threads <= 0) - n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); - if (verbose >= 1) - fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else { - if (verbose >= 1) - fprintf(stderr, "Using MLX GPU backend\n"); - } +/* ======================================================================== + * Daemon mode: read lines from stdin, write JSON to stdout + * ======================================================================== */ - /* ---- Load tokenizer ---- */ - char vocab_path[1024]; - snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); +static int run_daemon(engine_t *e) +{ + char line[65536]; - double t0_load = now_ms(); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); - if (!tok) { - fprintf(stderr, "pplx_embed: failed to load tokenizer from %s\n", vocab_path); - return 1; - } - if (verbose >= 1) - fprintf(stderr, "Tokenizer loaded in %.0f ms\n", now_ms() - t0_load); + if (pplx_verbose >= 1) + fprintf(stderr, "daemon: ready, reading from stdin\n"); - /* ---- Load model ---- */ - pplx_ctx_t *ctx = NULL; -#ifdef USE_MLX - pplx_mlx_ctx_t *mlx_ctx = NULL; -#endif + while (fgets(line, sizeof(line), stdin)) { + /* Strip newline */ + size_t len = strlen(line); + while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r')) + line[--len] = '\0'; + if (len == 0) continue; - double t0_model = now_ms(); + if (pplx_verbose >= 1) + fprintf(stderr, "daemon: \"%.*s%s\"\n", + (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); -#ifdef USE_MLX - if (use_mlx) { - mlx_ctx = pplx_mlx_load(model_dir); - if (!mlx_ctx) { - fprintf(stderr, "pplx_embed: failed to load MLX model from %s\n", model_dir); - qwen_tokenizer_free(tok); - return 1; - } - } else -#endif - { - ctx = pplx_load(model_dir); - if (!ctx) { - fprintf(stderr, "pplx_embed: failed to load model from %s\n", model_dir); - qwen_tokenizer_free(tok); - return 1; + embed_result_t r = embed_text(e, line); + if (r.emb) { + print_embedding_json(r.emb, e->dim, r.n_tokens, r.ms); + free(r.emb); + } else { + /* Output an error line so the consumer stays in sync */ + printf("{\"error\":\"embedding failed\"}\n"); + fflush(stdout); } } - if (verbose >= 1) - fprintf(stderr, "Model loaded in %.0f ms%s\n", now_ms() - t0_model, - use_mlx ? " (MLX)" : ""); - int dim = PPLX_HIDDEN_SIZE; + if (pplx_verbose >= 1) + fprintf(stderr, "daemon: stdin EOF\n"); - /* ---- Gather input texts ---- */ - /* texts[] and embeddings[] are grown dynamically */ - int n_texts = 0; - int texts_cap = 0; - char **texts = NULL; - float **embeddings = NULL; + return 0; +} - /* Helper to append a text */ -#define APPEND_TEXT(s) do { \ - if (n_texts == texts_cap) { \ - texts_cap = texts_cap ? texts_cap * 2 : 8; \ - texts = realloc(texts, texts_cap * sizeof(char *)); \ - embeddings = realloc(embeddings, texts_cap * sizeof(float *)); \ - if (!texts || !embeddings) { \ - fprintf(stderr, "pplx_embed: out of memory\n"); return 1; \ - } \ - } \ - texts[n_texts] = strdup(s); \ - embeddings[n_texts] = NULL; \ - n_texts++; \ +/* ======================================================================== + * Batch mode: embed args or stdin lines, then print similarity or vectors + * ======================================================================== */ + +static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int print_embs) +{ + int n_texts = 0, cap = 0; + char **texts = NULL; + float **embs = NULL; + +#define APPEND(s) do { \ + if (n_texts == cap) { \ + cap = cap ? cap * 2 : 8; \ + texts = realloc(texts, cap * sizeof(char *)); \ + embs = realloc(embs, cap * sizeof(float *)); \ + if (!texts || !embs) { fprintf(stderr, "OOM\n"); return 1; } \ + } \ + texts[n_texts] = strdup(s); \ + embs[n_texts] = NULL; \ + n_texts++; \ } while (0) if (arg_start < argc) { - /* Text arguments on command line */ - for (int i = arg_start; i < argc; i++) - APPEND_TEXT(argv[i]); + for (int i = arg_start; i < argc; i++) APPEND(argv[i]); } else { - /* Read lines from stdin */ char line[65536]; while (fgets(line, sizeof(line), stdin)) { - /* Strip trailing newline */ size_t l = strlen(line); while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) line[--l] = '\0'; if (l == 0) continue; - APPEND_TEXT(line); + APPEND(line); } } +#undef APPEND if (n_texts == 0) { - fprintf(stderr, "pplx_embed: no input texts\n"); - pplx_free(ctx); - qwen_tokenizer_free(tok); + fprintf(stderr, "no input texts\n"); + free(texts); free(embs); return 1; } - /* ---- Embed all texts ---- */ + int dim = e->dim; + for (int i = 0; i < n_texts; i++) { - if (verbose >= 1) + if (pplx_verbose >= 1) fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); - - embeddings[i] = process_text(ctx, -#ifdef USE_MLX - mlx_ctx, -#endif - tok, texts[i], - /*print_tokens=*/ verbose >= 2); - if (!embeddings[i]) { - /* Error already printed */ - goto cleanup; - } + embed_result_t r = embed_text(e, texts[i]); + embs[i] = r.emb; + if (!embs[i]) goto done; } - /* ---- Output ---- */ if (n_texts == 1) { - /* Single text: always print the embedding */ - print_embedding(embeddings[0], dim); + print_embedding_raw(embs[0], dim); } else { - /* Multiple texts: print cosine similarity matrix */ printf("Cosine similarity matrix (%d texts):\n", n_texts); - - /* Header */ printf("%-4s", ""); for (int j = 0; j < n_texts; j++) printf(" [%d] ", j); printf("\n"); - for (int i = 0; i < n_texts; i++) { printf("[%d] ", i); - for (int j = 0; j < n_texts; j++) { - float sim = pplx_cosine_similarity(embeddings[i], embeddings[j], dim); - printf(" %.4f", (double)sim); - } + for (int j = 0; j < n_texts; j++) + printf(" %.4f", (double)pplx_cosine_similarity(embs[i], embs[j], dim)); printf(" \"%s\"\n", texts[i]); } - - /* Print raw embeddings if requested */ if (print_embs) { printf("\nEmbeddings:\n"); for (int i = 0; i < n_texts; i++) { printf("[%d] ", i); - print_embedding(embeddings[i], dim); + print_embedding_raw(embs[i], dim); } } } -cleanup: - for (int i = 0; i < n_texts; i++) { - free(texts[i]); - free(embeddings[i]); +done: + for (int i = 0; i < n_texts; i++) { free(texts[i]); free(embs[i]); } + free(texts); free(embs); + return 0; +} + +/* ======================================================================== + * Main + * ======================================================================== */ + +int main(int argc, char *argv[]) +{ + const char *model_dir = NULL; + int n_threads = 0; + int print_embs = 0; + int verbose = 0; + int use_mlx = 0; + int daemon = 0; + + int arg_start = 1; + while (arg_start < argc && argv[arg_start][0] == '-') { + const char *f = argv[arg_start]; + if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } + else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); } + else if (!strcmp(f, "-e")) { print_embs = 1; } + else if (!strcmp(f, "-v")) { verbose++; } + else if (!strcmp(f, "-vv")) { verbose = 2; } + else if (!strcmp(f, "--daemon")){ daemon = 1; } + else if (!strcmp(f, "--mlx")) { +#ifdef USE_MLX + use_mlx = 1; +#else + fprintf(stderr, "--mlx not available (build with: make mlx)\n"); return 1; +#endif + } + else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); return 0; } + else break; + arg_start++; + } + + if (!model_dir || !model_dir[0]) { + fprintf(stderr, "model directory required (-d <dir>)\n"); + print_usage(argv[0]); + return 1; + } + + pplx_verbose = verbose; + qwen_verbose = verbose; + + /* Threads (CPU backend) */ + if (!use_mlx) { + if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else { + if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + } + + /* Tokenizer */ + char vocab_path[1024]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); + double t0 = now_ms(); + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; } + if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); + + /* Model */ + engine_t e = {0}; + e.tok = tok; + t0 = now_ms(); + +#ifdef USE_MLX + if (use_mlx) { + e.mlx_ctx = pplx_mlx_load(model_dir); + if (!e.mlx_ctx) { fprintf(stderr, "failed to load model\n"); return 1; } + e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size; + } else +#endif + { + e.ctx = pplx_load(model_dir); + if (!e.ctx) { fprintf(stderr, "failed to load model\n"); return 1; } + e.dim = e.ctx->config.hidden_size; } - free(texts); - free(embeddings); - if (ctx) pplx_free(ctx); + if (verbose >= 1) + fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", + e.dim, now_ms() - t0, use_mlx ? " (MLX)" : ""); + + /* Run */ + int rc; + if (daemon) + rc = run_daemon(&e); + else + rc = run_batch(&e, argc, argv, arg_start, print_embs); + + /* Cleanup */ + if (e.ctx) pplx_free(e.ctx); #ifdef USE_MLX - if (mlx_ctx) pplx_mlx_free(mlx_ctx); + if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif qwen_tokenizer_free(tok); - return 0; + return rc; } diff --git a/pplx_embed.c b/pplx_embed.c index 2220760..dcd1acd 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -1,21 +1,3 @@ -/* - * pplx_embed.c - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b - * - * Transformer forward pass: - * embed -> 28x[RMSNorm -> QKV (no bias) -> Q/K RMSNorm -> RoPE - * -> Bidirectional GQA attention -> O proj -> residual - * -> RMSNorm -> SwiGLU MLP -> residual] - * -> final RMSNorm -> mean pool -> L2 normalize - * - * Key implementation choices: - * - All weights are F32 (mmap'd direct, no copy) - * - Bidirectional attention: reuse qwen_causal_attention with - * q_offset = (seq_len - 1), which removes the causal mask - * (every query sees all seq_len keys) - * - GQA ratio 2:1 (16 Q-heads, 8 KV-heads) handled by causal_attention - * - RoPE cache grown lazily, positions [0 .. seq_len-1] - */ - #include "pplx_embed.h" #include "qwen_asr_kernels.h" #include "qwen_asr_safetensors.h" @@ -26,32 +8,128 @@ #include <math.h> /* ======================================================================== - * Global verbose flags - * qwen_verbose is declared extern in qwen_asr_kernels.h and qwen_asr_tokenizer.c; - * we own its definition here since we don't link qwen_asr.c. + * Globals * ======================================================================== */ int pplx_verbose = 0; -int qwen_verbose = 0; /* required by qwen_asr_kernels.c and qwen_asr_tokenizer.c */ +int qwen_verbose = 0; /* ======================================================================== - * Internal helpers: weight loading + * Minimal JSON helpers for config.json parsing + * ======================================================================== */ + +static const char *skip_ws(const char *p) +{ + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + return p; +} + +/* Find "key": <value> in a flat JSON object. Returns pointer to value start. */ +static const char *json_find_key(const char *json, const char *key) +{ + size_t klen = strlen(key); + const char *p = json; + while ((p = strstr(p, key)) != NULL) { + /* Check it's actually a quoted key: "key" */ + if (p > json && *(p - 1) == '"') { + const char *after = p + klen; + if (*after == '"') { + after = skip_ws(after + 1); + if (*after == ':') return skip_ws(after + 1); + } + } + p += klen; + } + return NULL; +} + +static int json_get_int(const char *json, const char *key, int fallback) +{ + const char *v = json_find_key(json, key); + if (!v) return fallback; + return atoi(v); +} + +static double json_get_double(const char *json, const char *key, double fallback) +{ + const char *v = json_find_key(json, key); + if (!v) return fallback; + return atof(v); +} + +static int parse_config(pplx_config_t *cfg, const char *model_dir) +{ + char path[1024]; + snprintf(path, sizeof(path), "%s/config.json", model_dir); + + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "pplx_load: cannot open %s\n", path); + return -1; + } + + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, 0, SEEK_SET); + char *buf = (char *)malloc(sz + 1); + if (!buf || fread(buf, 1, sz, f) != (size_t)sz) { + fclose(f); free(buf); return -1; + } + fclose(f); + buf[sz] = '\0'; + + cfg->hidden_size = json_get_int(buf, "hidden_size", 0); + cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); + cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); + cfg->head_dim = json_get_int(buf, "head_dim", PPLX_HEAD_DIM); + cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); + cfg->vocab_size = json_get_int(buf, "vocab_size", PPLX_VOCAB_SIZE); + cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); + cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + + cfg->q_dim = cfg->n_heads * cfg->head_dim; + cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; + + free(buf); + + /* Sanity checks */ + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0) { + fprintf(stderr, "pplx_load: invalid config in %s " + "(hidden=%d, layers=%d, heads=%d)\n", + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads); + return -1; + } + if (cfg->n_layers > PPLX_MAX_LAYERS) { + fprintf(stderr, "pplx_load: too many layers (%d > %d)\n", + cfg->n_layers, PPLX_MAX_LAYERS); + return -1; + } + + if (pplx_verbose >= 1) + fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " + "inter=%d, head_dim=%d\n", + cfg->hidden_size, cfg->n_layers, + cfg->n_heads, cfg->n_kv_heads, + cfg->intermediate_size, cfg->head_dim); + + return 0; +} + +/* ======================================================================== + * Weight loading (direct mmap pointers into safetensors) * ======================================================================== */ -/* - * Return a direct pointer into the mmap'd safetensors region for an F32 tensor. - * No allocation, no copy; caller must NOT free the returned pointer. - */ static const float *load_f32_direct(multi_safetensors_t *ms, const char *name) { safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { - fprintf(stderr, "pplx_embed: tensor not found: %s\n", name); + fprintf(stderr, "pplx: tensor not found: %s\n", name); return NULL; } if (t->dtype != DTYPE_F32) { - fprintf(stderr, "pplx_embed: expected F32 for %s, got dtype=%d\n", + fprintf(stderr, "pplx: expected F32 for %s, got dtype=%d\n", name, (int)t->dtype); return NULL; } @@ -59,96 +137,81 @@ static const float *load_f32_direct(multi_safetensors_t *ms, const char *name) } /* ======================================================================== - * Internal helpers: working buffers + * Working buffer management * ======================================================================== */ static int ensure_buffers(pplx_ctx_t *ctx, int seq) { if (seq <= ctx->buf_seq_cap) return 0; - int new_cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64; - while (new_cap < seq) new_cap *= 2; + int cap = ctx->buf_seq_cap > 0 ? ctx->buf_seq_cap : 64; + while (cap < seq) cap *= 2; - int hidden = ctx->config.hidden_size; - int q_dim = ctx->config.q_dim; - int kv_dim = ctx->config.kv_dim; - int inter = ctx->config.intermediate_size; + const pplx_config_t *c = &ctx->config; -#define PPLX_REALLOC(ptr, n) do { \ - void *_tmp = realloc((ptr), (size_t)(n) * sizeof(float)); \ - if (!_tmp) return -1; \ - (ptr) = (float *)_tmp; \ +#define R(ptr, n) do { \ + void *_t = realloc((ptr), (size_t)(n) * sizeof(float)); \ + if (!_t) return -1; \ + (ptr) = (float *)_t; \ } while (0) - PPLX_REALLOC(ctx->x, new_cap * hidden); - PPLX_REALLOC(ctx->x_norm, new_cap * hidden); - PPLX_REALLOC(ctx->q, new_cap * q_dim); - PPLX_REALLOC(ctx->k, new_cap * kv_dim); - PPLX_REALLOC(ctx->v, new_cap * kv_dim); - PPLX_REALLOC(ctx->attn_out, new_cap * q_dim); - PPLX_REALLOC(ctx->proj_out, new_cap * hidden); - PPLX_REALLOC(ctx->ffn_gate, new_cap * inter); - PPLX_REALLOC(ctx->ffn_up, new_cap * inter); - PPLX_REALLOC(ctx->ffn_out, new_cap * hidden); + R(ctx->x, cap * c->hidden_size); + R(ctx->x_norm, cap * c->hidden_size); + R(ctx->q, cap * c->q_dim); + R(ctx->k, cap * c->kv_dim); + R(ctx->v, cap * c->kv_dim); + R(ctx->attn_out, cap * c->q_dim); + R(ctx->proj_out, cap * c->hidden_size); + R(ctx->ffn_gate, cap * c->intermediate_size); + R(ctx->ffn_up, cap * c->intermediate_size); + R(ctx->ffn_out, cap * c->hidden_size); -#undef PPLX_REALLOC +#undef R - ctx->buf_seq_cap = new_cap; + ctx->buf_seq_cap = cap; return 0; } /* ======================================================================== - * Internal helpers: RoPE cache - * - * Stores cos/sin for positions [0 .. rope_cache_cap-1]. - * Layout: [pos, head_dim] - values are duplicated across the two halves - * so that qwen_apply_rope_neox can load contiguous cos[d] and cos[half+d] - * from a single row: cos_row[d] == cos_row[half+d]. + * RoPE cache * ======================================================================== */ static int ensure_rope_cache(pplx_ctx_t *ctx, int n_pos) { if (n_pos <= ctx->rope_cache_cap) return 0; - int new_cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512; - while (new_cap < n_pos) new_cap *= 2; + int cap = ctx->rope_cache_cap > 0 ? ctx->rope_cache_cap : 512; + while (cap < n_pos) cap *= 2; int head_dim = ctx->config.head_dim; int half = head_dim / 2; float theta = ctx->config.rope_theta; - size_t n = (size_t)new_cap * head_dim; - float *new_cos = (float *)realloc(ctx->rope_cos, n * sizeof(float)); - float *new_sin = (float *)realloc(ctx->rope_sin, n * sizeof(float)); - if (!new_cos || !new_sin) { - /* If realloc failed, partial state is still valid up to old cap */ - if (new_cos) ctx->rope_cos = new_cos; - if (new_sin) ctx->rope_sin = new_sin; + size_t n = (size_t)cap * head_dim; + float *nc = (float *)realloc(ctx->rope_cos, n * sizeof(float)); + float *ns = (float *)realloc(ctx->rope_sin, n * sizeof(float)); + if (!nc || !ns) { + if (nc) ctx->rope_cos = nc; + if (ns) ctx->rope_sin = ns; return -1; } - ctx->rope_cos = new_cos; - ctx->rope_sin = new_sin; + ctx->rope_cos = nc; + ctx->rope_sin = ns; - /* Compute only the newly needed positions */ - for (int pos = ctx->rope_cache_cap; pos < new_cap; pos++) { + for (int pos = ctx->rope_cache_cap; pos < cap; pos++) { float fp = (float)pos; - float *c = ctx->rope_cos + (size_t)pos * head_dim; - float *s = ctx->rope_sin + (size_t)pos * head_dim; + float *c = ctx->rope_cos + (size_t)pos * head_dim; + float *s = ctx->rope_sin + (size_t)pos * head_dim; for (int d = 0; d < half; d++) { - /* inv_freq[d] = 1 / (theta ^ (2d / head_dim)) */ - float inv_freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = fp * inv_freq; - float cv = cosf(angle); - float sv = sinf(angle); - /* Duplicate across both halves for the NeoX rotation kernel */ - c[d] = cv; - c[half + d] = cv; - s[d] = sv; - s[half + d] = sv; + float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = fp * inv; + float cv = cosf(angle), sv = sinf(angle); + c[d] = cv; c[half + d] = cv; + s[d] = sv; s[half + d] = sv; } } - ctx->rope_cache_cap = new_cap; + ctx->rope_cache_cap = cap; return 0; } @@ -161,21 +224,15 @@ pplx_ctx_t *pplx_load(const char *model_dir) pplx_ctx_t *ctx = (pplx_ctx_t *)calloc(1, sizeof(pplx_ctx_t)); if (!ctx) return NULL; - /* ---- Hardcoded config (mirrors config.json) ---- */ - pplx_config_t *cfg = &ctx->config; - cfg->hidden_size = PPLX_HIDDEN_SIZE; - cfg->n_layers = PPLX_NUM_LAYERS; - cfg->n_heads = PPLX_NUM_HEADS; - cfg->n_kv_heads = PPLX_NUM_KV_HEADS; - cfg->head_dim = PPLX_HEAD_DIM; - cfg->q_dim = PPLX_Q_DIM; - cfg->kv_dim = PPLX_KV_DIM; - cfg->intermediate_size = PPLX_INTERMEDIATE_SIZE; - cfg->vocab_size = PPLX_VOCAB_SIZE; - cfg->rms_norm_eps = 1e-6f; - cfg->rope_theta = 1000000.0f; + /* Parse config.json */ + if (parse_config(&ctx->config, model_dir) != 0) { + free(ctx); + return NULL; + } + + const pplx_config_t *cfg = &ctx->config; - /* ---- Open safetensors ---- */ + /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); if (!ms) { fprintf(stderr, "pplx_load: failed to open safetensors in %s\n", model_dir); @@ -187,12 +244,16 @@ pplx_ctx_t *pplx_load(const char *model_dir) if (pplx_verbose >= 1) fprintf(stderr, "pplx_load: loading weights from %s\n", model_dir); - /* ---- Token embeddings ---- */ - pplx_weights_t *w = &ctx->weights; + /* Token embeddings */ + pplx_weights_t *w = &ctx->weights; w->embed_tokens = load_f32_direct(ms, "embed_tokens.weight"); if (!w->embed_tokens) goto fail; - /* ---- Per-layer weights ---- */ + /* Allocate layer array */ + w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t)); + if (!w->layers) goto fail; + + /* Per-layer weights */ char name[256]; for (int i = 0; i < cfg->n_layers; i++) { pplx_layer_t *l = &w->layers[i]; @@ -221,12 +282,13 @@ pplx_ctx_t *pplx_load(const char *model_dir) fprintf(stderr, " layer %d loaded\n", i); } - /* ---- Final RMSNorm ---- */ + /* Final RMSNorm */ w->norm = load_f32_direct(ms, "norm.weight"); if (!w->norm) goto fail; if (pplx_verbose >= 1) - fprintf(stderr, "pplx_load: all weights loaded\n"); + fprintf(stderr, "pplx_load: %d layers loaded (%d-dim embeddings)\n", + cfg->n_layers, cfg->hidden_size); return ctx; @@ -244,34 +306,27 @@ void pplx_free(pplx_ctx_t *ctx) if (!ctx) return; if (ctx->safetensors) multi_safetensors_close((multi_safetensors_t *)ctx->safetensors); - - free(ctx->x); - free(ctx->x_norm); - free(ctx->q); - free(ctx->k); - free(ctx->v); - free(ctx->attn_out); - free(ctx->proj_out); - free(ctx->ffn_gate); - free(ctx->ffn_up); - free(ctx->ffn_out); - free(ctx->rope_cos); - free(ctx->rope_sin); + free(ctx->weights.layers); + free(ctx->x); free(ctx->x_norm); + free(ctx->q); free(ctx->k); free(ctx->v); + free(ctx->attn_out); free(ctx->proj_out); + free(ctx->ffn_gate); free(ctx->ffn_up); free(ctx->ffn_out); + free(ctx->rope_cos); free(ctx->rope_sin); free(ctx); } /* ======================================================================== - * Forward pass + * Forward pass (returns full token-level embeddings) * ======================================================================== */ -float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; const pplx_config_t *cfg = &ctx->config; const pplx_weights_t *w = &ctx->weights; - int hidden = cfg->hidden_size; + int hidden = cfg->hidden_size; int n_heads = cfg->n_heads; int n_kv_heads = cfg->n_kv_heads; int head_dim = cfg->head_dim; @@ -281,39 +336,28 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) float eps = cfg->rms_norm_eps; int seq = n_tokens; - /* Allocate/grow working buffers */ - if (ensure_buffers(ctx, seq) != 0) { - fprintf(stderr, "pplx_embed: buffer allocation failed\n"); - return NULL; - } + if (ensure_buffers(ctx, seq) != 0) return NULL; + if (ensure_rope_cache(ctx, seq) != 0) return NULL; - /* Pre-compute RoPE tables for positions [0, seq-1] */ - if (ensure_rope_cache(ctx, seq) != 0) { - fprintf(stderr, "pplx_embed: RoPE cache allocation failed\n"); - return NULL; - } - const float *rope_cos = ctx->rope_cos; /* [seq_cap, head_dim] */ + const float *rope_cos = ctx->rope_cos; const float *rope_sin = ctx->rope_sin; float *x = ctx->x; float *x_norm = ctx->x_norm; - float *q = ctx->q; - float *k = ctx->k; - float *v = ctx->v; + float *q_buf = ctx->q; + float *k_buf = ctx->k; + float *v_buf = ctx->v; float *attn_out = ctx->attn_out; float *proj_out = ctx->proj_out; float *ffn_gate = ctx->ffn_gate; float *ffn_up = ctx->ffn_up; float *ffn_out = ctx->ffn_out; - /* ------------------------------------------------------------------ - * Step 1: Token embedding lookup - * x[i, :] = embed_tokens[token_ids[i], :] - * ------------------------------------------------------------------ */ + /* 1. Token embedding lookup */ for (int i = 0; i < seq; i++) { int id = token_ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "pplx_embed: invalid token id %d at position %d\n", id, i); + fprintf(stderr, "pplx: invalid token id %d at position %d\n", id, i); return NULL; } memcpy(x + i * hidden, @@ -323,120 +367,97 @@ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) float scale = 1.0f / sqrtf((float)head_dim); - /* ------------------------------------------------------------------ - * Steps 2..29: 28 transformer layers - * ------------------------------------------------------------------ */ + /* 2. Transformer layers */ for (int layer = 0; layer < cfg->n_layers; layer++) { const pplx_layer_t *l = &w->layers[layer]; - /* ---- 2a. Input RMSNorm ---- */ + /* Input RMSNorm */ qwen_rms_norm(x_norm, x, l->input_norm, seq, hidden, eps); - /* ---- 2b. QKV projections (no bias) ---- - * q: [seq, q_dim] = x_norm @ wq^T (wq is [q_dim, hidden]) - * k: [seq, kv_dim] = x_norm @ wk^T - * v: [seq, kv_dim] = x_norm @ wv^T - */ - qwen_linear_nobias(q, x_norm, l->wq, seq, hidden, q_dim); - qwen_linear_nobias(k, x_norm, l->wk, seq, hidden, kv_dim); - qwen_linear_nobias(v, x_norm, l->wv, seq, hidden, kv_dim); + /* QKV projections */ + qwen_linear_nobias(q_buf, x_norm, l->wq, seq, hidden, q_dim); + qwen_linear_nobias(k_buf, x_norm, l->wk, seq, hidden, kv_dim); + qwen_linear_nobias(v_buf, x_norm, l->wv, seq, hidden, kv_dim); - /* ---- 2c. Per-head Q/K RMSNorm (Qwen3 feature) ---- - * Normalize each head's [head_dim] segment independently. - */ - qwen_rms_norm_per_head(q, l->q_norm, seq, n_heads, head_dim, eps); - qwen_rms_norm_per_head(k, l->k_norm, seq, n_kv_heads, head_dim, eps); + /* Per-head Q/K RMSNorm */ + qwen_rms_norm_per_head(q_buf, l->q_norm, seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(k_buf, l->k_norm, seq, n_kv_heads, head_dim, eps); - /* ---- 2d. Apply NeoX-style RoPE to Q and K ---- - * rope_cos/sin: [seq_cap, head_dim], positions 0..seq-1 cached - */ - qwen_apply_rope_neox(q, rope_cos, rope_sin, seq, n_heads, head_dim); - qwen_apply_rope_neox(k, rope_cos, rope_sin, seq, n_kv_heads, head_dim); + /* RoPE */ + qwen_apply_rope_neox(q_buf, rope_cos, rope_sin, seq, n_heads, head_dim); + qwen_apply_rope_neox(k_buf, rope_cos, rope_sin, seq, n_kv_heads, head_dim); - /* ---- 2e. Bidirectional GQA attention ---- - * - * We reuse qwen_causal_attention with q_offset = (seq - 1). - * - * Inside the kernel for query position i: - * global_pos = q_offset + i = (seq-1) + i - * k_end = min(global_pos + 1, seq_k) - * = min(seq + i, seq) = seq for all i >= 0 - * - * So every query attends to ALL seq key positions - full attention, - * no causal mask, while still benefiting from the GQA + threading - * in qwen_causal_attention. - */ - qwen_causal_attention(attn_out, q, k, v, - seq, /* seq_q */ - seq, /* seq_k */ - n_heads, n_kv_heads, head_dim, - scale, - seq - 1); /* q_offset - removes causal mask */ + /* Bidirectional GQA attention (q_offset = seq-1 removes causal mask) */ + qwen_causal_attention(attn_out, q_buf, k_buf, v_buf, + seq, seq, n_heads, n_kv_heads, + head_dim, scale, seq - 1); - /* ---- 2f. Output projection + residual ---- - * proj_out = attn_out @ wo^T (wo is [hidden, q_dim]) - * x += proj_out - */ + /* Output projection + residual */ qwen_linear_nobias(proj_out, attn_out, l->wo, seq, q_dim, hidden); qwen_add_inplace(x, proj_out, seq * hidden); - /* ---- 2g. Post-attention RMSNorm ---- */ + /* Post-attention RMSNorm */ qwen_rms_norm(x_norm, x, l->post_attn_norm, seq, hidden, eps); - /* ---- 2h. SwiGLU MLP ---- - * gate = gate_proj(x_norm) [seq, intermediate] - * up = up_proj(x_norm) [seq, intermediate] - * mid = SiLU(gate) * up [seq, intermediate] - * out = down_proj(mid) [seq, hidden] - */ + /* SwiGLU MLP */ qwen_linear_nobias(ffn_gate, x_norm, l->gate_proj, seq, hidden, inter); qwen_linear_nobias(ffn_up, x_norm, l->up_proj, seq, hidden, inter); - - qwen_silu(ffn_gate, seq * inter); /* SiLU(gate) in-place */ - qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); /* gate *= up */ - + qwen_silu(ffn_gate, seq * inter); + qwen_mul_inplace(ffn_gate, ffn_up, seq * inter); qwen_linear_nobias(ffn_out, ffn_gate, l->down_proj, seq, inter, hidden); qwen_add_inplace(x, ffn_out, seq * hidden); } - /* ------------------------------------------------------------------ - * Step 3: Final RMSNorm (in-place on x) - * ------------------------------------------------------------------ */ + /* 3. Final RMSNorm (in-place) */ qwen_rms_norm(x, x, w->norm, seq, hidden, eps); - /* ------------------------------------------------------------------ - * Step 4: Mean pooling across all token positions - * embedding[d] = mean over i in [0, seq) of x[i, d] - * ------------------------------------------------------------------ */ - float *embedding = (float *)calloc(hidden, sizeof(float)); - if (!embedding) return NULL; + /* Copy result out (caller frees) */ + size_t out_size = (size_t)seq * hidden * sizeof(float); + float *out = (float *)malloc(out_size); + if (out) memcpy(out, x, out_size); + return out; +} + +/* ======================================================================== + * Embed (forward + mean pool + L2 norm) + * ======================================================================== */ + +float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + float *all = pplx_forward(ctx, token_ids, n_tokens); + if (!all) return NULL; + + int hidden = ctx->config.hidden_size; + int seq = n_tokens; + + /* Mean pool */ + float *emb = (float *)calloc(hidden, sizeof(float)); + if (!emb) { free(all); return NULL; } for (int i = 0; i < seq; i++) { - const float *row = x + i * hidden; - for (int d = 0; d < hidden; d++) { - embedding[d] += row[d]; - } + const float *row = all + i * hidden; + for (int d = 0; d < hidden; d++) + emb[d] += row[d]; } + free(all); + float inv_seq = 1.0f / (float)seq; - for (int d = 0; d < hidden; d++) embedding[d] *= inv_seq; + for (int d = 0; d < hidden; d++) emb[d] *= inv_seq; - /* ------------------------------------------------------------------ - * Step 5: L2 normalization - * embedding /= ||embedding||_2 - * ------------------------------------------------------------------ */ + /* L2 normalize */ float norm_sq = 0.0f; - for (int d = 0; d < hidden; d++) norm_sq += embedding[d] * embedding[d]; - float norm_val = sqrtf(norm_sq); - if (norm_val > 1e-8f) { - float inv = 1.0f / norm_val; - for (int d = 0; d < hidden; d++) embedding[d] *= inv; + for (int d = 0; d < hidden; d++) norm_sq += emb[d] * emb[d]; + float nv = sqrtf(norm_sq); + if (nv > 1e-8f) { + float inv = 1.0f / nv; + for (int d = 0; d < hidden; d++) emb[d] *= inv; } - return embedding; + return emb; } /* ======================================================================== - * Cosine similarity (dot product of two L2-normalized vectors) + * Cosine similarity * ======================================================================== */ float pplx_cosine_similarity(const float *a, const float *b, int dim) diff --git a/pplx_embed.h b/pplx_embed.h index 69b67c3..67b193e 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -1,23 +1,7 @@ /* - * pplx_embed.h - Pure C inference for perplexity-ai/pplx-embed-v1-0.6b + * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models. * - * Architecture: Qwen3-family transformer with BIDIRECTIONAL (non-causal) - * attention. Based on antirez's qwen-asr kernels. Produces dense float32 - * embedding vectors via mean-pooling + L2-normalization. - * - * Model spec: - * hidden_size = 1024 - * num_hidden_layers = 28 - * num_attention_heads = 16 (query heads) - * num_key_value_heads = 8 (GQA: 2 query heads share 1 KV head) - * head_dim = 128 - * intermediate_size = 3072 - * vocab_size = 151936 - * rms_norm_eps = 1e-6 - * rope_theta = 1000000.0 (1e6) - * weights dtype = F32 (all weights float32) - * tie_word_embeddings = true - * use_bidirectional_attention = true + * Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B). */ #ifndef PPLX_EMBED_H @@ -27,32 +11,26 @@ #include <stdint.h> /* ======================================================================== - * Constants + * Shared constants (identical across all models) * ======================================================================== */ -#define PPLX_HIDDEN_SIZE 1024 -#define PPLX_NUM_LAYERS 28 -#define PPLX_NUM_HEADS 16 -#define PPLX_NUM_KV_HEADS 8 -#define PPLX_HEAD_DIM 128 -#define PPLX_Q_DIM (PPLX_NUM_HEADS * PPLX_HEAD_DIM) /* 2048 */ -#define PPLX_KV_DIM (PPLX_NUM_KV_HEADS * PPLX_HEAD_DIM) /* 1024 */ -#define PPLX_INTERMEDIATE_SIZE 3072 -#define PPLX_VOCAB_SIZE 151936 +#define PPLX_VOCAB_SIZE 151936 +#define PPLX_HEAD_DIM 128 +#define PPLX_MAX_LAYERS 64 /* upper bound for stack arrays */ /* ======================================================================== - * Model Configuration + * Model Configuration (populated from config.json at load time) * ======================================================================== */ typedef struct { - int hidden_size; /* 1024 */ - int n_layers; /* 28 */ - int n_heads; /* 16 */ - int n_kv_heads; /* 8 */ - int head_dim; /* 128 */ - int q_dim; /* n_heads * head_dim = 2048 */ - int kv_dim; /* n_kv_heads * head_dim = 1024 */ - int intermediate_size; /* 3072 */ + int hidden_size; /* 0.6B: 1024 4B: 2560 */ + int n_layers; /* 0.6B: 28 4B: 36 */ + int n_heads; /* 0.6B: 16 4B: 32 */ + int n_kv_heads; /* 0.6B: 8 4B: 8 */ + int head_dim; /* always 128 */ + int q_dim; /* n_heads * head_dim */ + int kv_dim; /* n_kv_heads * head_dim */ + int intermediate_size; /* 0.6B: 3072 4B: 9728 */ int vocab_size; /* 151936 */ float rms_norm_eps; /* 1e-6 */ float rope_theta; /* 1e6 */ @@ -60,37 +38,17 @@ typedef struct { /* ======================================================================== * Per-Layer Weights (all F32, direct mmap pointers - read-only) - * - * Tensor shapes (row-major): - * wq [q_dim, hidden] = [2048, 1024] - * wk [kv_dim, hidden] = [1024, 1024] - * wv [kv_dim, hidden] = [1024, 1024] - * wo [hidden, q_dim] = [1024, 2048] - * q_norm [head_dim] = [128] - * k_norm [head_dim] = [128] - * input_norm [hidden] = [1024] - * post_attn_norm [hidden] = [1024] - * gate_proj [intermediate, hidden] = [3072, 1024] - * up_proj [intermediate, hidden] = [3072, 1024] - * down_proj [hidden, intermediate] = [1024, 3072] * ======================================================================== */ typedef struct { - /* Self-attention projections (no bias) */ const float *wq; /* [q_dim, hidden] */ const float *wk; /* [kv_dim, hidden] */ const float *wv; /* [kv_dim, hidden] */ const float *wo; /* [hidden, q_dim] */ - - /* Per-head Q/K RMSNorm weights (Qwen3 feature) */ const float *q_norm; /* [head_dim] */ const float *k_norm; /* [head_dim] */ - - /* Layer norms (no bias - RMSNorm only) */ - const float *input_norm; /* [hidden] pre-attention */ - const float *post_attn_norm;/* [hidden] pre-MLP */ - - /* SwiGLU MLP (no bias) */ + const float *input_norm; /* [hidden] */ + const float *post_attn_norm;/* [hidden] */ const float *gate_proj; /* [intermediate, hidden] */ const float *up_proj; /* [intermediate, hidden] */ const float *down_proj; /* [hidden, intermediate] */ @@ -101,14 +59,9 @@ typedef struct { * ======================================================================== */ typedef struct { - /* Token embedding table (tied with output head, but we don't need lm_head) */ - const float *embed_tokens; /* [vocab_size, hidden] */ - - /* Transformer layers */ - pplx_layer_t layers[PPLX_NUM_LAYERS]; - - /* Final RMSNorm */ - const float *norm; /* [hidden] */ + const float *embed_tokens; /* [vocab_size, hidden] */ + pplx_layer_t *layers; /* [n_layers] heap-allocated */ + const float *norm; /* [hidden] */ } pplx_weights_t; /* ======================================================================== @@ -118,27 +71,25 @@ typedef struct { typedef struct { pplx_config_t config; pplx_weights_t weights; - - /* Open safetensors handle (keeps mmap alive) */ - void *safetensors; /* multi_safetensors_t* */ + void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ /* Working buffers, grown lazily to buf_seq_cap */ int buf_seq_cap; - float *x; /* [seq, hidden] main activations */ - float *x_norm; /* [seq, hidden] after RMSNorm */ - float *q; /* [seq, q_dim] query vectors */ - float *k; /* [seq, kv_dim] key vectors */ - float *v; /* [seq, kv_dim] value vectors */ - float *attn_out; /* [seq, q_dim] attention output */ - float *proj_out; /* [seq, hidden] after o_proj */ - float *ffn_gate; /* [seq, intermediate] gate branch */ - float *ffn_up; /* [seq, intermediate] up branch */ - float *ffn_out; /* [seq, hidden] after down_proj */ + float *x; /* [seq, hidden] */ + float *x_norm; /* [seq, hidden] */ + float *q; /* [seq, q_dim] */ + float *k; /* [seq, kv_dim] */ + float *v; /* [seq, kv_dim] */ + float *attn_out; /* [seq, q_dim] */ + float *proj_out; /* [seq, hidden] */ + float *ffn_gate; /* [seq, intermediate] */ + float *ffn_up; /* [seq, intermediate] */ + float *ffn_out; /* [seq, hidden] */ /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily */ float *rope_cos; float *rope_sin; - int rope_cache_cap; /* number of cached positions */ + int rope_cache_cap; } pplx_ctx_t; /* ======================================================================== @@ -146,11 +97,12 @@ typedef struct { * ======================================================================== */ /* - * Load the model from a directory containing: - * model.safetensors - weight tensors (F32) - * vocab.json - BPE vocabulary - * merges.txt - BPE merge rules + * Load model from directory containing: + * config.json - model hyperparameters + * model.safetensors (or sharded) - F32 weights + * vocab.json + merges.txt - BPE tokenizer * + * Reads config.json to determine model size (0.6B vs 4B). * Returns NULL on error. */ pplx_ctx_t *pplx_load(const char *model_dir); @@ -159,28 +111,35 @@ pplx_ctx_t *pplx_load(const char *model_dir); void pplx_free(pplx_ctx_t *ctx); /* - * Compute the embedding for a sequence of token IDs. + * Compute embedding for a token sequence. * - * Performs: * 1. Token embedding lookup - * 2. 28 × bidirectional transformer layers (GQA, RoPE, SwiGLU) + * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU) * 3. Final RMSNorm - * 4. Mean pooling across all token positions + * 4. Mean pooling over all positions * 5. L2 normalization * - * Returns a malloc'd float[hidden_size] (caller must free). - * Returns NULL on allocation error or empty input. + * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ float *pplx_embed(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); /* - * Cosine similarity between two L2-normalized embedding vectors. - * (This is just the dot product since both are unit vectors.) + * Run the transformer forward pass WITHOUT pooling. + * Returns the full [n_tokens * hidden_size] output after final RMSNorm. + * Caller frees the returned buffer. NULL on error. + * + * Useful for contextual (late-chunking) models where you need + * per-token embeddings before splitting by separator positions. + */ +float *pplx_forward(pplx_ctx_t *ctx, const int *token_ids, int n_tokens); + +/* + * Cosine similarity between two L2-normalized vectors. */ float pplx_cosine_similarity(const float *a, const float *b, int dim); /* Verbose level: 0=quiet, 1=info, 2=debug */ extern int pplx_verbose; -extern int qwen_verbose; /* same flag, used internally by kernel/tokenizer */ +extern int qwen_verbose; #endif /* PPLX_EMBED_H */ diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index d6637cd..3c82db1 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -1,21 +1,6 @@ -/* - * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference - * - * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference. - * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm. - * - * Key operations used: - * mlx_take - embedding lookup - * mlx_matmul - linear projections (no bias) - * mlx_fast_rms_norm - RMSNorm (fused Metal kernel) - * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel) - * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel) - * sigmoid * gate, multiply - SwiGLU activation - */ - #include "pplx_embed_mlx.h" -#include "pplx_embed.h" /* config constants */ -#include "qwen_asr_safetensors.h" /* safetensors reader */ +#include "pplx_embed.h" +#include "qwen_asr_safetensors.h" #include <mlx/c/mlx.h> @@ -25,30 +10,22 @@ #include <math.h> /* ======================================================================== - * Per-layer weight arrays (MLX) + * Per-layer MLX weight arrays * ======================================================================== */ typedef struct { - mlx_array wq; /* [q_dim, hidden] */ - mlx_array wk; /* [kv_dim, hidden] */ - mlx_array wv; /* [kv_dim, hidden] */ - mlx_array wo; /* [hidden, q_dim] */ - mlx_array q_norm; /* [head_dim] */ - mlx_array k_norm; /* [head_dim] */ - mlx_array input_norm; /* [hidden] */ - mlx_array post_attn_norm; /* [hidden] */ - mlx_array gate_proj; /* [intermediate, hidden] */ - mlx_array up_proj; /* [intermediate, hidden] */ - mlx_array down_proj; /* [hidden, intermediate] */ + mlx_array wq, wk, wv, wo; + mlx_array q_norm, k_norm; + mlx_array input_norm, post_attn_norm; + mlx_array gate_proj, up_proj, down_proj; } mlx_layer_t; struct pplx_mlx_ctx { - mlx_array embed_tokens; /* [vocab, hidden] */ - mlx_layer_t layers[PPLX_NUM_LAYERS]; - mlx_array norm; /* [hidden] */ + pplx_config_t config; + mlx_array embed_tokens; + mlx_layer_t *layers; /* heap-allocated [n_layers] */ + mlx_array norm; mlx_stream stream; - - /* Safetensors kept alive for mmap */ multi_safetensors_t *ms; }; @@ -56,13 +33,8 @@ struct pplx_mlx_ctx { * Helpers * ======================================================================== */ -/* Default GPU stream */ -static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; } +static int arr_ok(mlx_array a) { return a.ctx != NULL; } -/* - * Load an F32 tensor from safetensors into an mlx_array. - * The data is copied into MLX (which will move it to GPU as needed). - */ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { @@ -76,82 +48,156 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, fprintf(stderr, "mlx: expected F32 for %s\n", name); return (mlx_array){0}; } - const void *data = safetensors_data(sf, t); - return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32); + return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32); } -/* Quick check: is arr valid? */ -static int arr_ok(mlx_array a) { return a.ctx != NULL; } +/* y = x @ W^T */ +static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s) +{ + mlx_array Wt = mlx_array_new(); + mlx_transpose(&Wt, W, s); + mlx_array res = mlx_array_new(); + mlx_matmul(&res, x, Wt, s); + mlx_array_free(Wt); + return res; +} + +const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx) +{ + return ctx ? &ctx->config : NULL; +} /* ======================================================================== - * Load + * Load (reuses the same parse_config logic via pplx_load then copies) * ======================================================================== */ +/* We duplicate a minimal config parser here so MLX can load standalone. + * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ + +static const char *skip_ws_m(const char *p) +{ + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + return p; +} + +static const char *json_find_m(const char *json, const char *key) +{ + size_t klen = strlen(key); + const char *p = json; + while ((p = strstr(p, key)) != NULL) { + if (p > json && *(p - 1) == '"') { + const char *a = p + klen; + if (*a == '"') { + a = skip_ws_m(a + 1); + if (*a == ':') return skip_ws_m(a + 1); + } + } + p += klen; + } + return NULL; +} + +static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) +{ + char path[1024]; + snprintf(path, sizeof(path), "%s/config.json", model_dir); + FILE *f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, 0, SEEK_SET); + char *buf = (char *)malloc(sz + 1); + if (!buf || fread(buf, 1, sz, f) != (size_t)sz) { fclose(f); free(buf); return -1; } + fclose(f); + buf[sz] = '\0'; + +#define GI(k, fb) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : 0; } while(0) +#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) + GI("hidden_size", hidden_size); + GI("num_hidden_layers", n_layers); + GI("num_attention_heads", n_heads); + GI("num_key_value_heads", n_kv_heads); + GI("intermediate_size", intermediate_size); + cfg->head_dim = PPLX_HEAD_DIM; + cfg->vocab_size = PPLX_VOCAB_SIZE; + GF("rms_norm_eps", rms_norm_eps, 1e-6f); + GF("rope_theta", rope_theta, 1000000.0f); +#undef GI +#undef GF + + cfg->q_dim = cfg->n_heads * cfg->head_dim; + cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; + free(buf); + + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0) { + fprintf(stderr, "mlx: bad config (hidden=%d layers=%d)\n", + cfg->hidden_size, cfg->n_layers); + return -1; + } + return 0; +} + pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) { pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); if (!ctx) return NULL; + if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } + + const pplx_config_t *c = &ctx->config; ctx->stream = mlx_default_gpu_stream_new(); - /* Open safetensors */ ctx->ms = multi_safetensors_open(model_dir); if (!ctx->ms) { fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); - free(ctx); - return NULL; + free(ctx); return NULL; } - multi_safetensors_t *ms = ctx->ms; + int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; + int inter = c->intermediate_size, hd = c->head_dim; - /* Embedding table: [vocab_size, hidden] */ - int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE}; - ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2); + /* Embedding table */ + int emb_shape[] = {c->vocab_size, h}; + ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2); if (!arr_ok(ctx->embed_tokens)) goto fail; - /* Per-layer weights */ + /* Layers */ + ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); + if (!ctx->layers) goto fail; + char name[256]; - for (int i = 0; i < PPLX_NUM_LAYERS; i++) { + for (int i = 0; i < c->n_layers; i++) { mlx_layer_t *l = &ctx->layers[i]; - int hidden = PPLX_HIDDEN_SIZE; - int q_dim = PPLX_Q_DIM; - int kv_dim = PPLX_KV_DIM; - int inter = PPLX_INTERMEDIATE_SIZE; - int hdim = PPLX_HEAD_DIM; -#define LOAD(field, fmt, ...) do { \ +#define LD(fld, fmt, ...) do { \ snprintf(name, sizeof(name), fmt, i); \ int sh[] = {__VA_ARGS__}; \ - int nd = sizeof(sh) / sizeof(sh[0]); \ - l->field = load_tensor(ms, name, sh, nd); \ - if (!arr_ok(l->field)) goto fail; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!arr_ok(l->fld)) goto fail; \ } while (0) - LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden); - LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden); - LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden); - LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim); - LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim); - LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim); - LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden); - LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden); - LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden); - LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden); - LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter); + LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); + LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); + LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); + LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); + LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); + LD(input_norm, "layers.%d.input_layernorm.weight", h); + LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); + LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); + LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); + LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); +#undef LD -#undef LOAD - - if (pplx_verbose >= 2) - fprintf(stderr, " mlx layer %d loaded\n", i); + if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); } - /* Final norm */ - int norm_shape[] = {PPLX_HIDDEN_SIZE}; - ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1); + int ns[] = {h}; + ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); if (!arr_ok(ctx->norm)) goto fail; if (pplx_verbose >= 1) - fprintf(stderr, "mlx: all weights loaded\n"); + fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h); return ctx; @@ -164,18 +210,13 @@ fail: * Free * ======================================================================== */ -static void free_layer(mlx_layer_t *l) +static void free_mlx_layer(mlx_layer_t *l) { - mlx_array_free(l->wq); - mlx_array_free(l->wk); - mlx_array_free(l->wv); - mlx_array_free(l->wo); - mlx_array_free(l->q_norm); - mlx_array_free(l->k_norm); - mlx_array_free(l->input_norm); - mlx_array_free(l->post_attn_norm); - mlx_array_free(l->gate_proj); - mlx_array_free(l->up_proj); + mlx_array_free(l->wq); mlx_array_free(l->wk); + mlx_array_free(l->wv); mlx_array_free(l->wo); + mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); + mlx_array_free(l->gate_proj); mlx_array_free(l->up_proj); mlx_array_free(l->down_proj); } @@ -183,8 +224,11 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx) { if (!ctx) return; mlx_array_free(ctx->embed_tokens); - for (int i = 0; i < PPLX_NUM_LAYERS; i++) - free_layer(&ctx->layers[i]); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_mlx_layer(&ctx->layers[i]); + free(ctx->layers); + } mlx_array_free(ctx->norm); mlx_stream_free(ctx->stream); if (ctx->ms) multi_safetensors_close(ctx->ms); @@ -192,274 +236,168 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx) } /* ======================================================================== - * Forward pass (all on GPU stream, lazy-evaluated) - * - * Notation: S = gpu_stream(ctx) - * - * Linear without bias: y = x @ W^T - * Since mlx_matmul(x, W^T) requires transposing W, and our weights - * are stored as [out, in], we do: y = x @ W^T via - * Wt = transpose(W) [in, out] - * y = matmul(x, Wt) [seq, out] - * - * Alternatively: y = matmul(x, transpose(W)) - * All MLX ops are lazy (graph-built) so transpose is free. + * Forward pass * ======================================================================== */ -/* Convenience: transpose a 2D matrix */ -static mlx_array t2d(mlx_array a, mlx_stream s) -{ - mlx_array res = mlx_array_new(); - mlx_transpose(&res, a, s); - return res; -} - -/* y = x @ W^T (linear, no bias) */ -static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s) -{ - mlx_array Wt = t2d(W, s); - mlx_array res = mlx_array_new(); - mlx_matmul(&res, x, Wt, s); - mlx_array_free(Wt); - return res; -} - float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; - mlx_stream S = gpu_stream(ctx); - int hidden = PPLX_HIDDEN_SIZE; - int n_heads = PPLX_NUM_HEADS; - int n_kv_heads = PPLX_NUM_KV_HEADS; - int head_dim = PPLX_HEAD_DIM; - int q_dim = PPLX_Q_DIM; - int seq = n_tokens; - float scale = 1.0f / sqrtf((float)head_dim); + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; - (void)n_heads; (void)n_kv_heads; /* used only in shape literals */ + int hidden = c->hidden_size; + int n_heads = c->n_heads; + int n_kv_heads = c->n_kv_heads; + int head_dim = c->head_dim; + int q_dim = c->q_dim; + int seq = n_tokens; + float scale = 1.0f / sqrtf((float)head_dim); - /* ------------------------------------------------------------------ - * 1. Token embedding lookup: x = embed_tokens[token_ids] - * ------------------------------------------------------------------ */ + /* 1. Embedding lookup */ int ids_shape[] = {seq}; mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32); mlx_array x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); mlx_array_free(ids); - /* x: [seq, hidden] */ - /* ------------------------------------------------------------------ - * 2. Transformer layers (28x) - * ------------------------------------------------------------------ */ - for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) { + /* 2. Transformer layers */ + for (int layer = 0; layer < c->n_layers; layer++) { mlx_layer_t *l = &ctx->layers[layer]; - /* ---- 2a. Input RMSNorm ---- */ + /* Input RMSNorm */ mlx_array xn = mlx_array_new(); - mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S); + mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); - /* ---- 2b. QKV projections (no bias) ---- - * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim] - */ + /* QKV projections */ mlx_array q_flat = linear(xn, l->wq, S); mlx_array k_flat = linear(xn, l->wk, S); mlx_array v_flat = linear(xn, l->wv, S); mlx_array_free(xn); - /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */ + /* Reshape to [seq, n_heads, head_dim] */ int q_shape[] = {seq, n_heads, head_dim}; int k_shape[] = {seq, n_kv_heads, head_dim}; - mlx_array q = mlx_array_new(); mlx_array k = mlx_array_new(); mlx_array v = mlx_array_new(); mlx_reshape(&q, q_flat, q_shape, 3, S); mlx_reshape(&k, k_flat, k_shape, 3, S); mlx_reshape(&v, v_flat, k_shape, 3, S); - mlx_array_free(q_flat); - mlx_array_free(k_flat); - mlx_array_free(v_flat); + mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); - /* ---- 2d. Per-head Q/K RMSNorm ---- - * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim) - */ + /* Per-head Q/K RMSNorm */ mlx_array qn = mlx_array_new(); mlx_array kn = mlx_array_new(); - mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S); - mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S); - mlx_array_free(q); - mlx_array_free(k); + mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); + mlx_array_free(q); mlx_array_free(k); - /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ---- - * MLX rope expects (..., T, D) where T is second-to-last. - * [seq, n_heads, head_dim] - * - expand_dims(0) [1, seq, n_heads, head_dim] - * - transpose [0,2,1,3] [1, n_heads, seq, head_dim] - */ + /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */ int perm[] = {0, 2, 1, 3}; - - mlx_array qe = mlx_array_new(); - mlx_array ke = mlx_array_new(); - mlx_array ve = mlx_array_new(); + mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new(); mlx_expand_dims(&qe, qn, 0, S); mlx_expand_dims(&ke, kn, 0, S); mlx_expand_dims(&ve, v, 0, S); - mlx_array_free(qn); - mlx_array_free(kn); + mlx_array_free(qn); mlx_array_free(kn); - mlx_array qt = mlx_array_new(); - mlx_array kt = mlx_array_new(); - mlx_array vt = mlx_array_new(); + mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); mlx_transpose_axes(&qt, qe, perm, 4, S); mlx_transpose_axes(&kt, ke, perm, 4, S); mlx_transpose_axes(&vt, ve, perm, 4, S); - mlx_array_free(qe); - mlx_array_free(ke); - mlx_array_free(ve); + mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve); mlx_array_free(v); - /* ---- 2f. RoPE (NeoX-style: traditional=false) ---- - * qt: [1, n_heads, seq, head_dim] - * kt: [1, n_kv_heads, seq, head_dim] - * MLX rope: T = second-to-last dim = seq - */ - mlx_optional_float base = {.value = 1000000.0f, .has_value = true}; - mlx_array null_freqs = (mlx_array){0}; + /* RoPE (NeoX: traditional=false) */ + mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_array qr = mlx_array_new(), kr = mlx_array_new(); + mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_array_free(qt); mlx_array_free(kt); - mlx_array qr = mlx_array_new(); - mlx_array kr = mlx_array_new(); - mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false, - base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S); - mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false, - base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S); - mlx_array_free(qt); - mlx_array_free(kt); - - /* ---- 2g. Bidirectional GQA Attention ---- - * qr: [1, n_heads, seq, head_dim] - * kr: [1, n_kv_heads, seq, head_dim] - * vt: [1, n_kv_heads, seq, head_dim] - * mask_mode="" > no causal mask > full bidirectional - * GQA: SDPA broadcasts KV heads to match Q heads automatically. - */ + /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */ mlx_array attn = mlx_array_new(); - mlx_array null_mask = (mlx_array){0}; - mlx_array null_sinks = (mlx_array){0}; mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, - /*mask_mode=*/"", - null_mask, null_sinks, S); - mlx_array_free(qr); - mlx_array_free(kr); - mlx_array_free(vt); - /* attn: [1, n_heads, seq, head_dim] */ + &attn, qr, kr, vt, scale, "", null_arr, null_arr, S); + mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); - /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */ + /* Transpose back and reshape to [seq, q_dim] */ mlx_array attn_t = mlx_array_new(); - mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */ + mlx_transpose_axes(&attn_t, attn, perm, 4, S); mlx_array_free(attn); mlx_array attn_sq = mlx_array_new(); - mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */ + mlx_squeeze_axis(&attn_sq, attn_t, 0, S); mlx_array_free(attn_t); - int attn_flat_shape[] = {seq, q_dim}; + int flat_shape[] = {seq, q_dim}; mlx_array attn_flat = mlx_array_new(); - mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S); + mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S); mlx_array_free(attn_sq); - /* ---- 2i. Output projection + residual ---- */ + /* Output projection + residual */ mlx_array proj = linear(attn_flat, l->wo, S); mlx_array_free(attn_flat); - mlx_array x2 = mlx_array_new(); mlx_add(&x2, x, proj, S); - mlx_array_free(x); - mlx_array_free(proj); + mlx_array_free(x); mlx_array_free(proj); x = x2; - /* ---- 2j. Post-attention RMSNorm ---- */ + /* Post-attention RMSNorm */ mlx_array xn2 = mlx_array_new(); - mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S); + mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); - /* ---- 2k. SwiGLU MLP ---- - * gate = x_norm @ gate_proj^T [seq, inter] - * up = x_norm @ up_proj^T [seq, inter] - * mid = silu(gate) * up = (gate * sigmoid(gate)) * up - * out = mid @ down_proj^T [seq, hidden] - */ + /* SwiGLU MLP */ mlx_array gate = linear(xn2, l->gate_proj, S); mlx_array up = linear(xn2, l->up_proj, S); mlx_array_free(xn2); - /* SiLU(gate) = gate * sigmoid(gate) */ mlx_array gate_sig = mlx_array_new(); mlx_sigmoid(&gate_sig, gate, S); mlx_array silu_gate = mlx_array_new(); mlx_multiply(&silu_gate, gate, gate_sig, S); - mlx_array_free(gate); - mlx_array_free(gate_sig); + mlx_array_free(gate); mlx_array_free(gate_sig); - /* mid = silu_gate * up */ mlx_array mid = mlx_array_new(); mlx_multiply(&mid, silu_gate, up, S); - mlx_array_free(silu_gate); - mlx_array_free(up); + mlx_array_free(silu_gate); mlx_array_free(up); - /* down projection */ mlx_array ffn = linear(mid, l->down_proj, S); mlx_array_free(mid); - /* Residual */ mlx_array x3 = mlx_array_new(); mlx_add(&x3, x, ffn, S); - mlx_array_free(x); - mlx_array_free(ffn); + mlx_array_free(x); mlx_array_free(ffn); x = x3; } - /* ------------------------------------------------------------------ - * 3. Final RMSNorm - * ------------------------------------------------------------------ */ + /* 3. Final RMSNorm */ mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); mlx_array_free(x); - /* ------------------------------------------------------------------ - * 4. Mean pooling: mean(x, axis=0) [hidden] - * ------------------------------------------------------------------ */ + /* 4. Mean pool over seq dim */ mlx_array emb = mlx_array_new(); - mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S); + mlx_mean_axis(&emb, x_normed, 0, false, S); mlx_array_free(x_normed); - /* ------------------------------------------------------------------ - * 5. L2 normalization: emb / ||emb||_2 - * ------------------------------------------------------------------ */ + /* 5. L2 normalize */ int norm_axes[] = {0}; - mlx_array norm_val = mlx_array_new(); - mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S); - - mlx_array emb_normed = mlx_array_new(); - mlx_divide(&emb_normed, emb, norm_val, S); - mlx_array_free(emb); - mlx_array_free(norm_val); + mlx_array nv = mlx_array_new(); + mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S); + mlx_array emb_n = mlx_array_new(); + mlx_divide(&emb_n, emb, nv, S); + mlx_array_free(emb); mlx_array_free(nv); - /* ------------------------------------------------------------------ - * 6. Evaluate and copy result to CPU - * ------------------------------------------------------------------ */ - mlx_array_eval(emb_normed); - - const float *data = mlx_array_data_float32(emb_normed); - if (!data) { - fprintf(stderr, "mlx: failed to get embedding data\n"); - mlx_array_free(emb_normed); - return NULL; + /* 6. Eval and copy to CPU */ + mlx_array_eval(emb_n); + const float *data = mlx_array_data_float32(emb_n); + float *out = NULL; + if (data) { + out = (float *)malloc(hidden * sizeof(float)); + if (out) memcpy(out, data, hidden * sizeof(float)); } - - float *out = (float *)malloc(hidden * sizeof(float)); - if (out) memcpy(out, data, hidden * sizeof(float)); - - mlx_array_free(emb_normed); + mlx_array_free(emb_n); return out; } diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index 5327fd0..a1544ef 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -1,9 +1,7 @@ /* * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference * - * Uses Apple's mlx-c to run the transformer on Metal GPU. - * All operations run on the GPU default stream; the result is copied - * back to CPU as a float* embedding. + * Uses Apple's mlx-c to run the transformer on Metal. */ #ifndef PPLX_EMBED_MLX_H @@ -11,25 +9,26 @@ #include <stddef.h> #include <stdint.h> +#include "pplx_embed.h" /* Opaque MLX model context */ typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; /* - * Load model into MLX arrays from safetensors in model_dir. + * Load model into MLX arrays from safetensors + config.json. * Returns NULL on error. */ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); -/* - * Free all MLX resources. - */ void pplx_mlx_free(pplx_mlx_ctx_t *ctx); /* * Compute embedding for token_ids[0..n_tokens-1]. - * Returns malloc'd float[1024] (caller frees). NULL on error. + * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); +/* Get the config (so main.c can read hidden_size etc.) */ +const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); + #endif /* PPLX_EMBED_MLX_H */ diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index c0d9d70..0ea3cc6 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -1,8 +1,3 @@ -/* - * qwen_asr_kernels.c - Math kernels for Qwen3-ASR inference - * Adapted from voxtral-realtime project. - */ - #include "qwen_asr_kernels.h" #include "qwen_asr_kernels_impl.h" #include <math.h> diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c index 0bdf272..7afe899 100644 --- a/qwen_asr_safetensors.c +++ b/qwen_asr_safetensors.c @@ -1,8 +1,3 @@ -/* - * qwen_asr_safetensors.c - Safetensors reader with multi-shard support - * Adapted from voxtral-realtime project. - */ - #include "qwen_asr_safetensors.h" #include <stdio.h> #include <stdlib.h> diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c index 7bbc5db..0e6df38 100644 --- a/qwen_asr_tokenizer.c +++ b/qwen_asr_tokenizer.c @@ -1,11 +1,3 @@ -/* - * qwen_asr_tokenizer.c - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) - * - * Supports: - * - decode token IDs to UTF-8 text (for ASR output assembly) - * - encode UTF-8 text to token IDs using vocab.json + merges.txt - */ - #include "qwen_asr_tokenizer.h" #include "qwen_asr_kernels.h" #include <limits.h> |
