aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 08:24:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit2c37727e653d2340d8977dd5db4addc767dd81ea (patch)
tree578ad6affac511f847a346a0efc6d40ebf6a279f /main.c
parentec417bcb52f2aa8e24aeec312e8ddbf3a0798056 (diff)
downloadffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.tar.gz
ffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.zip
gitignore: track new binary and library names
Diffstat (limited to 'main.c')
-rw-r--r--main.c825
1 files changed, 0 insertions, 825 deletions
diff --git a/main.c b/main.c
deleted file mode 100644
index 140494b..0000000
--- a/main.c
+++ /dev/null
@@ -1,825 +0,0 @@
-/* main.c - pplx_embed command-line tool */
-
-#include "pplx_embed.h"
-#include "pplx_distributed.h"
-#include "qwen_kernels.h"
-#include "qwen_tokenizer.h"
-
-#ifdef USE_MLX
-#include "pplx_embed_mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "pplx_embed_cuda.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <time.h>
-
-static double now_ms(void)
-{
- struct timespec ts;
- clock_gettime(CLOCK_MONOTONIC, &ts);
- return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
-}
-
-static void print_usage(const char *prog)
-{
- fprintf(stderr,
- "Usage: %s -d <model_dir> [options] [text...]\n"
- "\n"
- "Options:\n"
- " -d <dir> Model directory (required)\n"
- " --mlx Use Apple MLX GPU backend\n"
- " --cuda Use CUDA/cuBLAS GPU backend\n"
- " --stream Read lines from stdin, write one JSON embedding per line\n"
- " --dist-worker\n"
- " Serve one final distributed layer shard\n"
- " --dist-remote HOST:PORT\n"
- " Use one remote final shard for CLI or --stream inference\n"
- " --layers A:B Distributed half-open layer range, for example 0:14\n"
- " --dist-activation-bits N\n"
- " Distributed activation transport width: 32 or 16 (default: 32)\n"
- " --mlx-quant-bits N\n"
- " Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quant-group-size N\n"
- " MLX quantization group size (default: 64)\n"
- " --cuda-gemm-mode MODE\n"
- " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weight-dtype DTYPE\n"
- " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
- " weight memory and uses BF16 tensor cores\n"
- " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n"
- " --port N Bind port for --dist-worker (default: 8000)\n"
- " -b, --batch-size N\n"
- " Max texts per engine batch (default: all; --stream: 1)\n"
- " -t, --threads N\n"
- " CPU threads (default: all cores)\n"
- " -e, --embeddings\n"
- " Print raw embeddings (with multiple texts)\n"
- " -v, --verbose\n"
- " Verbose (-vv for debug)\n"
- " -h Show this help\n"
- "\n"
- "Modes:\n"
- " 1 text arg Print embedding as space-separated floats\n"
- " 2+ text args Print cosine similarity matrix\n"
- " no args Batch: read all stdin lines, then similarity matrix\n"
- " --stream Streaming: read stdin lines, write JSON per line\n"
- " --dist-worker Final layer-shard worker\n"
- "\n"
- "The HTTP API is served by the separate pplx-embed-server binary.\n"
- "\n"
- "Examples:\n"
- " %s -d ./model \"query: what is AI?\"\n"
- " %s -d ./model --mlx --stream < texts.txt\n"
- " %s -d ./model --dist-worker --layers 14:28 --port 9000\n"
- " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n",
- prog, prog, prog, prog, prog);
-}
-
-/* ========================================================================
- * Embed one text: tokenize > forward > return float[dim]
- * ======================================================================== */
-
-typedef struct {
- pplx_model_t *model;
- pplx_workspace_t *workspace;
- pplx_dist_client_t *dist;
-#ifdef USE_MLX
- pplx_mlx_ctx_t *mlx_ctx;
-#endif
-#ifdef USE_CUDA
- pplx_cuda_ctx_t *cuda_ctx;
-#endif
- qwen_tokenizer_t *tok;
- qwen_tokenizer_workspace_t *tok_ws;
- int dim;
-} engine_t;
-
-typedef struct {
- int *ids;
- int n_tokens;
-} token_buf_t;
-
-static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
- int batch, float *out_embeddings)
-{
- if (e->dist)
- return pplx_dist_client_embed_batch(e->dist, inputs, batch,
- out_embeddings);
-#ifdef USE_MLX
- if (e->mlx_ctx)
- return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings);
-#endif
-#ifdef USE_CUDA
- if (e->cuda_ctx)
- return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings);
-#endif
- return pplx_model_embed_batch(e->model, e->workspace,
- inputs, batch, out_embeddings);
-}
-
-static int parse_layers(const char *arg, int *start, int *end)
-{
- int used = 0;
- if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 ||
- arg[used] != '\0' || *start < 0 || *start >= *end)
- return -1;
- return 0;
-}
-
-static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port)
-{
- const char *colon = arg ? strrchr(arg, ':') : NULL;
- if (!colon || colon == arg || !colon[1]) return -1;
- size_t n = (size_t)(colon - arg);
- if (n >= hostlen) return -1;
- memcpy(host, arg, n);
- host[n] = '\0';
- int used = 0;
- if (sscanf(colon + 1, "%d%n", port, &used) != 1 ||
- colon[1 + used] != '\0' || *port <= 0 || *port > 65535)
- return -1;
- return 0;
-}
-
-static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
-{
- memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
- text, &out->n_tokens);
- if (!out->ids || out->n_tokens == 0) {
- fprintf(stderr, "tokenization failed: %s\n", text);
- free(out->ids);
- out->ids = NULL;
- out->n_tokens = 0;
- return -1;
- }
-
- if (pplx_verbose >= 1) {
- fprintf(stderr, "tokens (%d): ", out->n_tokens);
- for (int i = 0; i < out->n_tokens && i < 20; i++)
- fprintf(stderr, "%d ", out->ids[i]);
- if (out->n_tokens > 20) fprintf(stderr, "...");
- fprintf(stderr, "\n");
- }
- return 0;
-}
-
-static void free_tokens(token_buf_t *tokens, int n)
-{
- if (!tokens) return;
- for (int i = 0; i < n; i++)
- free(tokens[i].ids);
-}
-
-/* ========================================================================
- * Output helpers
- * ======================================================================== */
-
-static void print_embedding_raw(const float *emb, int dim)
-{
- for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(' ');
- printf("%.8f", (double)emb[i]);
- }
- putchar('\n');
-}
-
-static size_t engine_workspace_nbytes(const engine_t *e)
-{
- return (e && e->workspace) ? pplx_workspace_nbytes(e->workspace) : 0;
-}
-
-static void print_embedding_json(const float *emb, int dim, int n_tokens,
- double ms, size_t workspace_bytes)
-{
- printf("{\"embedding\":[");
- for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(',');
- printf("%.8f", (double)emb[i]);
- }
- printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n",
- dim, n_tokens, ms, workspace_bytes);
- fflush(stdout);
-}
-
-/* ========================================================================
- * Stdin mode: read lines from stdin, write JSON to stdout
- * ======================================================================== */
-
-static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
-{
- if (n_lines <= 0) return 0;
-
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
- pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t));
- int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
- float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
- if (!tokens || !inputs || !input_to_line || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(input_to_line); free(embs);
- for (int i = 0; i < n_lines; i++)
- printf("{\"error\":\"embedding failed\"}\n");
- fflush(stdout);
- return 1;
- }
-
- int n_inputs = 0;
- for (int i = 0; i < n_lines; i++) {
- if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
- inputs[n_inputs].ids = tokens[i].ids;
- inputs[n_inputs].n_tokens = tokens[i].n_tokens;
- input_to_line[n_inputs] = i;
- n_inputs++;
- }
- }
-
- double ms = 0.0;
- int rc = 0;
- if (n_inputs > 0) {
- double t0 = now_ms();
- rc = engine_embed_batch(e, inputs, n_inputs, embs);
- ms = now_ms() - t0;
- if (pplx_verbose >= 1)
- fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
- }
-
- int next_valid = 0;
- for (int i = 0; i < n_lines; i++) {
- if (!tokens[i].ids) {
- printf("{\"error\":\"tokenization failed\"}\n");
- continue;
- }
- if (rc != 0) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- while (next_valid < n_inputs && input_to_line[next_valid] != i)
- next_valid++;
- if (next_valid >= n_inputs) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- print_embedding_json(embs + (size_t)next_valid * e->dim,
- e->dim, tokens[i].n_tokens, ms,
- engine_workspace_nbytes(e));
- next_valid++;
- }
- fflush(stdout);
-
- free_tokens(tokens, n_lines);
- free(tokens); free(inputs); free(input_to_line); free(embs);
- return rc == 0 ? 0 : 1;
-}
-
-static int run_stdin(engine_t *e, int batch_size)
-{
- char line[65536];
- if (batch_size <= 0) batch_size = 1;
-
- if (pplx_verbose >= 1)
- fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n",
- batch_size);
-
- char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
- if (!batch_lines) {
- fprintf(stderr, "OOM\n");
- return 1;
- }
- int n_batch = 0;
- int rc = 0;
-
- while (fgets(line, sizeof(line), stdin)) {
- /* Strip newline */
- size_t len = strlen(line);
- while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r'))
- line[--len] = '\0';
- if (len == 0) continue;
-
- if (pplx_verbose >= 1)
- fprintf(stderr, "stdin: \"%.*s%s\"\n",
- (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
-
- batch_lines[n_batch] = strdup(line);
- if (!batch_lines[n_batch]) {
- fprintf(stderr, "OOM\n");
- rc = 1;
- break;
- }
- n_batch++;
-
- if (n_batch == batch_size) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++) {
- free(batch_lines[i]);
- batch_lines[i] = NULL;
- }
- n_batch = 0;
- }
- }
-
- if (n_batch > 0) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++)
- free(batch_lines[i]);
- }
- free(batch_lines);
-
- if (pplx_verbose >= 1)
- fprintf(stderr, "stdin: EOF\n");
-
- return rc;
-}
-
-/* ========================================================================
- * Batch mode: embed args or stdin lines, then print similarity or vectors
- * ======================================================================== */
-
-static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
-{
- if (*n_texts == *cap) {
- int new_cap = *cap ? *cap * 2 : 8;
- char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
- if (!new_texts) return -1;
- *texts = new_texts;
- *cap = new_cap;
- }
-
- (*texts)[*n_texts] = strdup(s);
- if (!(*texts)[*n_texts]) return -1;
- (*n_texts)++;
- return 0;
-}
-
-static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
- int print_embs, int batch_size)
-{
- int n_texts = 0, cap = 0;
- char **texts = NULL;
- int rc = 0;
-
- if (arg_start < argc) {
- for (int i = arg_start; i < argc; i++) {
- if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
- goto oom_texts;
- }
- } else {
- char line[65536];
- while (fgets(line, sizeof(line), stdin)) {
- size_t l = strlen(line);
- while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
- line[--l] = '\0';
- if (l == 0) continue;
- if (append_text(&texts, &n_texts, &cap, line) != 0)
- goto oom_texts;
- }
- }
-
- if (n_texts == 0) {
- fprintf(stderr, "no input texts\n");
- free(texts);
- return 1;
- }
-
- int dim = e->dim;
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
- pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t));
- float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
- if (!tokens || !inputs || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(embs);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
- free(texts);
- return 1;
- }
-
- for (int i = 0; i < n_texts; i++) {
- if (pplx_verbose >= 1)
- fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
- if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
- rc = 1;
- goto done;
- }
- inputs[i].ids = tokens[i].ids;
- inputs[i].n_tokens = tokens[i].n_tokens;
- }
-
- int max_batch = batch_size > 0 ? batch_size : n_texts;
- for (int start = 0; start < n_texts; start += max_batch) {
- int cur = n_texts - start;
- if (cur > max_batch) cur = max_batch;
-
- double t0 = now_ms();
- if (engine_embed_batch(e, inputs + start, cur,
- embs + (size_t)start * dim) != 0) {
- fprintf(stderr, "forward pass failed\n");
- rc = 1;
- goto done;
- }
- if (pplx_verbose >= 1)
- fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n",
- start, start + cur - 1, cur, now_ms() - t0);
- }
-
- if (n_texts == 1) {
- print_embedding_raw(embs, dim);
- } else {
- printf("Cosine similarity matrix (%d texts):\n", n_texts);
- printf("%-4s", "");
- for (int j = 0; j < n_texts; j++) printf(" [%d] ", j);
- printf("\n");
- for (int i = 0; i < n_texts; i++) {
- printf("[%d] ", i);
- for (int j = 0; j < n_texts; j++)
- printf(" %.4f", (double)pplx_cosine_similarity(
- embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
- printf(" \"%s\"\n", texts[i]);
- }
- if (print_embs) {
- printf("\nEmbeddings:\n");
- for (int i = 0; i < n_texts; i++) {
- printf("[%d] ", i);
- print_embedding_raw(embs + (size_t)i * dim, dim);
- }
- }
- }
-
-done:
- free_tokens(tokens, n_texts);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
- free(texts); free(tokens); free(inputs); free(embs);
- return rc;
-
-oom_texts:
- fprintf(stderr, "OOM\n");
- for (int i = 0; i < n_texts; i++) free(texts[i]);
- free(texts);
- return 1;
-}
-
-/* ========================================================================
- * Main
- * ======================================================================== */
-
-int main(int argc, char *argv[])
-{
- const char *model_dir = NULL;
- int n_threads = 0;
- int print_embs = 0;
- int verbose = 0;
- int use_mlx = 0;
- int use_cuda = 0;
- int stdin_mode = 0;
- int dist_worker_mode = 0;
- const char *dist_remote = NULL;
- int layer_start = -1;
- int layer_end = -1;
- int batch_size = 0;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
- int dist_activation_bits = 32;
- const char *host = "127.0.0.1";
- int port = 8000;
-
- int arg_start = 1;
- while (arg_start < argc && argv[arg_start][0] == '-') {
- const char *f = argv[arg_start];
- if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg_start]);
- }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg_start]);
- }
- else if (!strcmp(f, "--backend")) {
- const char *backend = argv[++arg_start];
- if (!strcmp(backend, "mlx")) {
-#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
- return 1;
-#endif
- } else if (!strcmp(backend, "cuda")) {
-#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
- return 1;
-#endif
- } else if (!strcmp(backend, "cpu")) {
- use_mlx = 0;
- use_cuda = 0;
- } else {
- fprintf(stderr, "invalid --backend: %s\n", backend);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg_start]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg_start]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- return 1;
- }
- }
- else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg_start];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg_start];
- }
- else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
- else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
- else if (!strcmp(f, "--layers")) {
- if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) {
- fprintf(stderr, "--layers expects START:END with START < END\n");
- return 1;
- }
- }
- else if (!strcmp(f, "--dist-activation-bits")) {
- dist_activation_bits = atoi(argv[++arg_start]);
- if (dist_activation_bits != 32 && dist_activation_bits != 16) {
- fprintf(stderr, "--dist-activation-bits must be 32 or 16\n");
- return 1;
- }
- }
- else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
- print_embs = 1;
- }
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
- else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; }
- else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; }
- else if (!strcmp(f, "--mlx")) {
-#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "--mlx not available (build with: make mlx)\n");
- return 1;
-#endif
- }
- else if (!strcmp(f, "--cuda")) {
-#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "--cuda not available (build with: make cuda)\n");
- return 1;
-#endif
- }
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(argv[0]);
- return 0;
- }
- else break;
- arg_start++;
- }
-
- pplx_verbose = verbose;
- qwen_verbose = verbose;
-
- if (cuda_fast_gemm) {
-#ifdef USE_CUDA
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n");
- return 1;
- }
- if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- return 1;
- }
-#else
- fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
- return 1;
-#endif
- }
-
- if (cuda_weights) {
-#ifdef USE_CUDA
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n");
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- pplx_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- pplx_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- return 1;
- }
-#else
- fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
- return 1;
-#endif
- }
-
- if (!model_dir || !model_dir[0]) {
- fprintf(stderr, "model directory required (-d <dir>)\n");
- print_usage(argv[0]);
- return 1;
- }
-
- if (dist_worker_mode && dist_remote) {
- fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n");
- return 1;
- }
- if ((dist_worker_mode || dist_remote) &&
- (layer_start < 0 || layer_end < 0)) {
- fprintf(stderr, "distributed mode requires --layers START:END\n");
- return 1;
- }
- if (dist_activation_bits != 32 && !dist_remote) {
- fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n");
- return 1;
- }
- if (dist_remote && layer_start != 0) {
- fprintf(stderr, "--dist-remote coordinator layers must start at zero\n");
- return 1;
- }
- if ((dist_worker_mode || dist_remote) && use_cuda) {
- fprintf(stderr, "distributed mode is not available with --cuda yet\n");
- return 1;
- }
-
- /* Threads (CPU backend) */
- if (!use_mlx && !use_cuda) {
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
- qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else {
- if (verbose >= 1)
- fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
- }
-
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n");
- return 1;
- }
-
- pplx_dist_options_t dist_opts = {
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .activation_bits = dist_activation_bits,
- };
-
- if (dist_worker_mode)
- return pplx_dist_run_worker_with_options(
- model_dir, host, port, layer_start, layer_end, use_mlx,
- &dist_opts);
-
- /* Tokenizer */
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
- double t0 = now_ms();
- qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
- if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; }
- if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
-
- /* Model */
- engine_t e = {0};
- e.tok = tok;
- t0 = now_ms();
-
-#ifdef USE_MLX
- if (dist_remote) {
- char remote_host[1024];
- int remote_port = 0;
- if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
- &remote_port) != 0) {
- fprintf(stderr, "--dist-remote expects HOST:PORT\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dist = pplx_dist_client_open_with_options(
- model_dir, remote_host, remote_port, layer_end, use_mlx,
- &dist_opts);
- if (!e.dist) {
- fprintf(stderr, "failed to connect distributed worker\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_dist_client_config(e.dist)->hidden_size;
- } else if (use_mlx) {
- pplx_mlx_options_t mlx_opts = {
- .quantize_bits = mlx_quantize_bits,
- .quantize_group_size = mlx_quantize_group_size,
- };
- e.mlx_ctx = pplx_mlx_load_with_options(model_dir, &mlx_opts);
- if (!e.mlx_ctx) {
- fprintf(stderr, "failed to load model\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size;
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- e.cuda_ctx = pplx_cuda_load(model_dir);
- if (!e.cuda_ctx) {
- fprintf(stderr, "failed to load CUDA model\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size;
- } else
-#endif
- {
- if (dist_remote) {
- char remote_host[1024];
- int remote_port = 0;
- if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
- &remote_port) != 0) {
- fprintf(stderr, "--dist-remote expects HOST:PORT\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dist = pplx_dist_client_open_with_options(
- model_dir, remote_host, remote_port, layer_end, use_mlx,
- &dist_opts);
- if (!e.dist) {
- fprintf(stderr, "failed to connect distributed worker\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_dist_client_config(e.dist)->hidden_size;
- } else {
- e.model = pplx_model_load(model_dir);
- if (!e.model) {
- fprintf(stderr, "failed to load model\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.workspace = pplx_workspace_new(e.model);
- if (!e.workspace) {
- fprintf(stderr, "failed to allocate workspace\n");
- pplx_model_free(e.model);
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_model_config(e.model)->hidden_size;
- }
- }
- if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n",
- e.dim, now_ms() - t0,
- use_mlx ? " (MLX)" : "",
- use_cuda ? " (CUDA)" : "");
-
- e.tok_ws = qwen_tokenizer_workspace_new();
- if (!e.tok_ws) {
- fprintf(stderr, "failed to allocate tokenizer workspace\n");
- if (e.workspace) pplx_workspace_free(e.workspace);
- if (e.model) pplx_model_free(e.model);
- if (e.dist) pplx_dist_client_free(e.dist);
-#ifdef USE_MLX
- if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
-#endif
- qwen_tokenizer_free(tok);
- return 1;
- }
-
- /* Run */
- int rc;
- if (stdin_mode)
- rc = run_stdin(&e, batch_size > 0 ? batch_size : 1);
- else
- rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size);
-
- /* Cleanup */
- if (e.workspace) pplx_workspace_free(e.workspace);
- if (e.model) pplx_model_free(e.model);
- if (e.dist) pplx_dist_client_free(e.dist);
-#ifdef USE_MLX
- if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
-#endif
- qwen_tokenizer_workspace_free(e.tok_ws);
- qwen_tokenizer_free(tok);
- return rc;
-}