aboutsummaryrefslogtreecommitdiff
path: root/embed_cli.c
diff options
context:
space:
mode:
Diffstat (limited to 'embed_cli.c')
-rw-r--r--embed_cli.c825
1 files changed, 825 insertions, 0 deletions
diff --git a/embed_cli.c b/embed_cli.c
new file mode 100644
index 0000000..140494b
--- /dev/null
+++ b/embed_cli.c
@@ -0,0 +1,825 @@
+/* main.c - pplx_embed command-line tool */
+
+#include "pplx_embed.h"
+#include "pplx_distributed.h"
+#include "qwen_kernels.h"
+#include "qwen_tokenizer.h"
+
+#ifdef USE_MLX
+#include "pplx_embed_mlx.h"
+#endif
+#ifdef USE_CUDA
+#include "pplx_embed_cuda.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+
+static double now_ms(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
+}
+
+static void print_usage(const char *prog)
+{
+ fprintf(stderr,
+ "Usage: %s -d <model_dir> [options] [text...]\n"
+ "\n"
+ "Options:\n"
+ " -d <dir> Model directory (required)\n"
+ " --mlx Use Apple MLX GPU backend\n"
+ " --cuda Use CUDA/cuBLAS GPU backend\n"
+ " --stream Read lines from stdin, write one JSON embedding per line\n"
+ " --dist-worker\n"
+ " Serve one final distributed layer shard\n"
+ " --dist-remote HOST:PORT\n"
+ " Use one remote final shard for CLI or --stream inference\n"
+ " --layers A:B Distributed half-open layer range, for example 0:14\n"
+ " --dist-activation-bits N\n"
+ " Distributed activation transport width: 32 or 16 (default: 32)\n"
+ " --mlx-quant-bits N\n"
+ " Quantize MLX linear weights to 8 bits at load time\n"
+ " --mlx-quant-group-size N\n"
+ " MLX quantization group size (default: 64)\n"
+ " --cuda-gemm-mode MODE\n"
+ " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
+ " --cuda-weight-dtype DTYPE\n"
+ " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
+ " weight memory and uses BF16 tensor cores\n"
+ " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n"
+ " --port N Bind port for --dist-worker (default: 8000)\n"
+ " -b, --batch-size N\n"
+ " Max texts per engine batch (default: all; --stream: 1)\n"
+ " -t, --threads N\n"
+ " CPU threads (default: all cores)\n"
+ " -e, --embeddings\n"
+ " Print raw embeddings (with multiple texts)\n"
+ " -v, --verbose\n"
+ " Verbose (-vv for debug)\n"
+ " -h Show this help\n"
+ "\n"
+ "Modes:\n"
+ " 1 text arg Print embedding as space-separated floats\n"
+ " 2+ text args Print cosine similarity matrix\n"
+ " no args Batch: read all stdin lines, then similarity matrix\n"
+ " --stream Streaming: read stdin lines, write JSON per line\n"
+ " --dist-worker Final layer-shard worker\n"
+ "\n"
+ "The HTTP API is served by the separate pplx-embed-server binary.\n"
+ "\n"
+ "Examples:\n"
+ " %s -d ./model \"query: what is AI?\"\n"
+ " %s -d ./model --mlx --stream < texts.txt\n"
+ " %s -d ./model --dist-worker --layers 14:28 --port 9000\n"
+ " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n",
+ prog, prog, prog, prog, prog);
+}
+
+/* ========================================================================
+ * Embed one text: tokenize > forward > return float[dim]
+ * ======================================================================== */
+
+typedef struct {
+ pplx_model_t *model;
+ pplx_workspace_t *workspace;
+ pplx_dist_client_t *dist;
+#ifdef USE_MLX
+ pplx_mlx_ctx_t *mlx_ctx;
+#endif
+#ifdef USE_CUDA
+ pplx_cuda_ctx_t *cuda_ctx;
+#endif
+ qwen_tokenizer_t *tok;
+ qwen_tokenizer_workspace_t *tok_ws;
+ int dim;
+} engine_t;
+
+typedef struct {
+ int *ids;
+ int n_tokens;
+} token_buf_t;
+
+static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (e->dist)
+ return pplx_dist_client_embed_batch(e->dist, inputs, batch,
+ out_embeddings);
+#ifdef USE_MLX
+ if (e->mlx_ctx)
+ return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings);
+#endif
+#ifdef USE_CUDA
+ if (e->cuda_ctx)
+ return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings);
+#endif
+ return pplx_model_embed_batch(e->model, e->workspace,
+ inputs, batch, out_embeddings);
+}
+
+static int parse_layers(const char *arg, int *start, int *end)
+{
+ int used = 0;
+ if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 ||
+ arg[used] != '\0' || *start < 0 || *start >= *end)
+ return -1;
+ return 0;
+}
+
+static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port)
+{
+ const char *colon = arg ? strrchr(arg, ':') : NULL;
+ if (!colon || colon == arg || !colon[1]) return -1;
+ size_t n = (size_t)(colon - arg);
+ if (n >= hostlen) return -1;
+ memcpy(host, arg, n);
+ host[n] = '\0';
+ int used = 0;
+ if (sscanf(colon + 1, "%d%n", port, &used) != 1 ||
+ colon[1 + used] != '\0' || *port <= 0 || *port > 65535)
+ return -1;
+ return 0;
+}
+
+static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
+{
+ memset(out, 0, sizeof(*out));
+ out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
+ text, &out->n_tokens);
+ if (!out->ids || out->n_tokens == 0) {
+ fprintf(stderr, "tokenization failed: %s\n", text);
+ free(out->ids);
+ out->ids = NULL;
+ out->n_tokens = 0;
+ return -1;
+ }
+
+ if (pplx_verbose >= 1) {
+ fprintf(stderr, "tokens (%d): ", out->n_tokens);
+ for (int i = 0; i < out->n_tokens && i < 20; i++)
+ fprintf(stderr, "%d ", out->ids[i]);
+ if (out->n_tokens > 20) fprintf(stderr, "...");
+ fprintf(stderr, "\n");
+ }
+ return 0;
+}
+
+static void free_tokens(token_buf_t *tokens, int n)
+{
+ if (!tokens) return;
+ for (int i = 0; i < n; i++)
+ free(tokens[i].ids);
+}
+
+/* ========================================================================
+ * Output helpers
+ * ======================================================================== */
+
+static void print_embedding_raw(const float *emb, int dim)
+{
+ for (int i = 0; i < dim; i++) {
+ if (i > 0) putchar(' ');
+ printf("%.8f", (double)emb[i]);
+ }
+ putchar('\n');
+}
+
+static size_t engine_workspace_nbytes(const engine_t *e)
+{
+ return (e && e->workspace) ? pplx_workspace_nbytes(e->workspace) : 0;
+}
+
+static void print_embedding_json(const float *emb, int dim, int n_tokens,
+ double ms, size_t workspace_bytes)
+{
+ printf("{\"embedding\":[");
+ for (int i = 0; i < dim; i++) {
+ if (i > 0) putchar(',');
+ printf("%.8f", (double)emb[i]);
+ }
+ printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n",
+ dim, n_tokens, ms, workspace_bytes);
+ fflush(stdout);
+}
+
+/* ========================================================================
+ * Stdin mode: read lines from stdin, write JSON to stdout
+ * ======================================================================== */
+
+static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
+{
+ if (n_lines <= 0) return 0;
+
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
+ pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t));
+ int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
+ float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
+ if (!tokens || !inputs || !input_to_line || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens); free(inputs); free(input_to_line); free(embs);
+ for (int i = 0; i < n_lines; i++)
+ printf("{\"error\":\"embedding failed\"}\n");
+ fflush(stdout);
+ return 1;
+ }
+
+ int n_inputs = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
+ inputs[n_inputs].ids = tokens[i].ids;
+ inputs[n_inputs].n_tokens = tokens[i].n_tokens;
+ input_to_line[n_inputs] = i;
+ n_inputs++;
+ }
+ }
+
+ double ms = 0.0;
+ int rc = 0;
+ if (n_inputs > 0) {
+ double t0 = now_ms();
+ rc = engine_embed_batch(e, inputs, n_inputs, embs);
+ ms = now_ms() - t0;
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
+ }
+
+ int next_valid = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (!tokens[i].ids) {
+ printf("{\"error\":\"tokenization failed\"}\n");
+ continue;
+ }
+ if (rc != 0) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ while (next_valid < n_inputs && input_to_line[next_valid] != i)
+ next_valid++;
+ if (next_valid >= n_inputs) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ print_embedding_json(embs + (size_t)next_valid * e->dim,
+ e->dim, tokens[i].n_tokens, ms,
+ engine_workspace_nbytes(e));
+ next_valid++;
+ }
+ fflush(stdout);
+
+ free_tokens(tokens, n_lines);
+ free(tokens); free(inputs); free(input_to_line); free(embs);
+ return rc == 0 ? 0 : 1;
+}
+
+static int run_stdin(engine_t *e, int batch_size)
+{
+ char line[65536];
+ if (batch_size <= 0) batch_size = 1;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n",
+ batch_size);
+
+ char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
+ if (!batch_lines) {
+ fprintf(stderr, "OOM\n");
+ return 1;
+ }
+ int n_batch = 0;
+ int rc = 0;
+
+ while (fgets(line, sizeof(line), stdin)) {
+ /* Strip newline */
+ size_t len = strlen(line);
+ while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r'))
+ line[--len] = '\0';
+ if (len == 0) continue;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "stdin: \"%.*s%s\"\n",
+ (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
+
+ batch_lines[n_batch] = strdup(line);
+ if (!batch_lines[n_batch]) {
+ fprintf(stderr, "OOM\n");
+ rc = 1;
+ break;
+ }
+ n_batch++;
+
+ if (n_batch == batch_size) {
+ if (process_stdin_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++) {
+ free(batch_lines[i]);
+ batch_lines[i] = NULL;
+ }
+ n_batch = 0;
+ }
+ }
+
+ if (n_batch > 0) {
+ if (process_stdin_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++)
+ free(batch_lines[i]);
+ }
+ free(batch_lines);
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "stdin: EOF\n");
+
+ return rc;
+}
+
+/* ========================================================================
+ * Batch mode: embed args or stdin lines, then print similarity or vectors
+ * ======================================================================== */
+
+static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
+{
+ if (*n_texts == *cap) {
+ int new_cap = *cap ? *cap * 2 : 8;
+ char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
+ if (!new_texts) return -1;
+ *texts = new_texts;
+ *cap = new_cap;
+ }
+
+ (*texts)[*n_texts] = strdup(s);
+ if (!(*texts)[*n_texts]) return -1;
+ (*n_texts)++;
+ return 0;
+}
+
+static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
+ int print_embs, int batch_size)
+{
+ int n_texts = 0, cap = 0;
+ char **texts = NULL;
+ int rc = 0;
+
+ if (arg_start < argc) {
+ for (int i = arg_start; i < argc; i++) {
+ if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
+ goto oom_texts;
+ }
+ } else {
+ char line[65536];
+ while (fgets(line, sizeof(line), stdin)) {
+ size_t l = strlen(line);
+ while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
+ line[--l] = '\0';
+ if (l == 0) continue;
+ if (append_text(&texts, &n_texts, &cap, line) != 0)
+ goto oom_texts;
+ }
+ }
+
+ if (n_texts == 0) {
+ fprintf(stderr, "no input texts\n");
+ free(texts);
+ return 1;
+ }
+
+ int dim = e->dim;
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
+ pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t));
+ float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
+ if (!tokens || !inputs || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens); free(inputs); free(embs);
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts);
+ return 1;
+ }
+
+ for (int i = 0; i < n_texts; i++) {
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
+ if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
+ rc = 1;
+ goto done;
+ }
+ inputs[i].ids = tokens[i].ids;
+ inputs[i].n_tokens = tokens[i].n_tokens;
+ }
+
+ int max_batch = batch_size > 0 ? batch_size : n_texts;
+ for (int start = 0; start < n_texts; start += max_batch) {
+ int cur = n_texts - start;
+ if (cur > max_batch) cur = max_batch;
+
+ double t0 = now_ms();
+ if (engine_embed_batch(e, inputs + start, cur,
+ embs + (size_t)start * dim) != 0) {
+ fprintf(stderr, "forward pass failed\n");
+ rc = 1;
+ goto done;
+ }
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n",
+ start, start + cur - 1, cur, now_ms() - t0);
+ }
+
+ if (n_texts == 1) {
+ print_embedding_raw(embs, dim);
+ } else {
+ printf("Cosine similarity matrix (%d texts):\n", n_texts);
+ printf("%-4s", "");
+ for (int j = 0; j < n_texts; j++) printf(" [%d] ", j);
+ printf("\n");
+ for (int i = 0; i < n_texts; i++) {
+ printf("[%d] ", i);
+ for (int j = 0; j < n_texts; j++)
+ printf(" %.4f", (double)pplx_cosine_similarity(
+ embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
+ printf(" \"%s\"\n", texts[i]);
+ }
+ if (print_embs) {
+ printf("\nEmbeddings:\n");
+ for (int i = 0; i < n_texts; i++) {
+ printf("[%d] ", i);
+ print_embedding_raw(embs + (size_t)i * dim, dim);
+ }
+ }
+ }
+
+done:
+ free_tokens(tokens, n_texts);
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts); free(tokens); free(inputs); free(embs);
+ return rc;
+
+oom_texts:
+ fprintf(stderr, "OOM\n");
+ for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(texts);
+ return 1;
+}
+
+/* ========================================================================
+ * Main
+ * ======================================================================== */
+
+int main(int argc, char *argv[])
+{
+ const char *model_dir = NULL;
+ int n_threads = 0;
+ int print_embs = 0;
+ int verbose = 0;
+ int use_mlx = 0;
+ int use_cuda = 0;
+ int stdin_mode = 0;
+ int dist_worker_mode = 0;
+ const char *dist_remote = NULL;
+ int layer_start = -1;
+ int layer_end = -1;
+ int batch_size = 0;
+ int mlx_quantize_bits = 0;
+ int mlx_quantize_group_size = 64;
+ const char *cuda_fast_gemm = NULL;
+ const char *cuda_weights = NULL;
+ int dist_activation_bits = 32;
+ const char *host = "127.0.0.1";
+ int port = 8000;
+
+ int arg_start = 1;
+ while (arg_start < argc && argv[arg_start][0] == '-') {
+ const char *f = argv[arg_start];
+ if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
+ else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ n_threads = atoi(argv[++arg_start]);
+ }
+ else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg_start]);
+ }
+ else if (!strcmp(f, "--backend")) {
+ const char *backend = argv[++arg_start];
+ if (!strcmp(backend, "mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cpu")) {
+ use_mlx = 0;
+ use_cuda = 0;
+ } else {
+ fprintf(stderr, "invalid --backend: %s\n", backend);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-bits")) {
+ mlx_quantize_bits = atoi(argv[++arg_start]);
+ if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
+ fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-group-size")) {
+ mlx_quantize_group_size = atoi(argv[++arg_start]);
+ if (mlx_quantize_group_size <= 0) {
+ fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--cuda-gemm-mode")) {
+ cuda_fast_gemm = argv[++arg_start];
+ }
+ else if (!strcmp(f, "--cuda-weight-dtype")) {
+ cuda_weights = argv[++arg_start];
+ }
+ else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
+ else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
+ else if (!strcmp(f, "--layers")) {
+ if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) {
+ fprintf(stderr, "--layers expects START:END with START < END\n");
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--dist-activation-bits")) {
+ dist_activation_bits = atoi(argv[++arg_start]);
+ if (dist_activation_bits != 32 && dist_activation_bits != 16) {
+ fprintf(stderr, "--dist-activation-bits must be 32 or 16\n");
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
+ print_embs = 1;
+ }
+ else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
+ else if (!strcmp(f, "-vv")) { verbose = 2; }
+ else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
+ else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; }
+ else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; }
+ else if (!strcmp(f, "--mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "--mlx not available (build with: make mlx)\n");
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "--cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "--cuda not available (build with: make cuda)\n");
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(argv[0]);
+ return 0;
+ }
+ else break;
+ arg_start++;
+ }
+
+ pplx_verbose = verbose;
+ qwen_verbose = verbose;
+
+ if (cuda_fast_gemm) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n");
+ return 1;
+ }
+ if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
+ fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
+ return 1;
+#endif
+ }
+
+ if (cuda_weights) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n");
+ return 1;
+ }
+ if (!strcmp(cuda_weights, "bf16")) {
+ pplx_cuda_set_weights_bf16(1);
+ } else if (!strcmp(cuda_weights, "f32")) {
+ pplx_cuda_set_weights_bf16(0);
+ } else {
+ fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
+ return 1;
+#endif
+ }
+
+ if (!model_dir || !model_dir[0]) {
+ fprintf(stderr, "model directory required (-d <dir>)\n");
+ print_usage(argv[0]);
+ return 1;
+ }
+
+ if (dist_worker_mode && dist_remote) {
+ fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n");
+ return 1;
+ }
+ if ((dist_worker_mode || dist_remote) &&
+ (layer_start < 0 || layer_end < 0)) {
+ fprintf(stderr, "distributed mode requires --layers START:END\n");
+ return 1;
+ }
+ if (dist_activation_bits != 32 && !dist_remote) {
+ fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n");
+ return 1;
+ }
+ if (dist_remote && layer_start != 0) {
+ fprintf(stderr, "--dist-remote coordinator layers must start at zero\n");
+ return 1;
+ }
+ if ((dist_worker_mode || dist_remote) && use_cuda) {
+ fprintf(stderr, "distributed mode is not available with --cuda yet\n");
+ return 1;
+ }
+
+ /* Threads (CPU backend) */
+ if (!use_mlx && !use_cuda) {
+ if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ qwen_set_threads(n_threads);
+ if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ } else {
+ if (verbose >= 1)
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
+ }
+
+ if (mlx_quantize_bits && !use_mlx) {
+ fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n");
+ return 1;
+ }
+
+ pplx_dist_options_t dist_opts = {
+ .mlx_quantize_bits = mlx_quantize_bits,
+ .mlx_quantize_group_size = mlx_quantize_group_size,
+ .activation_bits = dist_activation_bits,
+ };
+
+ if (dist_worker_mode)
+ return pplx_dist_run_worker_with_options(
+ model_dir, host, port, layer_start, layer_end, use_mlx,
+ &dist_opts);
+
+ /* Tokenizer */
+ char vocab_path[1024];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+ double t0 = now_ms();
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; }
+ if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
+
+ /* Model */
+ engine_t e = {0};
+ e.tok = tok;
+ t0 = now_ms();
+
+#ifdef USE_MLX
+ if (dist_remote) {
+ char remote_host[1024];
+ int remote_port = 0;
+ if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
+ &remote_port) != 0) {
+ fprintf(stderr, "--dist-remote expects HOST:PORT\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dist = pplx_dist_client_open_with_options(
+ model_dir, remote_host, remote_port, layer_end, use_mlx,
+ &dist_opts);
+ if (!e.dist) {
+ fprintf(stderr, "failed to connect distributed worker\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_dist_client_config(e.dist)->hidden_size;
+ } else if (use_mlx) {
+ pplx_mlx_options_t mlx_opts = {
+ .quantize_bits = mlx_quantize_bits,
+ .quantize_group_size = mlx_quantize_group_size,
+ };
+ e.mlx_ctx = pplx_mlx_load_with_options(model_dir, &mlx_opts);
+ if (!e.mlx_ctx) {
+ fprintf(stderr, "failed to load model\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size;
+ } else
+#endif
+#ifdef USE_CUDA
+ if (use_cuda) {
+ e.cuda_ctx = pplx_cuda_load(model_dir);
+ if (!e.cuda_ctx) {
+ fprintf(stderr, "failed to load CUDA model\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size;
+ } else
+#endif
+ {
+ if (dist_remote) {
+ char remote_host[1024];
+ int remote_port = 0;
+ if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
+ &remote_port) != 0) {
+ fprintf(stderr, "--dist-remote expects HOST:PORT\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dist = pplx_dist_client_open_with_options(
+ model_dir, remote_host, remote_port, layer_end, use_mlx,
+ &dist_opts);
+ if (!e.dist) {
+ fprintf(stderr, "failed to connect distributed worker\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_dist_client_config(e.dist)->hidden_size;
+ } else {
+ e.model = pplx_model_load(model_dir);
+ if (!e.model) {
+ fprintf(stderr, "failed to load model\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.workspace = pplx_workspace_new(e.model);
+ if (!e.workspace) {
+ fprintf(stderr, "failed to allocate workspace\n");
+ pplx_model_free(e.model);
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_model_config(e.model)->hidden_size;
+ }
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n",
+ e.dim, now_ms() - t0,
+ use_mlx ? " (MLX)" : "",
+ use_cuda ? " (CUDA)" : "");
+
+ e.tok_ws = qwen_tokenizer_workspace_new();
+ if (!e.tok_ws) {
+ fprintf(stderr, "failed to allocate tokenizer workspace\n");
+ if (e.workspace) pplx_workspace_free(e.workspace);
+ if (e.model) pplx_model_free(e.model);
+ if (e.dist) pplx_dist_client_free(e.dist);
+#ifdef USE_MLX
+ if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
+#endif
+#ifdef USE_CUDA
+ if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
+#endif
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+
+ /* Run */
+ int rc;
+ if (stdin_mode)
+ rc = run_stdin(&e, batch_size > 0 ? batch_size : 1);
+ else
+ rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size);
+
+ /* Cleanup */
+ if (e.workspace) pplx_workspace_free(e.workspace);
+ if (e.model) pplx_model_free(e.model);
+ if (e.dist) pplx_dist_client_free(e.dist);
+#ifdef USE_MLX
+ if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
+#endif
+#ifdef USE_CUDA
+ if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
+#endif
+ qwen_tokenizer_workspace_free(e.tok_ws);
+ qwen_tokenizer_free(tok);
+ return rc;
+}