aboutsummaryrefslogtreecommitdiff
path: root/embed_cli.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061 /embed_cli.c
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
Diffstat (limited to 'embed_cli.c')
-rw-r--r--embed_cli.c388
1 files changed, 208 insertions, 180 deletions
diff --git a/embed_cli.c b/embed_cli.c
index caa586a..9baa104 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -18,68 +18,66 @@
#include <time.h>
#include <limits.h>
-static double now_ms(void)
-{
+static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
}
-static void print_usage(const char *prog)
-{
+static void print_usage(const char *prog) {
fprintf(stderr,
- "Usage: %s -d <model_dir> [options] [text...]\n"
- "\n"
- "Options:\n"
- " -d <dir> Model directory (required)\n"
+ "Usage: %s -d <model_dir> [options] [text...]\n"
+ "\n"
+ "Options:\n"
+ " -d <dir> Model directory (required)\n"
#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
+ " --mlx Use Apple MLX GPU backend\n"
#endif
#ifdef USE_CUDA
- " --cuda Use CUDA/cuBLAS GPU backend\n"
+ " --cuda Use CUDA/cuBLAS GPU backend\n"
#endif
- " --stream Read lines from stdin, write one JSON embedding per line\n"
- " --json Emit JSON instead of plain text (embedding or matrix)\n"
+ " --stream Read lines from stdin, write one JSON embedding per line\n"
+ " --json Emit JSON instead of plain text (embedding or matrix)\n"
#ifdef USE_MLX
- " --mlx-quant-bits N\n"
- " Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quant-group-size N\n"
- " MLX quantization group size (default: 64)\n"
+ " --mlx-quant-bits N\n"
+ " Quantize MLX linear weights to 8 bits at load time\n"
+ " --mlx-quant-group-size N\n"
+ " MLX quantization group size (default: 64)\n"
#endif
#ifdef USE_CUDA
- " --cuda-gemm-mode MODE\n"
- " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weight-dtype DTYPE\n"
- " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
- " weight memory and uses BF16 tensor cores\n"
+ " --cuda-gemm-mode MODE\n"
+ " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
+ " --cuda-weight-dtype DTYPE\n"
+ " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
+ " weight memory and uses BF16 tensor cores\n"
#endif
- " -b, --batch-size N\n"
- " Max texts per engine batch (default: all; --stream: 1)\n"
- " -t, --threads N\n"
- " CPU threads (default: all cores)\n"
- " -e, --embeddings\n"
- " Print raw embeddings (with multiple texts)\n"
- " -v, --verbose\n"
- " Verbose (-vv for debug)\n"
- " -V, --version\n"
- " Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h Show this help\n"
- "\n"
- "Modes:\n"
- " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
- " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
- " no args Batch: read all stdin lines, then similarity matrix\n"
- " --stream Streaming: read stdin lines, write JSON per line\n"
- "\n"
- "Examples:\n"
- " %s -d ./model \"what is AI?\"\n"
+ " -b, --batch-size N\n"
+ " Max texts per engine batch (default: all; --stream: 1)\n"
+ " -t, --threads N\n"
+ " CPU threads (default: all cores)\n"
+ " -e, --embeddings\n"
+ " Print raw embeddings (with multiple texts)\n"
+ " -v, --verbose\n"
+ " Verbose (-vv for debug)\n"
+ " -V, --version\n"
+ " Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h Show this help\n"
+ "\n"
+ "Modes:\n"
+ " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
+ " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
+ " no args Batch: read all stdin lines, then similarity matrix\n"
+ " --stream Streaming: read stdin lines, write JSON per line\n"
+ "\n"
+ "Examples:\n"
+ " %s -d ./model \"what is AI?\"\n"
#ifdef USE_MLX
- " %s -d ./model --mlx --stream < texts.txt\n",
- prog, prog, prog);
+ " %s -d ./model --mlx --stream < texts.txt\n",
+ prog, prog, prog);
#else
- ,
- prog, prog);
+ ,
+ prog, prog);
#endif
}
@@ -88,29 +86,28 @@ static void print_usage(const char *prog)
* ======================================================================== */
typedef struct {
- embed_model_t *model;
+ embed_model_t *model;
embed_workspace_t *workspace;
#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
+ embed_mlx_ctx_t *mlx_ctx;
#endif
#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
+ embed_cuda_ctx_t *cuda_ctx;
#endif
qwen_tokenizer_t *tok;
qwen_tokenizer_workspace_t *tok_ws;
- int dim;
- int terminal_token_id;
- int append_terminal_token;
+ int dim;
+ int terminal_token_id;
+ int append_terminal_token;
} engine_t;
typedef struct {
int *ids;
- int n_tokens;
+ int n_tokens;
} token_buf_t;
-static int engine_embed_batch(engine_t *e, const embed_input_t *inputs,
- int batch, float *out_embeddings)
-{
+static int
+engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) {
#ifdef USE_MLX
if (e->mlx_ctx)
return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings);
@@ -119,15 +116,12 @@ static int engine_embed_batch(engine_t *e, const embed_input_t *inputs,
if (e->cuda_ctx)
return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings);
#endif
- return embed_model_encode_batch(e->model, e->workspace,
- inputs, batch, out_embeddings);
+ return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings);
}
-static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
-{
+static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) {
memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
- text, &out->n_tokens);
+ out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens);
if (!out->ids || out->n_tokens == 0) {
fprintf(stderr, "tokenization failed: %s\n", text);
free(out->ids);
@@ -141,8 +135,7 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
memset(out, 0, sizeof(*out));
return -1;
}
- int *ids = (int *)realloc(
- out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
if (!ids) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -156,15 +149,16 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
fprintf(stderr, "tokens (%d): ", out->n_tokens);
for (int i = 0; i < out->n_tokens && i < 20; i++)
fprintf(stderr, "%d ", out->ids[i]);
- if (out->n_tokens > 20) fprintf(stderr, "...");
+ if (out->n_tokens > 20)
+ fprintf(stderr, "...");
fprintf(stderr, "\n");
}
return 0;
}
-static void free_tokens(token_buf_t *tokens, int n)
-{
- if (!tokens) return;
+static void free_tokens(token_buf_t *tokens, int n) {
+ if (!tokens)
+ return;
for (int i = 0; i < n; i++)
free(tokens[i].ids);
}
@@ -173,41 +167,40 @@ static void free_tokens(token_buf_t *tokens, int n)
* Output helpers
* ======================================================================== */
-static void print_embedding_raw(const float *emb, int dim)
-{
+static void print_embedding_raw(const float *emb, int dim) {
for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(' ');
+ if (i > 0)
+ putchar(' ');
printf("%.8f", (double)emb[i]);
}
putchar('\n');
}
/* Single embedding as a JSON array of floats. */
-static void print_embedding_array(const float *emb, int dim)
-{
+static void print_embedding_array(const float *emb, int dim) {
putchar('[');
for (int i = 0; i < dim; i++) {
- if (i) putchar(',');
+ if (i)
+ putchar(',');
printf("%.8f", (double)emb[i]);
}
puts("]");
}
-static size_t engine_workspace_nbytes(const engine_t *e)
-{
+static size_t engine_workspace_nbytes(const engine_t *e) {
return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0;
}
-static void print_embedding_json(const float *emb, int dim, int n_tokens,
- double ms, size_t workspace_bytes)
-{
+static void
+print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) {
printf("{\"embedding\":[");
for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(',');
+ if (i > 0)
+ putchar(',');
printf("%.8f", (double)emb[i]);
}
- printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n",
- dim, n_tokens, ms, workspace_bytes);
+ printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms,
+ workspace_bytes);
fflush(stdout);
}
@@ -215,9 +208,9 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens,
* Stdin mode: read lines from stdin, write JSON to stdout
* ======================================================================== */
-static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
-{
- if (n_lines <= 0) return 0;
+static int process_stdin_batch(engine_t *e, char **lines, int n_lines) {
+ if (n_lines <= 0)
+ return 0;
token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t));
@@ -225,7 +218,10 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
if (!tokens || !inputs || !input_to_line || !embs) {
fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(input_to_line); free(embs);
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
for (int i = 0; i < n_lines; i++)
printf("{\"error\":\"embedding failed\"}\n");
fflush(stdout);
@@ -268,26 +264,27 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
printf("{\"error\":\"embedding failed\"}\n");
continue;
}
- print_embedding_json(embs + (size_t)next_valid * e->dim,
- e->dim, tokens[i].n_tokens, ms,
+ print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms,
engine_workspace_nbytes(e));
next_valid++;
}
fflush(stdout);
free_tokens(tokens, n_lines);
- free(tokens); free(inputs); free(input_to_line); free(embs);
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
return rc == 0 ? 0 : 1;
}
-static int run_stdin(engine_t *e, int batch_size)
-{
+static int run_stdin(engine_t *e, int batch_size) {
char line[65536];
- if (batch_size <= 0) batch_size = 1;
+ if (batch_size <= 0)
+ batch_size = 1;
if (embed_verbose >= 1)
- fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n",
- batch_size);
+ fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size);
char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
if (!batch_lines) {
@@ -300,13 +297,14 @@ static int run_stdin(engine_t *e, int batch_size)
while (fgets(line, sizeof(line), stdin)) {
/* Strip newline */
size_t len = strlen(line);
- while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r'))
+ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
line[--len] = '\0';
- if (len == 0) continue;
+ if (len == 0)
+ continue;
if (embed_verbose >= 1)
- fprintf(stderr, "stdin: \"%.*s%s\"\n",
- (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
+ fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line,
+ len > 60 ? "..." : "");
batch_lines[n_batch] = strdup(line);
if (!batch_lines[n_batch]) {
@@ -345,18 +343,19 @@ static int run_stdin(engine_t *e, int batch_size)
* Batch mode: embed args or stdin lines, then print similarity or vectors
* ======================================================================== */
-static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
-{
+static int append_text(char ***texts, int *n_texts, int *cap, const char *s) {
if (*n_texts == *cap) {
int new_cap = *cap ? *cap * 2 : 8;
char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
- if (!new_texts) return -1;
+ if (!new_texts)
+ return -1;
*texts = new_texts;
*cap = new_cap;
}
(*texts)[*n_texts] = strdup(s);
- if (!(*texts)[*n_texts]) return -1;
+ if (!(*texts)[*n_texts])
+ return -1;
(*n_texts)++;
return 0;
}
@@ -364,18 +363,18 @@ static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one
* row of space-separated values per line. With print_embs the raw embeddings
* follow, one per line after a blank line (plain-text output only). */
-static void print_matrix(const float *embs, int n, int dim,
- int json, int print_embs)
-{
+static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) {
if (json) {
putchar('[');
for (int i = 0; i < n; i++) {
- if (i) putchar(',');
+ if (i)
+ putchar(',');
putchar('[');
for (int j = 0; j < n; j++) {
- if (j) putchar(',');
- printf("%.6f", (double)embed_cosine_similarity(
- embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
+ if (j)
+ putchar(',');
+ printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
}
putchar(']');
}
@@ -385,9 +384,10 @@ static void print_matrix(const float *embs, int n, int dim,
for (int i = 0; i < n; i++) {
for (int j = 0; j < n; j++) {
- if (j) putchar(' ');
- printf("%.6f", (double)embed_cosine_similarity(
- embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
+ if (j)
+ putchar(' ');
+ printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
}
putchar('\n');
}
@@ -398,12 +398,11 @@ static void print_matrix(const float *embs, int n, int dim,
}
}
-static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
- int print_embs, int batch_size, int json)
-{
- int n_texts = 0, cap = 0;
- char **texts = NULL;
- int rc = 0;
+static int run_batch(
+ engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) {
+ int n_texts = 0, cap = 0;
+ char **texts = NULL;
+ int rc = 0;
if (arg_start < argc) {
for (int i = arg_start; i < argc; i++) {
@@ -414,9 +413,10 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
char line[65536];
while (fgets(line, sizeof(line), stdin)) {
size_t l = strlen(line);
- while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
+ while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r'))
line[--l] = '\0';
- if (l == 0) continue;
+ if (l == 0)
+ continue;
if (append_text(&texts, &n_texts, &cap, line) != 0)
goto oom_texts;
}
@@ -434,15 +434,18 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
if (!tokens || !inputs || !embs) {
fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(embs);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(tokens);
+ free(inputs);
+ free(embs);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
free(texts);
return 1;
}
for (int i = 0; i < n_texts; i++) {
if (embed_verbose >= 1)
- fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
+ fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]);
if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
rc = 1;
goto done;
@@ -454,36 +457,43 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
int max_batch = batch_size > 0 ? batch_size : n_texts;
for (int start = 0; start < n_texts; start += max_batch) {
int cur = n_texts - start;
- if (cur > max_batch) cur = max_batch;
+ if (cur > max_batch)
+ cur = max_batch;
double t0 = now_ms();
- if (engine_embed_batch(e, inputs + start, cur,
- embs + (size_t)start * dim) != 0) {
+ if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) {
fprintf(stderr, "forward pass failed\n");
rc = 1;
goto done;
}
if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n",
- start, start + cur - 1, cur, now_ms() - t0);
+ fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1,
+ cur, now_ms() - t0);
}
if (n_texts == 1) {
- if (json) print_embedding_array(embs, dim);
- else print_embedding_raw(embs, dim);
+ if (json)
+ print_embedding_array(embs, dim);
+ else
+ print_embedding_raw(embs, dim);
} else {
print_matrix(embs, n_texts, dim, json, print_embs);
}
done:
free_tokens(tokens, n_texts);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
- free(texts); free(tokens); free(inputs); free(embs);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
+ free(texts);
+ free(tokens);
+ free(inputs);
+ free(embs);
return rc;
oom_texts:
fprintf(stderr, "OOM\n");
- for (int i = 0; i < n_texts; i++) free(texts[i]);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
free(texts);
return 1;
}
@@ -492,15 +502,14 @@ oom_texts:
* Main
* ======================================================================== */
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
const char *model_dir = NULL;
- int n_threads = 0;
+ int n_threads = 0;
int print_embs = 0;
- int verbose = 0;
+ int verbose = 0;
int stdin_mode = 0;
int batch_size = 0;
- int json = 0;
+ int json = 0;
const char *prog = embed_prog_name(argv[0]);
#ifdef USE_MLX
int use_mlx = 0;
@@ -516,22 +525,21 @@ int main(int argc, char *argv[])
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
const char *f = argv[arg_start];
- if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
- else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
+ if (!strcmp(f, "-d")) {
+ model_dir = argv[++arg_start];
+ } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
embed_print_version(prog);
return 0;
- }
- else if (!strcmp(f, "--build-info")) {
+ } else if (!strcmp(f, "--build-info")) {
embed_print_build_info(prog);
return 0;
- }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
n_threads = atoi(argv[++arg_start]);
- }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "--json")) {
+ json = 1;
}
- else if (!strcmp(f, "--json")) { json = 1; }
#ifdef USE_MLX
else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg_start]);
@@ -539,8 +547,7 @@ int main(int argc, char *argv[])
fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
return 1;
}
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
+ } else if (!strcmp(f, "--mlx-quant-group-size")) {
mlx_quantize_group_size = atoi(argv[++arg_start]);
if (mlx_quantize_group_size <= 0) {
fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
@@ -551,28 +558,34 @@ int main(int argc, char *argv[])
#ifdef USE_CUDA
else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg_start];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
+ } else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg_start];
}
#endif
else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
print_embs = 1;
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "--stream")) {
+ stdin_mode = 1;
}
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
+ else if (!strcmp(f, "--mlx")) {
+ use_mlx = 1;
+ }
#endif
#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
+ else if (!strcmp(f, "--cuda")) {
+ use_cuda = 1;
+ }
#endif
else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(prog);
return 0;
- }
- else break;
+ } else
+ break;
arg_start++;
}
@@ -620,18 +633,22 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (use_mlx) {
- if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ if (verbose >= 1)
+ fprintf(stderr, "Using MLX GPU backend\n");
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ if (use_cuda) {
+ if (verbose >= 1)
+ fprintf(stderr, "Using CUDA GPU backend\n");
} else
#endif
{
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ if (n_threads <= 0)
+ n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ if (verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
}
/* Tokenizer */
@@ -639,8 +656,12 @@ int main(int argc, char *argv[])
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
double t0 = now_ms();
qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
- if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; }
- if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
+ if (!tok) {
+ fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path);
+ return 1;
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
/* Model */
engine_t e = {0};
@@ -665,7 +686,7 @@ int main(int argc, char *argv[])
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
+ if (use_cuda) {
e.cuda_ctx = embed_cuda_load(model_dir);
if (!e.cuda_ctx) {
fprintf(stderr, "failed to load CUDA model\n");
@@ -699,30 +720,33 @@ int main(int argc, char *argv[])
* wins; otherwise the released-model family constant applies. */
if (e.append_terminal_token) {
int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>");
- e.terminal_token_id =
- eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
}
if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
- e.dim, now_ms() - t0,
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0,
#ifdef USE_MLX
use_mlx ? " (MLX)" :
#endif
#ifdef USE_CUDA
- use_cuda ? " (CUDA)" :
+ use_cuda ? " (CUDA)"
+ :
#endif
- "");
+ "");
e.tok_ws = qwen_tokenizer_workspace_new();
if (!e.tok_ws) {
fprintf(stderr, "failed to allocate tokenizer workspace\n");
- if (e.workspace) embed_workspace_free(e.workspace);
- if (e.model) embed_model_free(e.model);
+ if (e.workspace)
+ embed_workspace_free(e.workspace);
+ if (e.model)
+ embed_model_free(e.model);
#ifdef USE_MLX
- if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx);
+ if (e.mlx_ctx)
+ embed_mlx_free(e.mlx_ctx);
#endif
#ifdef USE_CUDA
- if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx);
+ if (e.cuda_ctx)
+ embed_cuda_free(e.cuda_ctx);
#endif
qwen_tokenizer_free(tok);
return 1;
@@ -736,13 +760,17 @@ int main(int argc, char *argv[])
rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json);
/* Cleanup */
- if (e.workspace) embed_workspace_free(e.workspace);
- if (e.model) embed_model_free(e.model);
+ if (e.workspace)
+ embed_workspace_free(e.workspace);
+ if (e.model)
+ embed_model_free(e.model);
#ifdef USE_MLX
- if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx);
+ if (e.mlx_ctx)
+ embed_mlx_free(e.mlx_ctx);
#endif
#ifdef USE_CUDA
- if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx);
+ if (e.cuda_ctx)
+ embed_cuda_free(e.cuda_ctx);
#endif
qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);