aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
-rw-r--r--bench/bench.h74
-rw-r--r--bench/bench_kernels.c115
-rw-r--r--bench/bench_model.c24
-rw-r--r--bench/bench_tokenizer.c63
-rw-r--r--embed.c1048
-rw-r--r--embed.h108
-rw-r--r--embed_cli.c388
-rw-r--r--embed_cuda.cu870
-rw-r--r--embed_cuda.h20
-rw-r--r--embed_internal.h54
-rw-r--r--embed_mlx.c1153
-rw-r--r--embed_mlx.h69
-rw-r--r--embed_server.c1300
-rw-r--r--embed_server.h2
-rw-r--r--qwen_kernels.c534
-rw-r--r--qwen_kernels.h111
-rw-r--r--qwen_kernels_avx.c259
-rw-r--r--qwen_kernels_generic.c16
-rw-r--r--qwen_kernels_impl.h30
-rw-r--r--qwen_kernels_neon.c16
-rw-r--r--qwen_safetensors.c285
-rw-r--r--qwen_safetensors.h9
-rw-r--r--qwen_tokenizer.c998
-rw-r--r--qwen_tokenizer.h22
-rw-r--r--tests/test_bf16_model.c20
-rw-r--r--tests/test_cli.c76
-rw-r--r--tests/test_kernels.c119
-rw-r--r--tests/test_late.c85
-rw-r--r--tests/test_qwen3.c25
-rw-r--r--tests/test_safetensors.c67
-rw-r--r--tests/test_server.c436
-rw-r--r--tests/test_tokenizer.c159
-rw-r--r--tests/test_workspace.c78
-rw-r--r--tests/tiny_model.h97
-rw-r--r--tests/tok_fixture.h36
35 files changed, 4742 insertions, 4024 deletions
diff --git a/bench/bench.h b/bench/bench.h
index 9a46c71..6089174 100644
--- a/bench/bench.h
+++ b/bench/bench.h
@@ -30,8 +30,8 @@
#include <unistd.h>
typedef struct {
- long n; /* iterations: the measured loop must run exactly n times */
- double t0_ns; /* set by bench_begin(); end of un-measured setup */
+ long n; /* iterations: the measured loop must run exactly n times */
+ double t0_ns; /* set by bench_begin(); end of un-measured setup */
} bench_state_t;
typedef void (*bench_fn_t)(bench_state_t *);
@@ -45,36 +45,29 @@ typedef struct {
* compiler cannot delete the measured loop. */
static volatile float bench_sink;
-static double bench_now_ns(void)
-{
+static double bench_now_ns(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1e9 + (double)ts.tv_nsec;
}
-static void bench_begin(bench_state_t *b)
-{
- b->t0_ns = bench_now_ns();
-}
+static void bench_begin(bench_state_t *b) { b->t0_ns = bench_now_ns(); }
/* One timed run of fn at n iterations; returns ns/op. */
-static double bench_run_once(bench_fn_t fn, long n)
-{
- bench_state_t b = { n, 0.0 };
+static double bench_run_once(bench_fn_t fn, long n) {
+ bench_state_t b = {n, 0.0};
b.t0_ns = bench_now_ns();
fn(&b);
double elapsed = bench_now_ns() - b.t0_ns;
return elapsed / (double)n;
}
-static int bench_cmp_double(const void *a, const void *c)
-{
+static int bench_cmp_double(const void *a, const void *c) {
double x = *(const double *)a, y = *(const double *)c;
return (x > y) - (x < y);
}
-static double bench_median(double *v, int n)
-{
+static double bench_median(double *v, int n) {
qsort(v, (size_t)n, sizeof(*v), bench_cmp_double);
return n % 2 ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]);
}
@@ -88,13 +81,13 @@ typedef struct {
const char *meta;
} bench_opts_t;
-static void bench_parse_args(bench_opts_t *o, int argc, char **argv)
-{
+static void bench_parse_args(bench_opts_t *o, int argc, char **argv) {
o->benchtime_ms = 300;
o->count = 10;
o->filter = NULL;
o->json_path = NULL;
- if (!o->meta) o->meta = "";
+ if (!o->meta)
+ o->meta = "";
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--benchtime-ms") && i + 1 < argc)
o->benchtime_ms = atol(argv[++i]);
@@ -105,15 +98,16 @@ static void bench_parse_args(bench_opts_t *o, int argc, char **argv)
else if (!strcmp(argv[i], "--json") && i + 1 < argc)
o->json_path = argv[++i];
}
- if (o->benchtime_ms < 1) o->benchtime_ms = 1;
- if (o->count < 1) o->count = 1;
- if (o->count > 1000) o->count = 1000;
+ if (o->benchtime_ms < 1)
+ o->benchtime_ms = 1;
+ if (o->count < 1)
+ o->count = 1;
+ if (o->count > 1000)
+ o->count = 1000;
}
/* Run all cases (subject to the filter) and report. Returns 0 on success. */
-static int bench_main(const bench_opts_t *o,
- const bench_case_t *cases, int n_cases)
-{
+static int bench_main(const bench_opts_t *o, const bench_case_t *cases, int n_cases) {
FILE *jf = NULL;
if (o->json_path) {
jf = fopen(o->json_path, "w");
@@ -123,14 +117,16 @@ static int bench_main(const bench_opts_t *o,
}
char host[256] = "unknown";
gethostname(host, sizeof(host) - 1);
- fprintf(jf, "{\"host\":\"%s\",\"benchtime_ms\":%ld,\"count\":%d,"
+ fprintf(jf,
+ "{\"host\":\"%s\",\"benchtime_ms\":%ld,\"count\":%d,"
"\"meta\":\"%s\",\"benchmarks\":[",
host, o->benchtime_ms, o->count, o->meta);
}
double *samples = (double *)malloc((size_t)o->count * sizeof(double));
if (!samples) {
- if (jf) fclose(jf);
+ if (jf)
+ fclose(jf);
return 1;
}
@@ -140,7 +136,8 @@ static int bench_main(const bench_opts_t *o,
free(all);
free(iters);
free(samples);
- if (jf) fclose(jf);
+ if (jf)
+ fclose(jf);
return 1;
}
@@ -156,8 +153,10 @@ static int bench_main(const bench_opts_t *o,
while (ns_per_op * (double)n < target_ns) {
double want = target_ns / (ns_per_op > 0.0 ? ns_per_op : 1.0);
long next = (long)(want * 1.2) + 1;
- if (next > n * 100) next = n * 100; /* distrust tiny timings */
- if (next <= n) next = n + 1;
+ if (next > n * 100)
+ next = n * 100; /* distrust tiny timings */
+ if (next <= n)
+ next = n + 1;
n = next;
ns_per_op = bench_run_once(cases[c].fn, n);
}
@@ -174,8 +173,7 @@ static int bench_main(const bench_opts_t *o,
for (int s = 0; s < o->count; s++) {
for (int c = 0; c < n_cases; c++)
if (iters[c])
- all[(size_t)c * o->count + s] =
- bench_run_once(cases[c].fn, iters[c]);
+ all[(size_t)c * o->count + s] = bench_run_once(cases[c].fn, iters[c]);
fprintf(stderr, "pass %d/%d (%.0fs elapsed)\n", s + 1, o->count,
(bench_now_ns() - sample_t0) / 1e9);
}
@@ -184,22 +182,20 @@ static int bench_main(const bench_opts_t *o,
for (int c = 0; c < n_cases; c++) {
if (!iters[c])
continue;
- memcpy(samples, all + (size_t)c * o->count,
- (size_t)o->count * sizeof(double));
+ memcpy(samples, all + (size_t)c * o->count, (size_t)o->count * sizeof(double));
if (jf) {
- fprintf(jf, "%s{\"name\":\"%s\",\"iters\":%ld,\"ns_per_op\":[",
- emitted ? "," : "", cases[c].name, iters[c]);
+ fprintf(jf, "%s{\"name\":\"%s\",\"iters\":%ld,\"ns_per_op\":[", emitted ? "," : "",
+ cases[c].name, iters[c]);
for (int s = 0; s < o->count; s++)
fprintf(jf, "%s%.3f", s ? "," : "", samples[s]);
fprintf(jf, "]}");
}
double med = bench_median(samples, o->count);
- double spread = o->count > 1
- ? 100.0 * (samples[o->count - 1] - samples[0]) / med : 0.0;
- printf("%-44s %10ld iters %14.1f ns/op (spread %.1f%%)\n",
- cases[c].name, iters[c], med, spread);
+ double spread = o->count > 1 ? 100.0 * (samples[o->count - 1] - samples[0]) / med : 0.0;
+ printf("%-44s %10ld iters %14.1f ns/op (spread %.1f%%)\n", cases[c].name, iters[c], med,
+ spread);
fflush(stdout);
emitted++;
}
diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c
index cc5dc54..4639d18 100644
--- a/bench/bench_kernels.c
+++ b/bench/bench_kernels.c
@@ -12,12 +12,12 @@
#include <stdint.h>
-int qwen_verbose = 0; /* defined by embed.c in full builds */
+int qwen_verbose = 0; /* defined by embed.c in full builds */
enum {
HIDDEN = 1024,
- Q_DIM = 2048, /* 16 heads * 128 */
- KV_DIM = 1024, /* 8 heads * 128 */
+ Q_DIM = 2048, /* 16 heads * 128 */
+ KV_DIM = 1024, /* 8 heads * 128 */
HEAD_DIM = 128,
INTER = 3072,
SEQ = 8,
@@ -25,8 +25,7 @@ enum {
};
/* Deterministic fill in [-1, 1]; small values keep softmax/silu sane. */
-static void fill_f32(float *x, size_t n, unsigned seed)
-{
+static void fill_f32(float *x, size_t n, unsigned seed) {
unsigned s = seed * 2654435761u + 1u;
for (size_t i = 0; i < n; i++) {
s = s * 1664525u + 1013904223u;
@@ -34,15 +33,13 @@ static void fill_f32(float *x, size_t n, unsigned seed)
}
}
-static uint16_t f32_to_bf16(float x)
-{
+static uint16_t f32_to_bf16(float x) {
uint32_t u;
memcpy(&u, &x, sizeof(u));
return (uint16_t)(u >> 16);
}
-static void fill_bf16(uint16_t *x, size_t n, unsigned seed)
-{
+static void fill_bf16(uint16_t *x, size_t n, unsigned seed) {
unsigned s = seed * 2654435761u + 1u;
for (size_t i = 0; i < n; i++) {
s = s * 1664525u + 1013904223u;
@@ -50,26 +47,25 @@ static void fill_bf16(uint16_t *x, size_t n, unsigned seed)
}
}
-static float *alloc_f32(size_t n, unsigned seed)
-{
+static float *alloc_f32(size_t n, unsigned seed) {
float *x = (float *)malloc(n * sizeof(float));
- if (!x) exit(2);
+ if (!x)
+ exit(2);
fill_f32(x, n, seed);
return x;
}
-static uint16_t *alloc_bf16(size_t n, unsigned seed)
-{
+static uint16_t *alloc_bf16(size_t n, unsigned seed) {
uint16_t *x = (uint16_t *)malloc(n * sizeof(uint16_t));
- if (!x) exit(2);
+ if (!x)
+ exit(2);
fill_bf16(x, n, seed);
return x;
}
/* ---- F32 projections (matvec at seq 1, GEMM at seq 8) ---- */
-static void bm_linear_f32(bench_state_t *b, int seq, int in, int out)
-{
+static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) {
float *x = alloc_f32((size_t)seq * in, 1);
float *w = alloc_f32((size_t)out * in, 2);
float *y = alloc_f32((size_t)seq * out, 3);
@@ -77,18 +73,19 @@ static void bm_linear_f32(bench_state_t *b, int seq, int in, int out)
for (long i = 0; i < b->n; i++)
qwen_linear_nobias(y, x, w, seq, in, out);
bench_sink += y[0];
- free(y); free(w); free(x);
+ free(y);
+ free(w);
+ free(x);
}
-static void bm_linear_f32_s1_qproj(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, Q_DIM); }
-static void bm_linear_f32_s1_gate(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, INTER); }
-static void bm_linear_f32_s1_down(bench_state_t *b) { bm_linear_f32(b, 1, INTER, HIDDEN); }
-static void bm_linear_f32_s8_gate(bench_state_t *b) { bm_linear_f32(b, SEQ, HIDDEN, INTER); }
+static void bm_linear_f32_s1_qproj(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, Q_DIM); }
+static void bm_linear_f32_s1_gate(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, INTER); }
+static void bm_linear_f32_s1_down(bench_state_t *b) { bm_linear_f32(b, 1, INTER, HIDDEN); }
+static void bm_linear_f32_s8_gate(bench_state_t *b) { bm_linear_f32(b, SEQ, HIDDEN, INTER); }
/* ---- BF16 weights ---- */
-static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out)
-{
+static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) {
float *x = alloc_f32((size_t)seq * in, 4);
uint16_t *w = alloc_bf16((size_t)out * in, 5);
float *y = alloc_f32((size_t)seq * out, 6);
@@ -96,14 +93,15 @@ static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out)
for (long i = 0; i < b->n; i++)
qwen_linear_nobias_bf16(y, x, w, seq, in, out);
bench_sink += y[0];
- free(y); free(w); free(x);
+ free(y);
+ free(w);
+ free(x);
}
static void bm_linear_bf16_s1_gate(bench_state_t *b) { bm_linear_bf16(b, 1, HIDDEN, INTER); }
static void bm_linear_bf16_s8_gate(bench_state_t *b) { bm_linear_bf16(b, SEQ, HIDDEN, INTER); }
-static void bm_qkv_bf16(bench_state_t *b)
-{
+static void bm_qkv_bf16(bench_state_t *b) {
float *x = alloc_f32((size_t)SEQ * HIDDEN, 7);
uint16_t *wq = alloc_bf16((size_t)Q_DIM * HIDDEN, 8);
uint16_t *wk = alloc_bf16((size_t)KV_DIM * HIDDEN, 9);
@@ -113,14 +111,18 @@ static void bm_qkv_bf16(bench_state_t *b)
float *v = alloc_f32((size_t)SEQ * KV_DIM, 13);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv,
- SEQ, HIDDEN, Q_DIM, KV_DIM);
+ qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM);
bench_sink += q[0] + k[0] + v[0];
- free(v); free(k); free(q); free(wv); free(wk); free(wq); free(x);
+ free(v);
+ free(k);
+ free(q);
+ free(wv);
+ free(wk);
+ free(wq);
+ free(x);
}
-static void bm_bf16_widen(bench_state_t *b)
-{
+static void bm_bf16_widen(bench_state_t *b) {
size_t n = (size_t)INTER * HIDDEN;
uint16_t *src = alloc_bf16(n, 14);
float *dst = alloc_f32(n, 15);
@@ -128,13 +130,13 @@ static void bm_bf16_widen(bench_state_t *b)
for (long i = 0; i < b->n; i++)
qwen_bf16_to_f32_buf(dst, src, n);
bench_sink += dst[0];
- free(dst); free(src);
+ free(dst);
+ free(src);
}
/* ---- attention pieces ---- */
-static void bm_attn_scores(bench_state_t *b)
-{
+static void bm_attn_scores(bench_state_t *b) {
/* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */
float *qm = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 16);
float *km = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 17);
@@ -143,11 +145,12 @@ static void bm_attn_scores(bench_state_t *b)
for (long i = 0; i < b->n; i++)
qwen_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ);
bench_sink += s[0];
- free(s); free(km); free(qm);
+ free(s);
+ free(km);
+ free(qm);
}
-static void bm_softmax(bench_state_t *b)
-{
+static void bm_softmax(bench_state_t *b) {
float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19);
bench_begin(b);
for (long i = 0; i < b->n; i++)
@@ -156,23 +159,22 @@ static void bm_softmax(bench_state_t *b)
free(x);
}
-static void bm_rope(bench_state_t *b)
-{
+static void bm_rope(bench_state_t *b) {
float *x = alloc_f32((size_t)ATTN_SEQ * Q_DIM, 20);
float *cosv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 21);
float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM,
- HEAD_DIM);
+ qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM);
bench_sink += x[0];
- free(sinv); free(cosv); free(x);
+ free(sinv);
+ free(cosv);
+ free(x);
}
/* ---- norms, activations, reductions ---- */
-static void bm_rms_norm(bench_state_t *b)
-{
+static void bm_rms_norm(bench_state_t *b) {
float *x = alloc_f32((size_t)SEQ * HIDDEN, 23);
float *w = alloc_f32(HIDDEN, 24);
float *y = alloc_f32((size_t)SEQ * HIDDEN, 25);
@@ -180,13 +182,14 @@ static void bm_rms_norm(bench_state_t *b)
for (long i = 0; i < b->n; i++)
qwen_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f);
bench_sink += y[0];
- free(y); free(w); free(x);
+ free(y);
+ free(w);
+ free(x);
}
/* The engine's SwiGLU op mutates gate in place, so each iteration restores
* it first; the memcpy is a constant part of the measured loop. */
-static void bm_silu_mul(bench_state_t *b)
-{
+static void bm_silu_mul(bench_state_t *b) {
size_t n = (size_t)SEQ * INTER;
float *gate0 = alloc_f32(n, 26);
float *gate = alloc_f32(n, 26);
@@ -197,17 +200,19 @@ static void bm_silu_mul(bench_state_t *b)
qwen_silu_mul_inplace(gate, up, (int)n);
}
bench_sink += gate[0];
- free(up); free(gate); free(gate0);
+ free(up);
+ free(gate);
+ free(gate0);
}
-static void bm_dot(bench_state_t *b)
-{
+static void bm_dot(bench_state_t *b) {
float *x = alloc_f32(HIDDEN, 28);
float *y = alloc_f32(HIDDEN, 29);
bench_begin(b);
for (long i = 0; i < b->n; i++)
bench_sink += qwen_dot_f32(x, y, HIDDEN);
- free(y); free(x);
+ free(y);
+ free(x);
}
static const bench_case_t CASES[] = {
@@ -227,14 +232,14 @@ static const bench_case_t CASES[] = {
{"reduce/dot_1024", bm_dot},
};
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
int threads = 1;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--threads") && i + 1 < argc)
threads = atoi(argv[i + 1]);
}
- if (threads < 1) threads = 1;
+ if (threads < 1)
+ threads = 1;
qwen_set_threads(threads);
char meta[128];
diff --git a/bench/bench_model.c b/bench/bench_model.c
index 9b897e9..5cff015 100644
--- a/bench/bench_model.c
+++ b/bench/bench_model.c
@@ -19,15 +19,15 @@ static int g_hidden;
enum { MAX_BATCH = 8 };
-static void bm_embed(bench_state_t *b, int batch, int len)
-{
+static void bm_embed(bench_state_t *b, int batch, int len) {
int *ids = (int *)malloc((size_t)batch * len * sizeof(int));
float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float));
- if (!ids || !out) exit(2);
+ if (!ids || !out)
+ exit(2);
unsigned s = 99u;
for (int i = 0; i < batch * len; i++) {
s = s * 1664525u + 1013904223u;
- ids[i] = (int)(s % 1000u); /* ordinary vocab rows */
+ ids[i] = (int)(s % 1000u); /* ordinary vocab rows */
}
embed_input_t inputs[MAX_BATCH];
for (int i = 0; i < batch; i++) {
@@ -44,8 +44,8 @@ static void bm_embed(bench_state_t *b, int batch, int len)
free(ids);
}
-static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); }
-static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); }
+static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); }
+static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); }
static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); }
static const bench_case_t CASES[] = {
@@ -54,11 +54,9 @@ static const bench_case_t CASES[] = {
{"embed/b1_len256", bm_embed_b1_len256},
};
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
if (argc < 2 || argv[1][0] == '-') {
- fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n",
- argv[0]);
+ fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n", argv[0]);
return 2;
}
@@ -67,7 +65,8 @@ int main(int argc, char **argv)
if (!strcmp(argv[i], "--threads") && i + 1 < argc)
threads = atoi(argv[i + 1]);
}
- if (threads < 1) threads = 1;
+ if (threads < 1)
+ threads = 1;
qwen_set_threads(threads);
g_model = embed_model_load(argv[1]);
@@ -83,8 +82,7 @@ int main(int argc, char **argv)
g_hidden = embed_model_config(g_model)->hidden_size;
char meta[160];
- snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d",
- threads, g_hidden);
+ snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", threads, g_hidden);
bench_opts_t opts = {0};
opts.meta = meta;
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 1033f2e..fb6d37a 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -10,24 +10,23 @@
int qwen_verbose = 0;
-static double now_ms(void)
-{
+static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
}
-static int same_ids(const int *a, int na, const int *b, int nb)
-{
- if (na != nb) return 0;
+static int same_ids(const int *a, int na, const int *b, int nb) {
+ if (na != nb)
+ return 0;
for (int i = 0; i < na; i++) {
- if (a[i] != b[i]) return 0;
+ if (a[i] != b[i])
+ return 0;
}
return 1;
}
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
int status = 1;
int **gold = NULL;
int *gold_n = NULL;
@@ -41,7 +40,8 @@ int main(int argc, char **argv)
const char *vocab = argv[1];
int runs = atoi(argv[2]);
- if (runs <= 0) return 2;
+ if (runs <= 0)
+ return 2;
int n_texts = argc - 3;
char **texts = argv + 3;
@@ -56,17 +56,18 @@ int main(int argc, char **argv)
gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
- if (!gold || !gold_n || !bufs || !caps) goto cleanup;
+ if (!gold || !gold_n || !bufs || !caps)
+ goto cleanup;
int tokens_per_pass = 0;
for (int i = 0; i < n_texts; i++) {
gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
- if (!gold[i] || gold_n[i] <= 0) goto cleanup;
+ if (!gold[i] || gold_n[i] <= 0)
+ goto cleanup;
tokens_per_pass += gold_n[i];
int n_ws = 0;
- int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
- &n_ws);
+ int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws);
if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
free(ids_ws);
goto cleanup;
@@ -74,12 +75,13 @@ int main(int argc, char **argv)
free(ids_ws);
caps[i] = (int)(strlen(texts[i]) * 2 + 8);
- if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
+ if (caps[i] < gold_n[i])
+ caps[i] = gold_n[i];
bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
- if (!bufs[i]) goto cleanup;
+ if (!bufs[i])
+ goto cleanup;
int n_into = 0;
- if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
- &n_into) != 0 ||
+ if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 ||
!same_ids(gold[i], gold_n[i], bufs[i], n_into))
goto cleanup;
}
@@ -105,8 +107,7 @@ int main(int argc, char **argv)
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
- &n);
+ int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n);
if (!ids || n != gold_n[i]) {
free(ids);
goto cleanup;
@@ -122,30 +123,30 @@ int main(int argc, char **argv)
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
- caps[i], &n);
- if (rc != 0 || n != gold_n[i]) goto cleanup;
+ int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n);
+ if (rc != 0 || n != gold_n[i])
+ goto cleanup;
into_tokens += n;
}
}
double into_ms = now_ms() - t0;
- printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs,
- tokens_per_pass);
+ printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, tokens_per_pass);
printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms,
(double)old_tokens / (old_ms / 1000.0));
- printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n",
- ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
- printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n",
- into_ms, (double)into_tokens / (into_ms / 1000.0),
- old_ms / into_ms);
+ printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", ws_ms,
+ (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
+ printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", into_ms,
+ (double)into_tokens / (into_ms / 1000.0), old_ms / into_ms);
status = 0;
cleanup:
for (int i = 0; i < n_texts; i++) {
- if (gold) free(gold[i]);
- if (bufs) free(bufs[i]);
+ if (gold)
+ free(gold[i]);
+ if (bufs)
+ free(bufs[i]);
}
free(gold);
free(gold_n);
diff --git a/embed.c b/embed.c
index bd72846..d9e7936 100644
--- a/embed.c
+++ b/embed.c
@@ -31,32 +31,32 @@ struct embed_late_workspace {
int embed_verbose = 0;
int qwen_verbose = 0;
-#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
-#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
-#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
+#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
+#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
+#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
#define EMBED_WORKSPACE_SEQ_GRANULARITY 16
/* ========================================================================
* Size helpers
* ======================================================================== */
-static int mul_size(size_t a, size_t b, size_t *out)
-{
- if (a != 0 && b > SIZE_MAX / a) return -1;
+static int mul_size(size_t a, size_t b, size_t *out) {
+ if (a != 0 && b > SIZE_MAX / a)
+ return -1;
*out = a * b;
return 0;
}
-static int add_size(size_t a, size_t b, size_t *out)
-{
- if (b > SIZE_MAX - a) return -1;
+static int add_size(size_t a, size_t b, size_t *out) {
+ if (b > SIZE_MAX - a)
+ return -1;
*out = a + b;
return 0;
}
-static int grow_cap(int current, int needed, int *out)
-{
- if (needed <= 0) return -1;
+static int grow_cap(int current, int needed, int *out) {
+ if (needed <= 0)
+ return -1;
int cap = needed;
if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP)
@@ -65,7 +65,8 @@ static int grow_cap(int current, int needed, int *out)
int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY;
if (rem != 0) {
int add = EMBED_WORKSPACE_SEQ_GRANULARITY - rem;
- if (cap > INT_MAX - add) return -1;
+ if (cap > INT_MAX - add)
+ return -1;
cap += add;
}
@@ -76,54 +77,57 @@ static int grow_cap(int current, int needed, int *out)
return 0;
}
-static int realloc_floats(float **ptr, size_t count)
-{
+static int realloc_floats(float **ptr, size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0) return -1;
+ if (mul_size(count, sizeof(float), &bytes) != 0)
+ return -1;
void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0) return -1;
+ if (!p && bytes != 0)
+ return -1;
*ptr = (float *)p;
return 0;
}
-static float *malloc_floats(size_t count)
-{
+static float *malloc_floats(size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0) return NULL;
+ if (mul_size(count, sizeof(float), &bytes) != 0)
+ return NULL;
return (float *)malloc(bytes);
}
-static int model_dir_has_file(const char *model_dir, const char *rel)
-{
+static int model_dir_has_file(const char *model_dir, const char *rel) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path)) return 0;
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
FILE *f = fopen(path, "rb");
- if (!f) return 0;
+ if (!f)
+ return 0;
fclose(f);
return 1;
}
-static int model_dir_has_late_projection(const char *model_dir)
-{
+static int model_dir_has_late_projection(const char *model_dir) {
return model_dir_has_file(model_dir, "1_Dense/config.json") &&
model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static int realloc_floats_2d(float **ptr, int rows, int cols)
-{
- if (rows < 0 || cols < 0) return -1;
+static int realloc_floats_2d(float **ptr, int rows, int cols) {
+ if (rows < 0 || cols < 0)
+ return -1;
size_t count;
- if (mul_size((size_t)rows, (size_t)cols, &count) != 0) return -1;
+ if (mul_size((size_t)rows, (size_t)cols, &count) != 0)
+ return -1;
return realloc_floats(ptr, count);
}
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens)
-{
- if (!ws || !ws->model || n_tokens <= 0) return -1;
- if (n_tokens <= ws->states_seq_cap) return 0;
+static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
+ if (!ws || !ws->model || n_tokens <= 0)
+ return -1;
+ if (n_tokens <= ws->states_seq_cap)
+ return 0;
int hidden = ws->model->base->config.hidden_size;
if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
return -1;
@@ -131,13 +135,14 @@ static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens)
return 0;
}
-static int realloc_ints(int **ptr, size_t count)
-{
+static int realloc_ints(int **ptr, size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(int), &bytes) != 0) return -1;
+ if (mul_size(count, sizeof(int), &bytes) != 0)
+ return -1;
void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0) return -1;
+ if (!p && bytes != 0)
+ return -1;
*ptr = (int *)p;
return 0;
}
@@ -146,15 +151,14 @@ static int realloc_ints(int **ptr, size_t count)
* Minimal JSON helpers for config.json parsing
* ======================================================================== */
-static const char *skip_ws(const char *p)
-{
- while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+static const char *skip_ws(const char *p) {
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
+ p++;
return p;
}
/* Find "key": <value> in a flat JSON object. Returns pointer to value start. */
-static const char *json_find_key(const char *json, const char *key)
-{
+static const char *json_find_key(const char *json, const char *key) {
size_t klen = strlen(key);
const char *p = json;
while ((p = strstr(p, key)) != NULL) {
@@ -163,7 +167,8 @@ static const char *json_find_key(const char *json, const char *key)
const char *after = p + klen;
if (*after == '"') {
after = skip_ws(after + 1);
- if (*after == ':') return skip_ws(after + 1);
+ if (*after == ':')
+ return skip_ws(after + 1);
}
}
p += klen;
@@ -171,32 +176,30 @@ static const char *json_find_key(const char *json, const char *key)
return NULL;
}
-static int json_get_int(const char *json, const char *key, int fallback)
-{
+static int json_get_int(const char *json, const char *key, int fallback) {
const char *v = json_find_key(json, key);
- if (!v) return fallback;
+ if (!v)
+ return fallback;
return atoi(v);
}
-static double json_get_double(const char *json, const char *key, double fallback)
-{
+static double json_get_double(const char *json, const char *key, double fallback) {
const char *v = json_find_key(json, key);
- if (!v) return fallback;
+ if (!v)
+ return fallback;
return atof(v);
}
-static int json_string_equals(const char *json, const char *key,
- const char *expected)
-{
+static int json_string_equals(const char *json, const char *key, const char *expected) {
const char *v = json_find_key(json, key);
- if (!v || *v != '"') return 0;
+ if (!v || *v != '"')
+ return 0;
v++;
size_t len = strlen(expected);
return strncmp(v, expected, len) == 0 && v[len] == '"';
}
-static int parse_config(embed_config_t *cfg, const char *model_dir)
-{
+static int parse_config(embed_config_t *cfg, const char *model_dir) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
@@ -221,22 +224,24 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
}
char *buf = (char *)malloc((size_t)sz + 1);
if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f); free(buf); return -1;
+ fclose(f);
+ free(buf);
+ return -1;
}
fclose(f);
buf[sz] = '\0';
- cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
- cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
- cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
- cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
- cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
+ cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
+ cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
+ cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
+ cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
+ cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0);
- cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
- cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
- cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
- cfg->pooling_mode = EMBED_POOL_MEAN;
+ cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
+ cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
+ cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
+ cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
+ cfg->pooling_mode = EMBED_POOL_MEAN;
cfg->normalize_embeddings = 0;
cfg->append_terminal_token = 0;
@@ -249,45 +254,40 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->q_dim = 0;
cfg->kv_dim = 0;
- if (cfg->n_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim)
cfg->q_dim = cfg->n_heads * cfg->head_dim;
- if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
free(buf);
/* Sanity checks */
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 ||
- cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
- cfg->head_dim <= 0 || cfg->intermediate_size <= 0 ||
- cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
- (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
- !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
- fprintf(stderr, "embed_model_load: invalid config in %s "
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
+ cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
+ cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
+ cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
+ cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ fprintf(stderr,
+ "embed_model_load: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
- path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
- cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size);
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
+ cfg->intermediate_size);
return -1;
}
if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n",
- cfg->n_layers, EMBED_MAX_LAYERS);
+ fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n", cfg->n_layers,
+ EMBED_MAX_LAYERS);
return -1;
}
if (embed_verbose >= 1)
- fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, "
+ fprintf(stderr,
+ "config: hidden=%d, layers=%d, heads=%d/%d, "
"inter=%d, head_dim=%d, attention=%s, pooling=%s\n",
- cfg->hidden_size, cfg->n_layers,
- cfg->n_heads, cfg->n_kv_heads,
+ cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
cfg->intermediate_size, cfg->head_dim,
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN
- ? "last-token" : "mean");
+ cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
+ cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean");
return 0;
}
@@ -296,48 +296,57 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
* Weight loading (direct mmap pointers into safetensors)
* ======================================================================== */
-static size_t dtype_size(safetensor_dtype_t dtype)
-{
+static size_t dtype_size(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return sizeof(float);
- case DTYPE_BF16: return sizeof(uint16_t);
- default: return 0;
+ case DTYPE_F32:
+ return sizeof(float);
+ case DTYPE_BF16:
+ return sizeof(uint16_t);
+ default:
+ return 0;
}
}
-static const char *dtype_name(safetensor_dtype_t dtype)
-{
+static const char *dtype_name(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return "F32";
- case DTYPE_F16: return "F16";
- case DTYPE_BF16: return "BF16";
- case DTYPE_I32: return "I32";
- case DTYPE_I64: return "I64";
- case DTYPE_BOOL: return "BOOL";
- default: return "UNKNOWN";
+ case DTYPE_F32:
+ return "F32";
+ case DTYPE_F16:
+ return "F16";
+ case DTYPE_BF16:
+ return "BF16";
+ case DTYPE_I32:
+ return "I32";
+ case DTYPE_I64:
+ return "I64";
+ case DTYPE_BOOL:
+ return "BOOL";
+ default:
+ return "UNKNOWN";
}
}
static int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t, const char *name,
- const int64_t *shape, int ndim,
- int bf16_ok)
-{
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim,
+ int bf16_ok) {
if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n",
- name, dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
+ fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
+ dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n",
- name, t->ndim, ndim);
+ fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
size_t numel = 1;
for (int i = 0; i < ndim; i++) {
if (t->shape[i] != shape[i]) {
- fprintf(stderr, "embed: bad shape for %s at dim %d: "
+ fprintf(stderr,
+ "embed: bad shape for %s at dim %d: "
"got %lld, expected %lld\n",
name, i, (long long)t->shape[i], (long long)shape[i]);
return 0;
@@ -351,8 +360,7 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
size_t elem_size = dtype_size(t->dtype);
if (elem_size == 0) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s\n",
- name, dtype_name(t->dtype));
+ fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype));
return 0;
}
@@ -362,14 +370,13 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
return 0;
}
if (t->data_size != bytes) {
- fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n",
- name, t->data_size, bytes);
+ fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
+ bytes);
return 0;
}
size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size ||
- t->data_offset > sf->file_size - data_start ||
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
fprintf(stderr, "embed: tensor data out of bounds for %s\n", name);
return 0;
@@ -377,13 +384,11 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms,
- const char *name,
- const int64_t *shape, int ndim)
-{
- embed_weight_ref_t ref = { DTYPE_UNKNOWN, NULL };
+static embed_weight_ref_t
+load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
+ embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL};
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "embed: tensor not found: %s\n", name);
return ref;
@@ -396,11 +401,10 @@ static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms,
return ref;
}
-static float *load_norm_f32(multi_safetensors_t *ms, const char *name,
- const int64_t *shape, int ndim)
-{
+static float *
+load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "embed: tensor not found: %s\n", name);
return NULL;
@@ -410,17 +414,14 @@ static float *load_norm_f32(multi_safetensors_t *ms, const char *name,
return safetensors_get_f32(sf, t);
}
-static void bf16_row_to_f32(float *dst, const uint16_t *src, int n)
-{
+static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) {
for (int i = 0; i < n; i++) {
uint32_t u = ((uint32_t)src[i]) << 16;
memcpy(dst + i, &u, sizeof(float));
}
}
-static void copy_weight_row(float *dst, const embed_weight_ref_t *w,
- size_t row, int row_width)
-{
+static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) {
if (w->dtype == DTYPE_F32) {
const float *src = (const float *)w->data + row * (size_t)row_width;
memcpy(dst, src, (size_t)row_width * sizeof(float));
@@ -430,23 +431,26 @@ static void copy_weight_row(float *dst, const embed_weight_ref_t *w,
}
}
-static int ensure_bf16_widen(embed_workspace_t *ws, size_t count)
-{
- if (ws->bf16_widen_count >= count) return 0;
+static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
+ if (ws->bf16_widen_count >= count)
+ return 0;
float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float));
- if (!p) return -1;
+ if (!p)
+ return -1;
ws->bf16_widen = p;
ws->bf16_widen_count = count;
return 0;
}
-static void linear_nobias_weight(embed_workspace_t *ws, float *y,
- const float *x, const embed_weight_ref_t *w,
- int seq_len, int in_dim, int out_dim)
-{
+static void linear_nobias_weight(embed_workspace_t *ws,
+ float *y,
+ const float *x,
+ const embed_weight_ref_t *w,
+ int seq_len,
+ int in_dim,
+ int out_dim) {
if (w->dtype == DTYPE_F32) {
- qwen_linear_nobias(y, x, (const float *)w->data,
- seq_len, in_dim, out_dim);
+ qwen_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
return;
}
if (seq_len > 16) {
@@ -454,31 +458,31 @@ static void linear_nobias_weight(embed_workspace_t *ws, float *y,
* BLAS-backed F32 path; below that the fused BF16 row kernels win. */
size_t count = (size_t)out_dim * (size_t)in_dim;
if (ensure_bf16_widen(ws, count) == 0) {
- qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data,
- count);
+ qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count);
qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
return;
}
}
- qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data,
- seq_len, in_dim, out_dim);
+ qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim);
}
-static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k,
- float *v, const float *x,
+static void linear_qkv_weight(embed_workspace_t *ws,
+ float *q,
+ float *k,
+ float *v,
+ const float *x,
const embed_weight_ref_t *wq,
const embed_weight_ref_t *wk,
const embed_weight_ref_t *wv,
- int seq_len, int in_dim, int q_dim, int kv_dim)
-{
- if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 &&
- wv->dtype == DTYPE_BF16 &&
+ int seq_len,
+ int in_dim,
+ int q_dim,
+ int kv_dim) {
+ if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) {
- qwen_linear_nobias_bf16_qkv(q, k, v, x,
- (const uint16_t *)wq->data,
- (const uint16_t *)wk->data,
- (const uint16_t *)wv->data,
- seq_len, in_dim, q_dim, kv_dim);
+ qwen_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data,
+ (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len,
+ in_dim, q_dim, kv_dim);
return;
}
@@ -487,18 +491,20 @@ static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k,
linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim);
}
-static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b,
+static void linear_pair_weight(embed_workspace_t *ws,
+ float *a,
+ float *b,
const float *x,
const embed_weight_ref_t *wa,
const embed_weight_ref_t *wb,
- int seq_len, int in_dim, int a_dim, int b_dim)
-{
+ int seq_len,
+ int in_dim,
+ int a_dim,
+ int b_dim) {
if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) {
- qwen_linear_nobias_bf16_pair(a, b, x,
- (const uint16_t *)wa->data,
- (const uint16_t *)wb->data,
- seq_len, in_dim, a_dim, b_dim);
+ qwen_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data,
+ (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
return;
}
@@ -510,26 +516,29 @@ static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b,
* Working buffer management
* ======================================================================== */
-static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq)
-{
- if (seq <= ws->buf_seq_cap) return 0;
+static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) {
+ if (seq <= ws->buf_seq_cap)
+ return 0;
int cap;
- if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) return -1;
+ if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
+ return -1;
-#define R(ptr, n) do { \
- if (realloc_floats_2d(&(ptr), cap, (n)) != 0) return -1; \
-} while (0)
+#define R(ptr, n) \
+ do { \
+ if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
+ return -1; \
+ } while (0)
- R(ws->x, c->hidden_size);
- R(ws->x_norm, c->hidden_size);
- R(ws->q, c->q_dim);
- R(ws->k, c->kv_dim);
- R(ws->v, c->kv_dim);
+ R(ws->x, c->hidden_size);
+ R(ws->x_norm, c->hidden_size);
+ R(ws->q, c->q_dim);
+ R(ws->k, c->kv_dim);
+ R(ws->v, c->kv_dim);
R(ws->attn_out, c->q_dim);
R(ws->proj_out, c->hidden_size);
R(ws->ffn_gate, c->intermediate_size);
- R(ws->ffn_up, c->intermediate_size);
+ R(ws->ffn_up, c->intermediate_size);
#undef R
@@ -537,15 +546,14 @@ static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int se
return 0;
}
-static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets,
- int batch)
-{
- size_t bytes =
- qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (bytes <= ws->attn_scores_bytes) return;
+static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) {
+ size_t bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes)
+ return;
void *p = realloc(ws->attn_scores, bytes);
- if (!p) return;
+ if (!p)
+ return;
ws->attn_scores = (float *)p;
ws->attn_scores_bytes = bytes;
}
@@ -554,20 +562,20 @@ static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets,
* RoPE cache
* ======================================================================== */
-static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
- int n_pos)
-{
- if (n_pos <= ws->rope_cache_cap) return 0;
+static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) {
+ if (n_pos <= ws->rope_cache_cap)
+ return 0;
int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512;
while (cap < n_pos) {
- if (cap > INT_MAX / 2) return -1;
+ if (cap > INT_MAX / 2)
+ return -1;
cap *= 2;
}
- int head_dim = cfg->head_dim;
- int half = head_dim / 2;
- float theta = cfg->rope_theta;
+ int head_dim = cfg->head_dim;
+ int half = head_dim / 2;
+ float theta = cfg->rope_theta;
size_t count;
if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0)
@@ -585,8 +593,10 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
float angle = fp * inv;
float cv = cosf(angle), sv = sinf(angle);
- c[d] = cv; c[half + d] = cv;
- s[d] = sv; s[half + d] = sv;
+ c[d] = cv;
+ c[half + d] = cv;
+ s[d] = sv;
+ s[half + d] = sv;
}
}
@@ -594,12 +604,13 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
return 0;
}
-static int ensure_offsets(embed_workspace_t *ws, int batch)
-{
- if (!ws || batch <= 0 || batch == INT_MAX) return -1;
+static int ensure_offsets(embed_workspace_t *ws, int batch) {
+ if (!ws || batch <= 0 || batch == INT_MAX)
+ return -1;
int needed = batch + 1;
- if (needed <= ws->offsets_cap) return 0;
+ if (needed <= ws->offsets_cap)
+ return 0;
if (realloc_ints(&ws->offsets, (size_t)needed) != 0)
return -1;
@@ -611,19 +622,18 @@ static int ensure_offsets(embed_workspace_t *ws, int batch)
* Model / workspace load and free
* ======================================================================== */
-static embed_model_t *model_load_range_ex(const char *model_dir,
- int layer_start, int layer_end,
- int allow_late)
-{
+static embed_model_t *
+model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
if (!allow_late && model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_model_load: late-interaction models require "
- "token-level MaxSim support and are not valid pooled "
- "embedding models yet\n");
+ "token-level MaxSim support and are not valid pooled "
+ "embedding models yet\n");
return NULL;
}
embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t));
- if (!model) return NULL;
+ if (!model)
+ return NULL;
/* Parse config.json */
if (parse_config(&model->config, model_dir) != 0) {
@@ -634,19 +644,19 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
const embed_config_t *cfg = &model->config;
if (layer_end == -1)
layer_end = cfg->n_layers;
- if (layer_start < 0 || layer_start >= layer_end ||
- layer_end > cfg->n_layers) {
- fprintf(stderr, "embed_model_load: invalid layer range [%d, %d) "
- "for %d-layer model\n", layer_start, layer_end, cfg->n_layers);
+ if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) {
+ fprintf(stderr,
+ "embed_model_load: invalid layer range [%d, %d) "
+ "for %d-layer model\n",
+ layer_start, layer_end, cfg->n_layers);
free(model);
return NULL;
}
model->layer_start = layer_start;
model->layer_end = layer_end;
- model->attention =
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? qwen_causal_gqa_attention_packed_with_scratch
- : qwen_bidirectional_gqa_attention_packed_with_scratch;
+ model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL
+ ? qwen_causal_gqa_attention_packed_with_scratch
+ : qwen_bidirectional_gqa_attention_packed_with_scratch;
/* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
@@ -663,57 +673,53 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
/* Only the first stage needs token embeddings. */
embed_weights_t *w = &model->weights;
if (layer_start == 0) {
- const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size };
- w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight",
- embed_shape, 2);
- if (!w->embed_tokens.data) goto fail;
+ const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size};
+ w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", embed_shape, 2);
+ if (!w->embed_tokens.data)
+ goto fail;
}
/* Allocate layer array */
w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
- if (!w->layers) goto fail;
+ if (!w->layers)
+ goto fail;
/* Per-layer weights */
char name[256];
for (int i = layer_start; i < layer_end; i++) {
embed_layer_t *l = &w->layers[i];
-#define LOAD_NORM(field, fmt, d0) do { \
- const int64_t expect[1] = { (d0) }; \
- snprintf(name, sizeof(name), fmt, i); \
- l->field = load_norm_f32(ms, name, expect, 1); \
- if (!l->field) goto fail; \
-} while (0)
+#define LOAD_NORM(field, fmt, d0) \
+ do { \
+ const int64_t expect[1] = {(d0)}; \
+ snprintf(name, sizeof(name), fmt, i); \
+ l->field = load_norm_f32(ms, name, expect, 1); \
+ if (!l->field) \
+ goto fail; \
+ } while (0)
-#define LOAD2(field, fmt, d0, d1) do { \
- const int64_t expect[2] = { (d0), (d1) }; \
- snprintf(name, sizeof(name), fmt, i); \
- l->field = load_weight_direct(ms, name, expect, 2); \
- if (!l->field.data) goto fail; \
-} while (0)
+#define LOAD2(field, fmt, d0, d1) \
+ do { \
+ const int64_t expect[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), fmt, i); \
+ l->field = load_weight_direct(ms, name, expect, 2); \
+ if (!l->field.data) \
+ goto fail; \
+ } while (0)
- LOAD2(wq, "layers.%d.self_attn.q_proj.weight",
- cfg->q_dim, cfg->hidden_size);
- LOAD2(wk, "layers.%d.self_attn.k_proj.weight",
- cfg->kv_dim, cfg->hidden_size);
- LOAD2(wv, "layers.%d.self_attn.v_proj.weight",
- cfg->kv_dim, cfg->hidden_size);
- LOAD2(wo, "layers.%d.self_attn.o_proj.weight",
- cfg->hidden_size, cfg->q_dim);
- LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight",
- cfg->head_dim);
- LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight",
- cfg->head_dim);
- LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight",
- cfg->hidden_size);
- LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight",
- cfg->hidden_size);
- LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight",
- cfg->intermediate_size, cfg->hidden_size);
- LOAD2(up_proj, "layers.%d.mlp.up_proj.weight",
- cfg->intermediate_size, cfg->hidden_size);
- LOAD2(down_proj, "layers.%d.mlp.down_proj.weight",
- cfg->hidden_size, cfg->intermediate_size);
+ LOAD2(wq, "layers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size);
+ LOAD2(wk, "layers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size);
+ LOAD2(wv, "layers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size);
+ LOAD2(wo, "layers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim);
+ LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", cfg->head_dim);
+ LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", cfg->head_dim);
+ LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", cfg->hidden_size);
+ LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", cfg->hidden_size);
+ LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", cfg->intermediate_size,
+ cfg->hidden_size);
+ LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size);
+ LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", cfg->hidden_size,
+ cfg->intermediate_size);
#undef LOAD2
#undef LOAD_NORM
@@ -724,13 +730,15 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
/* Only the final stage applies the output RMSNorm. */
if (layer_end == cfg->n_layers) {
- const int64_t norm_shape[1] = { cfg->hidden_size };
+ const int64_t norm_shape[1] = {cfg->hidden_size};
w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1);
- if (!w->norm) goto fail;
+ if (!w->norm)
+ goto fail;
}
if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: layers [%d, %d) loaded "
+ fprintf(stderr,
+ "embed_model_load: layers [%d, %d) loaded "
"(%d-dim embeddings)\n",
layer_start, layer_end, cfg->hidden_size);
@@ -741,14 +749,13 @@ fail:
return NULL;
}
-embed_model_t *embed_model_load(const char *model_dir)
-{
+embed_model_t *embed_model_load(const char *model_dir) {
return model_load_range_ex(model_dir, 0, -1, 0);
}
-void embed_model_free(embed_model_t *model)
-{
- if (!model) return;
+void embed_model_free(embed_model_t *model) {
+ if (!model)
+ return;
if (model->weights.layers) {
for (int i = 0; i < model->config.n_layers; i++) {
embed_layer_t *l = &model->weights.layers[i];
@@ -765,38 +772,43 @@ void embed_model_free(embed_model_t *model)
free(model);
}
-embed_workspace_t *embed_workspace_new(const embed_model_t *model)
-{
- if (!model) return NULL;
+embed_workspace_t *embed_workspace_new(const embed_model_t *model) {
+ if (!model)
+ return NULL;
embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t));
- if (!ws) return NULL;
+ if (!ws)
+ return NULL;
ws->model = model;
return ws;
}
-void embed_workspace_free(embed_workspace_t *ws)
-{
- if (!ws) return;
- free(ws->x); free(ws->x_norm);
- free(ws->q); free(ws->k); free(ws->v);
- free(ws->attn_out); free(ws->proj_out);
- free(ws->ffn_gate); free(ws->ffn_up);
+void embed_workspace_free(embed_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->x);
+ free(ws->x_norm);
+ free(ws->q);
+ free(ws->k);
+ free(ws->v);
+ free(ws->attn_out);
+ free(ws->proj_out);
+ free(ws->ffn_gate);
+ free(ws->ffn_up);
free(ws->attn_scores);
free(ws->offsets);
- free(ws->rope_cos); free(ws->rope_sin);
+ free(ws->rope_cos);
+ free(ws->rope_sin);
free(ws->bf16_widen);
free(ws);
}
-const embed_config_t *embed_model_config(const embed_model_t *model)
-{
+const embed_config_t *embed_model_config(const embed_model_t *model) {
return model ? &model->config : NULL;
}
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf,
- const char *name)
-{
- if (!sf || !name) return NULL;
+static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
+ if (!sf || !name)
+ return NULL;
for (int i = 0; i < sf->num_tensors; i++) {
if (strcmp(sf->tensors[i].name, name) == 0)
return &sf->tensors[i];
@@ -804,22 +816,22 @@ static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf,
return NULL;
}
-embed_late_model_t *embed_late_model_load(const char *model_dir)
-{
+embed_late_model_t *embed_late_model_load(const char *model_dir) {
if (!model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
return NULL;
}
embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late) return NULL;
+ if (!late)
+ return NULL;
late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base) goto fail;
+ if (!late->base)
+ goto fail;
char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors",
- model_dir);
+ int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
fprintf(stderr, "embed_late_model_load: model path too long\n");
goto fail;
@@ -831,8 +843,7 @@ embed_late_model_t *embed_late_model_load(const char *model_dir)
goto fail;
}
- const safetensor_t *t = find_single_safetensor(late->dense_sf,
- "linear.weight");
+ const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
if (!t) {
fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
goto fail;
@@ -842,9 +853,8 @@ embed_late_model_t *embed_late_model_load(const char *model_dir)
goto fail;
}
- int64_t shape[2] = { t->shape[0], late->base->config.hidden_size };
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight",
- shape, 2, 1))
+ int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
+ if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
goto fail;
late->projection.dtype = t->dtype;
@@ -857,19 +867,20 @@ fail:
return NULL;
}
-void embed_late_model_free(embed_late_model_t *model)
-{
- if (!model) return;
+void embed_late_model_free(embed_late_model_t *model) {
+ if (!model)
+ return;
embed_model_free(model->base);
safetensors_close(model->dense_sf);
free(model);
}
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model)
-{
- if (!model || !model->base) return NULL;
+embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
+ if (!model || !model->base)
+ return NULL;
embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws) return NULL;
+ if (!ws)
+ return NULL;
ws->model = model;
ws->base_ws = embed_workspace_new(model->base);
if (!ws->base_ws) {
@@ -879,43 +890,41 @@ embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model
return ws;
}
-void embed_late_workspace_free(embed_late_workspace_t *ws)
-{
- if (!ws) return;
+void embed_late_workspace_free(embed_late_workspace_t *ws) {
+ if (!ws)
+ return;
embed_workspace_free(ws->base_ws);
free(ws->states);
free(ws);
}
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model)
-{
+const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
return model && model->base ? &model->base->config : NULL;
}
-int embed_late_model_token_dim(const embed_late_model_t *model)
-{
+int embed_late_model_token_dim(const embed_late_model_t *model) {
return model ? model->token_dim : 0;
}
int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors)
-{
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 ||
- !out_vectors || model->token_dim <= 0)
+ embed_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
+ model->token_dim <= 0)
return -1;
int hidden = model->base->config.hidden_size;
int dim = model->token_dim;
if (ensure_late_state_buffer(ws, n_tokens) != 0)
return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids,
- n_tokens, ws->states) != 0)
+ if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
return -1;
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states,
- &model->projection, n_tokens, hidden, dim);
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
+ dim);
if (normalize) {
for (int i = 0; i < n_tokens; i++) {
if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
@@ -925,12 +934,12 @@ int embed_late_model_encode_tokens(const embed_late_model_t *model,
return 0;
}
-float embed_late_maxsim(const float *query_vectors, int query_tokens,
- const float *doc_vectors, int doc_tokens,
- int dim)
-{
- if (!query_vectors || !doc_vectors || query_tokens <= 0 ||
- doc_tokens <= 0 || dim <= 0)
+float embed_late_maxsim(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ int doc_tokens,
+ int dim) {
+ if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
return 0.0f;
float score = 0.0f;
@@ -940,20 +949,23 @@ float embed_late_maxsim(const float *query_vectors, int query_tokens,
for (int di = 0; di < doc_tokens; di++) {
const float *d = doc_vectors + (size_t)di * dim;
float dot = qwen_dot_f32(q, d, dim);
- if (dot > best) best = dot;
+ if (dot > best)
+ best = dot;
}
score += best;
}
return score;
}
-int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets, int docs,
- int dim, float *scores)
-{
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores ||
- query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
+int embed_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores) {
+ if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
+ docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
return -1;
for (int i = 0; i < docs; i++) {
@@ -969,12 +981,12 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
int total = doc_offsets[docs];
int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens *
- (size_t)(total < budget ? total : budget);
+ size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
int largest = 0;
for (int i = 0; i < docs; i++) {
int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest) largest = len;
+ if (len > largest)
+ largest = len;
}
if ((size_t)query_tokens * (size_t)largest > cap)
cap = (size_t)query_tokens * (size_t)largest;
@@ -985,16 +997,13 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
int first = doc;
int start = doc_offsets[first];
doc++;
- while (doc < docs &&
- doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <=
- (int)(cap / (size_t)query_tokens))
+ while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
+ doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
doc++;
int group_tokens = doc_offsets[doc] - start;
- qwen_matmul_t(sim, query_vectors,
- doc_vectors + (size_t)start * dim,
- query_tokens, dim, group_tokens);
+ qwen_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
+ group_tokens);
for (int i = first; i < doc; i++) {
int s0 = doc_offsets[i] - start;
@@ -1004,7 +1013,8 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
const float *row = sim + (size_t)qi * group_tokens;
float best = row[s0];
for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best) best = row[t];
+ if (row[t] > best)
+ best = row[t];
score += best;
}
scores[i] = score;
@@ -1020,63 +1030,58 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
int start = doc_offsets[i];
int end = doc_offsets[i + 1];
scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim,
- end - start, dim);
+ doc_vectors + (size_t)start * dim, end - start, dim);
}
return 0;
}
-size_t embed_workspace_nbytes(const embed_workspace_t *ws)
-{
- if (!ws) return 0;
+size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
+ if (!ws)
+ return 0;
size_t total = sizeof(*ws);
const embed_model_t *model = ws->model;
- if (!model) return total;
+ if (!model)
+ return total;
const embed_config_t *c = &model->config;
size_t row_floats = 0;
const int dims[] = {
- c->hidden_size, /* x */
- c->hidden_size, /* x_norm */
- c->q_dim, /* q */
- c->kv_dim, /* k */
- c->kv_dim, /* v */
- c->q_dim, /* attn_out */
- c->hidden_size, /* proj_out */
- c->intermediate_size, /* ffn_gate */
- c->intermediate_size, /* ffn_up */
+ c->hidden_size, /* x */
+ c->hidden_size, /* x_norm */
+ c->q_dim, /* q */
+ c->kv_dim, /* k */
+ c->kv_dim, /* v */
+ c->q_dim, /* attn_out */
+ c->hidden_size, /* proj_out */
+ c->intermediate_size, /* ffn_gate */
+ c->intermediate_size, /* ffn_up */
};
for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) {
- if (dims[i] < 0 ||
- add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
+ if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
return SIZE_MAX;
}
size_t buf_floats = 0;
- if (ws->buf_seq_cap > 0 &&
- mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
+ if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
return SIZE_MAX;
size_t rope_floats = 0;
if (ws->rope_cache_cap > 0) {
size_t rope_rows = 0;
- if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim,
- &rope_rows) != 0 ||
+ if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 ||
mul_size(rope_rows, 2, &rope_floats) != 0)
return SIZE_MAX;
}
size_t all_floats = 0, bytes = 0;
if (add_size(buf_floats, rope_floats, &all_floats) != 0 ||
- mul_size(all_floats, sizeof(float), &bytes) != 0 ||
- add_size(total, bytes, &total) != 0)
+ mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0)
return SIZE_MAX;
size_t offset_bytes = 0;
- if (ws->offsets_cap > 0 &&
- mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
+ if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
return SIZE_MAX;
if (add_size(total, offset_bytes, &total) != 0)
return SIZE_MAX;
@@ -1097,9 +1102,12 @@ size_t embed_workspace_nbytes(const embed_workspace_t *ws)
* Packed forward pass
* ======================================================================== */
-static int build_offsets(const embed_input_t *inputs, int batch, int require_ids,
- int *offsets, int *total_out, int *max_out)
-{
+static int build_offsets(const embed_input_t *inputs,
+ int batch,
+ int require_ids,
+ int *offsets,
+ int *total_out,
+ int *max_out) {
if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
return -1;
@@ -1124,52 +1132,54 @@ static int build_offsets(const embed_input_t *inputs, int batch, int require_ids
static int forward_packed_slice_inplace(const embed_model_t *model,
embed_workspace_t *ws,
const embed_input_t *inputs,
- int batch, const int *offsets,
- int total_seq, int max_seq,
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm)
-{
- if (!model || !ws || ws->model != model || !inputs ||
- batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0)
+ int layer_start,
+ int layer_end,
+ int apply_final_norm) {
+ if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
+ total_seq <= 0 || max_seq <= 0)
return -1;
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
- if (layer_start < 0 || layer_start > layer_end ||
- layer_end > cfg->n_layers ||
+ const embed_config_t *cfg = &model->config;
+ const embed_weights_t *w = &model->weights;
+ if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers ||
layer_start < model->layer_start || layer_end > model->layer_end ||
- (layer_start == 0 && input_states) ||
- (layer_start != 0 && !input_states) ||
+ (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) ||
(layer_start == 0 && !w->embed_tokens.data) ||
(apply_final_norm && (layer_end != cfg->n_layers || !w->norm)))
return -1;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
int n_kv_heads = cfg->n_kv_heads;
- int head_dim = cfg->head_dim;
- int q_dim = cfg->q_dim;
- int kv_dim = cfg->kv_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->rms_norm_eps;
+ int head_dim = cfg->head_dim;
+ int q_dim = cfg->q_dim;
+ int kv_dim = cfg->kv_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->rms_norm_eps;
- if (ensure_buffers(ws, cfg, total_seq) != 0) return -1;
- if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1;
+ if (ensure_buffers(ws, cfg, total_seq) != 0)
+ return -1;
+ if (ensure_rope_cache(ws, cfg, max_seq) != 0)
+ return -1;
ensure_attention_scores(ws, offsets, batch);
const float *rope_cos = ws->rope_cos;
const float *rope_sin = ws->rope_sin;
- float *x = ws->x;
- float *x_norm = ws->x_norm;
- float *q_buf = ws->q;
- float *k_buf = ws->k;
- float *v_buf = ws->v;
+ float *x = ws->x;
+ float *x_norm = ws->x_norm;
+ float *q_buf = ws->q;
+ float *k_buf = ws->k;
+ float *v_buf = ws->v;
float *attn_out = ws->attn_out;
float *proj_out = ws->proj_out;
float *ffn_gate = ws->ffn_gate;
- float *ffn_up = ws->ffn_up;
+ float *ffn_up = ws->ffn_up;
/* A first stage performs embedding lookup. Later stages resume from a
* packed hidden-state tensor received from the preceding stage. */
@@ -1179,12 +1189,12 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
for (int i = 0; i < inputs[b].n_tokens; i++) {
int id = inputs[b].ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n",
- id, b, i);
+ fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b,
+ i);
return -1;
}
- copy_weight_row(x + (size_t)(start + i) * hidden,
- &w->embed_tokens, (size_t)id, hidden);
+ copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id,
+ hidden);
}
}
} else {
@@ -1200,35 +1210,31 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm,
- &l->wq, &l->wk, &l->wv,
- total_seq, hidden, q_dim, kv_dim);
+ linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq,
+ hidden, q_dim, kv_dim);
- qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
for (int b = 0; b < batch; b++) {
int start = offsets[b];
int len = offsets[b + 1] - start;
- qwen_apply_rope_neox(q_buf + (size_t)start * q_dim,
- rope_cos, rope_sin, len, n_heads, head_dim);
- qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim,
- rope_cos, rope_sin, len, n_kv_heads, head_dim);
+ qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
+ head_dim);
+ qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len,
+ n_kv_heads, head_dim);
}
- model->attention(
- attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads,
- n_kv_heads, head_dim, scale, ws->attn_scores,
- ws->attn_scores_bytes);
+ model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, ws->attn_scores, ws->attn_scores_bytes);
linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- linear_pair_weight(ws, ffn_gate, ffn_up, x_norm,
- &l->gate_proj, &l->up_proj,
- total_seq, hidden, inter, inter);
+ linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq,
+ hidden, inter, inter);
qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
@@ -1242,21 +1248,20 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
static int forward_packed_inplace(const embed_model_t *model,
embed_workspace_t *ws,
const embed_input_t *inputs,
- int batch, const int *offsets,
- int total_seq, int max_seq,
- int apply_final_norm)
-{
- return forward_packed_slice_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq, NULL, 0,
- model->config.n_layers,
- apply_final_norm);
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
+ int apply_final_norm) {
+ return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL,
+ 0, model->config.n_layers, apply_final_norm);
}
static int pool_embeddings(const embed_model_t *model,
const embed_workspace_t *ws,
- const int *offsets, int batch,
- float *out_embeddings)
-{
+ const int *offsets,
+ int batch,
+ float *out_embeddings) {
const embed_config_t *cfg = &model->config;
int hidden = cfg->hidden_size;
float eps = cfg->rms_norm_eps;
@@ -1297,8 +1302,7 @@ static int pool_embeddings(const embed_model_t *model,
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings &&
- embed_l2_normalize(emb, hidden) != 0)
+ if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0)
return -1;
}
return 0;
@@ -1308,38 +1312,40 @@ static int pool_embeddings(const embed_model_t *model,
* Public forward/embed APIs
* ======================================================================== */
-int embed_model_forward_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_states)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1;
+int embed_model_forward_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states) {
+ if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states)
+ return -1;
size_t count, bytes;
- if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size,
- &count) != 0 ||
+ if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 ||
mul_size(count, sizeof(float), &bytes) != 0)
return -1;
- embed_input_t input = { token_ids, n_tokens };
- int offsets[2] = { 0, n_tokens };
- if (forward_packed_inplace(model, ws, &input, 1, offsets,
- n_tokens, n_tokens, 1) != 0)
+ embed_input_t input = {token_ids, n_tokens};
+ int offsets[2] = {0, n_tokens};
+ if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0)
return -1;
memcpy(out_states, ws->x, bytes);
return 0;
}
-int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings)
-{
- if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 ||
- !out_embeddings)
+int embed_pool_spans(const embed_config_t *cfg,
+ const float *states,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
+ if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings)
return -1;
int hidden = cfg->hidden_size;
- if (hidden <= 0) return -1;
+ if (hidden <= 0)
+ return -1;
for (int s = 0; s < n_spans; s++) {
int start = spans[s].start;
@@ -1365,17 +1371,20 @@ int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_token
}
int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings)
-{
+ embed_workspace_t *ws,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
- if (ensure_offsets(ws, batch) != 0) return -1;
- if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) return -1;
+ if (ensure_offsets(ws, batch) != 0)
+ return -1;
+ if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t))
+ return -1;
embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
- if (!packed_inputs) return -1;
+ if (!packed_inputs)
+ return -1;
int *offsets = ws->offsets;
int total_seq = 0;
int max_seq = 0;
@@ -1383,9 +1392,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans ||
- input->n_spans <= 0 || total_seq > INT_MAX - n_tokens ||
- total_spans > INT_MAX - input->n_spans) {
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) {
free(packed_inputs);
return -1;
}
@@ -1393,12 +1401,12 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
offsets[b] = total_seq;
total_seq += n_tokens;
total_spans += input->n_spans;
- if (n_tokens > max_seq) max_seq = n_tokens;
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
for (int s = 0; s < input->n_spans; s++) {
int start = input->spans[s].start;
int len = input->spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens ||
- len > n_tokens - start) {
+ if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) {
free(packed_inputs);
return -1;
}
@@ -1406,8 +1414,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
}
offsets[batch] = total_seq;
- if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets,
- total_seq, max_seq, 1) != 0) {
+ if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) !=
+ 0) {
free(packed_inputs);
return -1;
}
@@ -1417,39 +1425,43 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
int hidden = model->config.hidden_size;
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
- if (embed_pool_spans(&model->config,
- ws->x + (size_t)offsets[b] * hidden,
- input->input.n_tokens, input->spans, input->n_spans,
- out_embeddings + (size_t)span_offset * hidden) != 0)
+ if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
+ input->input.n_tokens, input->spans, input->n_spans,
+ out_embeddings + (size_t)span_offset * hidden) != 0)
return -1;
span_offset += input->n_spans;
}
return 0;
}
-int embed_model_encode_spans(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings)
-{
+int embed_model_encode_spans(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
embed_context_input_t input = {
- .input = { token_ids, n_tokens },
+ .input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings);
}
-float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0) return NULL;
+float *embed_model_forward(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
+ if (!model || !ws || !token_ids || n_tokens <= 0)
+ return NULL;
size_t n;
if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0)
return NULL;
float *out = malloc_floats(n);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) {
free(out);
@@ -1458,13 +1470,16 @@ float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws,
return out;
}
-int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
- const embed_input_t *inputs, int batch,
- float *out_embeddings)
-{
- if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1;
+int embed_model_encode_batch(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
- if (ensure_offsets(ws, batch) != 0) return -1;
+ if (ensure_offsets(ws, batch) != 0)
+ return -1;
int *offsets = ws->offsets;
int total_seq = 0;
@@ -1472,20 +1487,19 @@ int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0)
return -1;
- int rc = forward_packed_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq, 0);
+ int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0);
if (rc == 0)
rc = pool_embeddings(model, ws, offsets, batch, out_embeddings);
return rc;
}
-int embed_pool_batch(const embed_config_t *cfg, const float *states,
- const int *seq_lengths, int batch,
- float *out_embeddings)
-{
- if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings ||
- cfg->hidden_size <= 0)
+int embed_pool_batch(const embed_config_t *cfg,
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings) {
+ if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0)
return -1;
int offset = 0;
@@ -1496,14 +1510,12 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states,
float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
- const float *row =
- states + (size_t)(offset + len - 1) * cfg->hidden_size;
+ const float *row = states + (size_t)(offset + len - 1) * cfg->hidden_size;
memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
} else {
memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
for (int t = 0; t < len; t++) {
- const float *row =
- states + (size_t)(offset + t) * cfg->hidden_size;
+ const float *row = states + (size_t)(offset + t) * cfg->hidden_size;
for (int d = 0; d < cfg->hidden_size; d++)
emb[d] += row[d];
}
@@ -1511,31 +1523,35 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states,
for (int d = 0; d < cfg->hidden_size; d++)
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings &&
- embed_l2_normalize(emb, cfg->hidden_size) != 0)
+ if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0)
return -1;
offset += len;
}
return 0;
}
-int embed_model_encode_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_embedding)
-{
- embed_input_t input = { token_ids, n_tokens };
+int embed_model_encode_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
return embed_model_encode_batch(model, ws, &input, 1, out_embedding);
}
-float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0) return NULL;
+float *embed_model_encode(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
+ if (!model || !ws || !token_ids || n_tokens <= 0)
+ return NULL;
int hidden = model->config.hidden_size;
- if (hidden <= 0) return NULL;
+ if (hidden <= 0)
+ return NULL;
float *emb = malloc_floats((size_t)hidden);
- if (!emb) return NULL;
+ if (!emb)
+ return NULL;
if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) {
free(emb);
@@ -1548,15 +1564,16 @@ float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
* Vector helpers
* ======================================================================== */
-int embed_l2_normalize(float *vec, int dim)
-{
- if (!vec || dim <= 0) return -1;
+int embed_l2_normalize(float *vec, int dim) {
+ if (!vec || dim <= 0)
+ return -1;
float norm_sq = 0.0f;
for (int i = 0; i < dim; i++)
norm_sq += vec[i] * vec[i];
- if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) return -1;
+ if (!(norm_sq > 0.0f) || !isfinite(norm_sq))
+ return -1;
float inv_norm = 1.0f / sqrtf(norm_sq);
for (int i = 0; i < dim; i++)
@@ -1564,9 +1581,9 @@ int embed_l2_normalize(float *vec, int dim)
return 0;
}
-float embed_cosine_similarity(const float *a, const float *b, int dim)
-{
- if (!a || !b || dim <= 0) return 0.0f;
+float embed_cosine_similarity(const float *a, const float *b, int dim) {
+ if (!a || !b || dim <= 0)
+ return 0.0f;
float dot = 0.0f;
float norm_a_sq = 0.0f;
@@ -1577,8 +1594,7 @@ float embed_cosine_similarity(const float *a, const float *b, int dim)
norm_b_sq += b[i] * b[i];
}
- if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) ||
- !isfinite(norm_a_sq) || !isfinite(norm_b_sq))
+ if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || !isfinite(norm_b_sq))
return 0.0f;
return dot / sqrtf(norm_a_sq * norm_b_sq);
}
diff --git a/embed.h b/embed.h
index f7048b4..24e53e1 100644
--- a/embed.h
+++ b/embed.h
@@ -23,9 +23,9 @@
* Shared constants and family defaults
* ======================================================================== */
-#define EMBED_VOCAB_SIZE 151936
-#define EMBED_HEAD_DIM 128
-#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */
+#define EMBED_VOCAB_SIZE 151936
+#define EMBED_HEAD_DIM 128
+#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */
#define EMBED_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */
/* ========================================================================
@@ -48,8 +48,8 @@ typedef struct {
int n_heads;
int n_kv_heads;
int head_dim;
- int q_dim; /* n_heads * head_dim */
- int kv_dim; /* n_kv_heads * head_dim */
+ int q_dim; /* n_heads * head_dim */
+ int kv_dim; /* n_kv_heads * head_dim */
int intermediate_size;
int vocab_size;
float rms_norm_eps;
@@ -141,16 +141,20 @@ EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws);
*
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
-EMBED_API float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens);
+EMBED_API float *embed_model_encode(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
/*
* Compute one embedding into caller-provided out[hidden_size].
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_model_encode_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_embedding);
+EMBED_API int embed_model_encode_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
/*
* Compute embeddings for a true packed/ragged batch.
@@ -161,9 +165,11 @@ EMBED_API int embed_model_encode_into(const embed_model_t *model, embed_workspac
*
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
- const embed_input_t *inputs, int batch,
- float *out_embeddings);
+EMBED_API int embed_model_encode_batch(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/*
* Apply the config's pooling mode to packed final hidden states.
@@ -171,9 +177,11 @@ EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspa
* states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive
* token count per sequence. The states must already include final RMSNorm.
*/
-EMBED_API int embed_pool_batch(const embed_config_t *cfg, const float *states,
- const int *seq_lengths, int batch,
- float *out_embeddings);
+EMBED_API int embed_pool_batch(const embed_config_t *cfg,
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings);
/*
* Run the transformer forward pass WITHOUT pooling.
@@ -183,16 +191,20 @@ EMBED_API int embed_pool_batch(const embed_config_t *cfg, const float *states,
* Useful for contextual (late-chunking) models where you need
* per-token embeddings before splitting by separator positions.
*/
-EMBED_API float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens);
+EMBED_API float *embed_model_forward(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
/*
* Run the transformer forward pass into caller-provided
* out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_model_forward_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_states);
+EMBED_API int embed_model_forward_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states);
/*
* Pool embeddings from final hidden states.
@@ -201,18 +213,24 @@ EMBED_API int embed_model_forward_into(const embed_model_t *model, embed_workspa
* embed_model_forward_into(). Each span is mean pooled into
* out_embeddings[n_spans, hidden_size] without L2 normalization.
*/
-EMBED_API int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings);
+EMBED_API int embed_pool_spans(const embed_config_t *cfg,
+ const float *states,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
/*
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_model_encode_spans(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings);
+EMBED_API int embed_model_encode_spans(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
/*
* Run a packed/ragged contextual document batch and pool every selected span.
@@ -221,9 +239,10 @@ EMBED_API int embed_model_encode_spans(const embed_model_t *model, embed_workspa
* and out_embeddings contains spans in document order.
*/
EMBED_API int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings);
+ embed_workspace_t *ws,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/*
* L2-normalize vec[dim] in place.
@@ -261,16 +280,20 @@ EMBED_API int embed_late_model_token_dim(const embed_late_model_t *model);
* If normalize is non-zero, every non-zero token vector is L2-normalized.
*/
EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors);
+ embed_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
/*
* Sum over query tokens of max dot product against document tokens.
*/
-EMBED_API float embed_late_maxsim(const float *query_vectors, int query_tokens,
- const float *doc_vectors, int doc_tokens,
- int dim);
+EMBED_API float embed_late_maxsim(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ int doc_tokens,
+ int dim);
/*
* Score one query against a packed/ragged batch of documents.
@@ -279,10 +302,13 @@ EMBED_API float embed_late_maxsim(const float *query_vectors, int query_tokens,
* doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim).
* scores is caller-provided [docs].
*/
-EMBED_API int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets, int docs,
- int dim, float *scores);
+EMBED_API int embed_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores);
/* Verbose level: 0=quiet, 1=info, 2=debug */
EMBED_API extern int embed_verbose;
diff --git a/embed_cli.c b/embed_cli.c
index caa586a..9baa104 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -18,68 +18,66 @@
#include <time.h>
#include <limits.h>
-static double now_ms(void)
-{
+static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
}
-static void print_usage(const char *prog)
-{
+static void print_usage(const char *prog) {
fprintf(stderr,
- "Usage: %s -d <model_dir> [options] [text...]\n"
- "\n"
- "Options:\n"
- " -d <dir> Model directory (required)\n"
+ "Usage: %s -d <model_dir> [options] [text...]\n"
+ "\n"
+ "Options:\n"
+ " -d <dir> Model directory (required)\n"
#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
+ " --mlx Use Apple MLX GPU backend\n"
#endif
#ifdef USE_CUDA
- " --cuda Use CUDA/cuBLAS GPU backend\n"
+ " --cuda Use CUDA/cuBLAS GPU backend\n"
#endif
- " --stream Read lines from stdin, write one JSON embedding per line\n"
- " --json Emit JSON instead of plain text (embedding or matrix)\n"
+ " --stream Read lines from stdin, write one JSON embedding per line\n"
+ " --json Emit JSON instead of plain text (embedding or matrix)\n"
#ifdef USE_MLX
- " --mlx-quant-bits N\n"
- " Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quant-group-size N\n"
- " MLX quantization group size (default: 64)\n"
+ " --mlx-quant-bits N\n"
+ " Quantize MLX linear weights to 8 bits at load time\n"
+ " --mlx-quant-group-size N\n"
+ " MLX quantization group size (default: 64)\n"
#endif
#ifdef USE_CUDA
- " --cuda-gemm-mode MODE\n"
- " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weight-dtype DTYPE\n"
- " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
- " weight memory and uses BF16 tensor cores\n"
+ " --cuda-gemm-mode MODE\n"
+ " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
+ " --cuda-weight-dtype DTYPE\n"
+ " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
+ " weight memory and uses BF16 tensor cores\n"
#endif
- " -b, --batch-size N\n"
- " Max texts per engine batch (default: all; --stream: 1)\n"
- " -t, --threads N\n"
- " CPU threads (default: all cores)\n"
- " -e, --embeddings\n"
- " Print raw embeddings (with multiple texts)\n"
- " -v, --verbose\n"
- " Verbose (-vv for debug)\n"
- " -V, --version\n"
- " Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h Show this help\n"
- "\n"
- "Modes:\n"
- " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
- " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
- " no args Batch: read all stdin lines, then similarity matrix\n"
- " --stream Streaming: read stdin lines, write JSON per line\n"
- "\n"
- "Examples:\n"
- " %s -d ./model \"what is AI?\"\n"
+ " -b, --batch-size N\n"
+ " Max texts per engine batch (default: all; --stream: 1)\n"
+ " -t, --threads N\n"
+ " CPU threads (default: all cores)\n"
+ " -e, --embeddings\n"
+ " Print raw embeddings (with multiple texts)\n"
+ " -v, --verbose\n"
+ " Verbose (-vv for debug)\n"
+ " -V, --version\n"
+ " Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h Show this help\n"
+ "\n"
+ "Modes:\n"
+ " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
+ " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
+ " no args Batch: read all stdin lines, then similarity matrix\n"
+ " --stream Streaming: read stdin lines, write JSON per line\n"
+ "\n"
+ "Examples:\n"
+ " %s -d ./model \"what is AI?\"\n"
#ifdef USE_MLX
- " %s -d ./model --mlx --stream < texts.txt\n",
- prog, prog, prog);
+ " %s -d ./model --mlx --stream < texts.txt\n",
+ prog, prog, prog);
#else
- ,
- prog, prog);
+ ,
+ prog, prog);
#endif
}
@@ -88,29 +86,28 @@ static void print_usage(const char *prog)
* ======================================================================== */
typedef struct {
- embed_model_t *model;
+ embed_model_t *model;
embed_workspace_t *workspace;
#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
+ embed_mlx_ctx_t *mlx_ctx;
#endif
#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
+ embed_cuda_ctx_t *cuda_ctx;
#endif
qwen_tokenizer_t *tok;
qwen_tokenizer_workspace_t *tok_ws;
- int dim;
- int terminal_token_id;
- int append_terminal_token;
+ int dim;
+ int terminal_token_id;
+ int append_terminal_token;
} engine_t;
typedef struct {
int *ids;
- int n_tokens;
+ int n_tokens;
} token_buf_t;
-static int engine_embed_batch(engine_t *e, const embed_input_t *inputs,
- int batch, float *out_embeddings)
-{
+static int
+engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) {
#ifdef USE_MLX
if (e->mlx_ctx)
return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings);
@@ -119,15 +116,12 @@ static int engine_embed_batch(engine_t *e, const embed_input_t *inputs,
if (e->cuda_ctx)
return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings);
#endif
- return embed_model_encode_batch(e->model, e->workspace,
- inputs, batch, out_embeddings);
+ return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings);
}
-static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
-{
+static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) {
memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
- text, &out->n_tokens);
+ out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens);
if (!out->ids || out->n_tokens == 0) {
fprintf(stderr, "tokenization failed: %s\n", text);
free(out->ids);
@@ -141,8 +135,7 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
memset(out, 0, sizeof(*out));
return -1;
}
- int *ids = (int *)realloc(
- out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
if (!ids) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -156,15 +149,16 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
fprintf(stderr, "tokens (%d): ", out->n_tokens);
for (int i = 0; i < out->n_tokens && i < 20; i++)
fprintf(stderr, "%d ", out->ids[i]);
- if (out->n_tokens > 20) fprintf(stderr, "...");
+ if (out->n_tokens > 20)
+ fprintf(stderr, "...");
fprintf(stderr, "\n");
}
return 0;
}
-static void free_tokens(token_buf_t *tokens, int n)
-{
- if (!tokens) return;
+static void free_tokens(token_buf_t *tokens, int n) {
+ if (!tokens)
+ return;
for (int i = 0; i < n; i++)
free(tokens[i].ids);
}
@@ -173,41 +167,40 @@ static void free_tokens(token_buf_t *tokens, int n)
* Output helpers
* ======================================================================== */
-static void print_embedding_raw(const float *emb, int dim)
-{
+static void print_embedding_raw(const float *emb, int dim) {
for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(' ');
+ if (i > 0)
+ putchar(' ');
printf("%.8f", (double)emb[i]);
}
putchar('\n');
}
/* Single embedding as a JSON array of floats. */
-static void print_embedding_array(const float *emb, int dim)
-{
+static void print_embedding_array(const float *emb, int dim) {
putchar('[');
for (int i = 0; i < dim; i++) {
- if (i) putchar(',');
+ if (i)
+ putchar(',');
printf("%.8f", (double)emb[i]);
}
puts("]");
}
-static size_t engine_workspace_nbytes(const engine_t *e)
-{
+static size_t engine_workspace_nbytes(const engine_t *e) {
return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0;
}
-static void print_embedding_json(const float *emb, int dim, int n_tokens,
- double ms, size_t workspace_bytes)
-{
+static void
+print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) {
printf("{\"embedding\":[");
for (int i = 0; i < dim; i++) {
- if (i > 0) putchar(',');
+ if (i > 0)
+ putchar(',');
printf("%.8f", (double)emb[i]);
}
- printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n",
- dim, n_tokens, ms, workspace_bytes);
+ printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms,
+ workspace_bytes);
fflush(stdout);
}
@@ -215,9 +208,9 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens,
* Stdin mode: read lines from stdin, write JSON to stdout
* ======================================================================== */
-static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
-{
- if (n_lines <= 0) return 0;
+static int process_stdin_batch(engine_t *e, char **lines, int n_lines) {
+ if (n_lines <= 0)
+ return 0;
token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t));
@@ -225,7 +218,10 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
if (!tokens || !inputs || !input_to_line || !embs) {
fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(input_to_line); free(embs);
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
for (int i = 0; i < n_lines; i++)
printf("{\"error\":\"embedding failed\"}\n");
fflush(stdout);
@@ -268,26 +264,27 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines)
printf("{\"error\":\"embedding failed\"}\n");
continue;
}
- print_embedding_json(embs + (size_t)next_valid * e->dim,
- e->dim, tokens[i].n_tokens, ms,
+ print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms,
engine_workspace_nbytes(e));
next_valid++;
}
fflush(stdout);
free_tokens(tokens, n_lines);
- free(tokens); free(inputs); free(input_to_line); free(embs);
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
return rc == 0 ? 0 : 1;
}
-static int run_stdin(engine_t *e, int batch_size)
-{
+static int run_stdin(engine_t *e, int batch_size) {
char line[65536];
- if (batch_size <= 0) batch_size = 1;
+ if (batch_size <= 0)
+ batch_size = 1;
if (embed_verbose >= 1)
- fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n",
- batch_size);
+ fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size);
char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
if (!batch_lines) {
@@ -300,13 +297,14 @@ static int run_stdin(engine_t *e, int batch_size)
while (fgets(line, sizeof(line), stdin)) {
/* Strip newline */
size_t len = strlen(line);
- while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r'))
+ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
line[--len] = '\0';
- if (len == 0) continue;
+ if (len == 0)
+ continue;
if (embed_verbose >= 1)
- fprintf(stderr, "stdin: \"%.*s%s\"\n",
- (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : "");
+ fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line,
+ len > 60 ? "..." : "");
batch_lines[n_batch] = strdup(line);
if (!batch_lines[n_batch]) {
@@ -345,18 +343,19 @@ static int run_stdin(engine_t *e, int batch_size)
* Batch mode: embed args or stdin lines, then print similarity or vectors
* ======================================================================== */
-static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
-{
+static int append_text(char ***texts, int *n_texts, int *cap, const char *s) {
if (*n_texts == *cap) {
int new_cap = *cap ? *cap * 2 : 8;
char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
- if (!new_texts) return -1;
+ if (!new_texts)
+ return -1;
*texts = new_texts;
*cap = new_cap;
}
(*texts)[*n_texts] = strdup(s);
- if (!(*texts)[*n_texts]) return -1;
+ if (!(*texts)[*n_texts])
+ return -1;
(*n_texts)++;
return 0;
}
@@ -364,18 +363,18 @@ static int append_text(char ***texts, int *n_texts, int *cap, const char *s)
/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one
* row of space-separated values per line. With print_embs the raw embeddings
* follow, one per line after a blank line (plain-text output only). */
-static void print_matrix(const float *embs, int n, int dim,
- int json, int print_embs)
-{
+static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) {
if (json) {
putchar('[');
for (int i = 0; i < n; i++) {
- if (i) putchar(',');
+ if (i)
+ putchar(',');
putchar('[');
for (int j = 0; j < n; j++) {
- if (j) putchar(',');
- printf("%.6f", (double)embed_cosine_similarity(
- embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
+ if (j)
+ putchar(',');
+ printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
}
putchar(']');
}
@@ -385,9 +384,10 @@ static void print_matrix(const float *embs, int n, int dim,
for (int i = 0; i < n; i++) {
for (int j = 0; j < n; j++) {
- if (j) putchar(' ');
- printf("%.6f", (double)embed_cosine_similarity(
- embs + (size_t)i * dim, embs + (size_t)j * dim, dim));
+ if (j)
+ putchar(' ');
+ printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
}
putchar('\n');
}
@@ -398,12 +398,11 @@ static void print_matrix(const float *embs, int n, int dim,
}
}
-static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
- int print_embs, int batch_size, int json)
-{
- int n_texts = 0, cap = 0;
- char **texts = NULL;
- int rc = 0;
+static int run_batch(
+ engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) {
+ int n_texts = 0, cap = 0;
+ char **texts = NULL;
+ int rc = 0;
if (arg_start < argc) {
for (int i = arg_start; i < argc; i++) {
@@ -414,9 +413,10 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
char line[65536];
while (fgets(line, sizeof(line), stdin)) {
size_t l = strlen(line);
- while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r'))
+ while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r'))
line[--l] = '\0';
- if (l == 0) continue;
+ if (l == 0)
+ continue;
if (append_text(&texts, &n_texts, &cap, line) != 0)
goto oom_texts;
}
@@ -434,15 +434,18 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
if (!tokens || !inputs || !embs) {
fprintf(stderr, "OOM\n");
- free(tokens); free(inputs); free(embs);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
+ free(tokens);
+ free(inputs);
+ free(embs);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
free(texts);
return 1;
}
for (int i = 0; i < n_texts; i++) {
if (embed_verbose >= 1)
- fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]);
+ fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]);
if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
rc = 1;
goto done;
@@ -454,36 +457,43 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start,
int max_batch = batch_size > 0 ? batch_size : n_texts;
for (int start = 0; start < n_texts; start += max_batch) {
int cur = n_texts - start;
- if (cur > max_batch) cur = max_batch;
+ if (cur > max_batch)
+ cur = max_batch;
double t0 = now_ms();
- if (engine_embed_batch(e, inputs + start, cur,
- embs + (size_t)start * dim) != 0) {
+ if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) {
fprintf(stderr, "forward pass failed\n");
rc = 1;
goto done;
}
if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n",
- start, start + cur - 1, cur, now_ms() - t0);
+ fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1,
+ cur, now_ms() - t0);
}
if (n_texts == 1) {
- if (json) print_embedding_array(embs, dim);
- else print_embedding_raw(embs, dim);
+ if (json)
+ print_embedding_array(embs, dim);
+ else
+ print_embedding_raw(embs, dim);
} else {
print_matrix(embs, n_texts, dim, json, print_embs);
}
done:
free_tokens(tokens, n_texts);
- for (int i = 0; i < n_texts; i++) free(texts[i]);
- free(texts); free(tokens); free(inputs); free(embs);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
+ free(texts);
+ free(tokens);
+ free(inputs);
+ free(embs);
return rc;
oom_texts:
fprintf(stderr, "OOM\n");
- for (int i = 0; i < n_texts; i++) free(texts[i]);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
free(texts);
return 1;
}
@@ -492,15 +502,14 @@ oom_texts:
* Main
* ======================================================================== */
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
const char *model_dir = NULL;
- int n_threads = 0;
+ int n_threads = 0;
int print_embs = 0;
- int verbose = 0;
+ int verbose = 0;
int stdin_mode = 0;
int batch_size = 0;
- int json = 0;
+ int json = 0;
const char *prog = embed_prog_name(argv[0]);
#ifdef USE_MLX
int use_mlx = 0;
@@ -516,22 +525,21 @@ int main(int argc, char *argv[])
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
const char *f = argv[arg_start];
- if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
- else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
+ if (!strcmp(f, "-d")) {
+ model_dir = argv[++arg_start];
+ } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
embed_print_version(prog);
return 0;
- }
- else if (!strcmp(f, "--build-info")) {
+ } else if (!strcmp(f, "--build-info")) {
embed_print_build_info(prog);
return 0;
- }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
n_threads = atoi(argv[++arg_start]);
- }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "--json")) {
+ json = 1;
}
- else if (!strcmp(f, "--json")) { json = 1; }
#ifdef USE_MLX
else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg_start]);
@@ -539,8 +547,7 @@ int main(int argc, char *argv[])
fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
return 1;
}
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
+ } else if (!strcmp(f, "--mlx-quant-group-size")) {
mlx_quantize_group_size = atoi(argv[++arg_start]);
if (mlx_quantize_group_size <= 0) {
fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
@@ -551,28 +558,34 @@ int main(int argc, char *argv[])
#ifdef USE_CUDA
else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg_start];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
+ } else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg_start];
}
#endif
else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
print_embs = 1;
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "--stream")) {
+ stdin_mode = 1;
}
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
+ else if (!strcmp(f, "--mlx")) {
+ use_mlx = 1;
+ }
#endif
#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
+ else if (!strcmp(f, "--cuda")) {
+ use_cuda = 1;
+ }
#endif
else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(prog);
return 0;
- }
- else break;
+ } else
+ break;
arg_start++;
}
@@ -620,18 +633,22 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (use_mlx) {
- if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ if (verbose >= 1)
+ fprintf(stderr, "Using MLX GPU backend\n");
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ if (use_cuda) {
+ if (verbose >= 1)
+ fprintf(stderr, "Using CUDA GPU backend\n");
} else
#endif
{
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ if (n_threads <= 0)
+ n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ if (verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
}
/* Tokenizer */
@@ -639,8 +656,12 @@ int main(int argc, char *argv[])
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
double t0 = now_ms();
qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
- if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; }
- if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
+ if (!tok) {
+ fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path);
+ return 1;
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
/* Model */
engine_t e = {0};
@@ -665,7 +686,7 @@ int main(int argc, char *argv[])
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
+ if (use_cuda) {
e.cuda_ctx = embed_cuda_load(model_dir);
if (!e.cuda_ctx) {
fprintf(stderr, "failed to load CUDA model\n");
@@ -699,30 +720,33 @@ int main(int argc, char *argv[])
* wins; otherwise the released-model family constant applies. */
if (e.append_terminal_token) {
int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>");
- e.terminal_token_id =
- eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
}
if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
- e.dim, now_ms() - t0,
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0,
#ifdef USE_MLX
use_mlx ? " (MLX)" :
#endif
#ifdef USE_CUDA
- use_cuda ? " (CUDA)" :
+ use_cuda ? " (CUDA)"
+ :
#endif
- "");
+ "");
e.tok_ws = qwen_tokenizer_workspace_new();
if (!e.tok_ws) {
fprintf(stderr, "failed to allocate tokenizer workspace\n");
- if (e.workspace) embed_workspace_free(e.workspace);
- if (e.model) embed_model_free(e.model);
+ if (e.workspace)
+ embed_workspace_free(e.workspace);
+ if (e.model)
+ embed_model_free(e.model);
#ifdef USE_MLX
- if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx);
+ if (e.mlx_ctx)
+ embed_mlx_free(e.mlx_ctx);
#endif
#ifdef USE_CUDA
- if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx);
+ if (e.cuda_ctx)
+ embed_cuda_free(e.cuda_ctx);
#endif
qwen_tokenizer_free(tok);
return 1;
@@ -736,13 +760,17 @@ int main(int argc, char *argv[])
rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json);
/* Cleanup */
- if (e.workspace) embed_workspace_free(e.workspace);
- if (e.model) embed_model_free(e.model);
+ if (e.workspace)
+ embed_workspace_free(e.workspace);
+ if (e.model)
+ embed_model_free(e.model);
#ifdef USE_MLX
- if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx);
+ if (e.mlx_ctx)
+ embed_mlx_free(e.mlx_ctx);
#endif
#ifdef USE_CUDA
- if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx);
+ if (e.cuda_ctx)
+ embed_cuda_free(e.cuda_ctx);
#endif
qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 1ec0573..3955907 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -15,10 +15,10 @@ extern "C" {
#include <string.h>
typedef struct {
- float *d; // device buffer; holds __nv_bfloat16 elements when bf16
+ float *d; // device buffer; holds __nv_bfloat16 elements when bf16
int rows;
int cols;
- int bf16; // 1 if d holds __nv_bfloat16, 0 if F32
+ int bf16; // 1 if d holds __nv_bfloat16, 0 if F32
} cuda_matrix_t;
typedef struct {
@@ -47,7 +47,7 @@ struct embed_cuda_ctx {
float *qkv;
float *attn_out;
float *ffn_gate_up;
- void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
+ void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
float *pooled_out;
float *rope_cos;
float *rope_sin;
@@ -58,16 +58,16 @@ struct embed_cuda_ctx {
float *kexp;
float *vexp;
float *attn_scores;
- void *attn_probs; // BF16 softmax output (bf16 weights)
+ void *attn_probs; // BF16 softmax output (bf16 weights)
long long attn_scores_elems;
long long attn_probs_elems;
// Equal-length micro-batch attention: pointer arrays for batched GEMMs,
// built once per forward pass (layer-invariant buffer addresses).
// attn_G is the number of sequences whose score tensors fit the scratch
// budget at once; 0 disables the batched path for this forward.
- const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads
+ const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads
const void **attn_ptrs_host;
- int attn_ptrs_cap; // capacity in pointer entries
+ int attn_ptrs_cap; // capacity in pointer entries
int attn_G;
int attn_L;
int *span_starts;
@@ -83,26 +83,25 @@ enum { CUDA_ATTN_SCORES_BUDGET = 64 * 1024 * 1024 };
// >=256; both paths are identical precision).
enum { CUDA_ATTN_BATCHED_MAX_LEN = 192 };
-#define CUDA_CHECK(expr) do { \
- cudaError_t _e = (expr); \
- if (_e != cudaSuccess) { \
- fprintf(stderr, "cuda: %s failed: %s\n", #expr, \
- cudaGetErrorString(_e)); \
- return -1; \
- } \
-} while (0)
+#define CUDA_CHECK(expr) \
+ do { \
+ cudaError_t _e = (expr); \
+ if (_e != cudaSuccess) { \
+ fprintf(stderr, "cuda: %s failed: %s\n", #expr, cudaGetErrorString(_e)); \
+ return -1; \
+ } \
+ } while (0)
-#define CUBLAS_CHECK(expr) do { \
- cublasStatus_t _s = (expr); \
- if (_s != CUBLAS_STATUS_SUCCESS) { \
- fprintf(stderr, "cuda: %s failed: cublas status %d\n", \
- #expr, (int)_s); \
- return -1; \
- } \
-} while (0)
+#define CUBLAS_CHECK(expr) \
+ do { \
+ cublasStatus_t _s = (expr); \
+ if (_s != CUBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "cuda: %s failed: cublas status %d\n", #expr, (int)_s); \
+ return -1; \
+ } \
+ } while (0)
-static int parse_gemm_compute(const char *mode, cublasComputeType_t *out)
-{
+static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) {
if (!mode || !mode[0] || !strcmp(mode, "f32")) {
*out = CUBLAS_COMPUTE_32F;
return 0;
@@ -125,8 +124,7 @@ static int parse_gemm_compute(const char *mode, cublasComputeType_t *out)
static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
static int g_gemm_compute_set = 0;
-int embed_cuda_set_fast_gemm(const char *mode)
-{
+int embed_cuda_set_fast_gemm(const char *mode) {
cublasComputeType_t ct;
if (parse_gemm_compute(mode, &ct) != 0)
return -1;
@@ -135,15 +133,14 @@ int embed_cuda_set_fast_gemm(const char *mode)
return 0;
}
-static void cuda_matrix_free(cuda_matrix_t *m)
-{
- if (!m) return;
+static void cuda_matrix_free(cuda_matrix_t *m) {
+ if (!m)
+ return;
cudaFree(m->d);
memset(m, 0, sizeof(*m));
}
-static float bf16_to_f32(uint16_t v)
-{
+static float bf16_to_f32(uint16_t v) {
uint32_t u = ((uint32_t)v) << 16;
float f;
memcpy(&f, &u, sizeof(f));
@@ -152,13 +149,12 @@ static float bf16_to_f32(uint16_t v)
// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are
// preserved. BF16 shares F32's 8-bit exponent, so this never overflows.
-static uint16_t f32_to_bf16(float f)
-{
+static uint16_t f32_to_bf16(float f) {
uint32_t x;
memcpy(&x, &f, sizeof(x));
- if ((x & 0x7fffffffu) > 0x7f800000u) // NaN
+ if ((x & 0x7fffffffu) > 0x7f800000u) // NaN
return (uint16_t)((x >> 16) | 0x0040u);
- x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even
+ x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even
return (uint16_t)(x >> 16);
}
@@ -169,17 +165,15 @@ static uint16_t f32_to_bf16(float f)
static int g_weights_bf16 = 0;
static int g_weights_bf16_set = 0;
-int embed_cuda_set_weights_bf16(int on)
-{
+int embed_cuda_set_weights_bf16(int on) {
g_weights_bf16 = on ? 1 : 0;
g_weights_bf16_set = 1;
return 0;
}
-static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w,
- size_t count)
-{
- if (!dst || !w || !w->data) return -1;
+static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) {
+ if (!dst || !w || !w->data)
+ return -1;
if (w->dtype == DTYPE_F32) {
memcpy(dst, w->data, count * sizeof(float));
return 0;
@@ -195,13 +189,12 @@ static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w,
// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is
// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16.
-static int upload_weight(cuda_matrix_t *m, const float *src, size_t count,
- int rows, int cols)
-{
+static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int rows, int cols) {
cudaError_t e;
if (g_weights_bf16) {
uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t));
- if (!tmp16) return -1;
+ if (!tmp16)
+ return -1;
for (size_t i = 0; i < count; i++)
tmp16[i] = f32_to_bf16(src[i]);
e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16));
@@ -210,8 +203,7 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count,
free(tmp16);
return -1;
}
- e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16),
- cudaMemcpyHostToDevice);
+ e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice);
free(tmp16);
m->bf16 = 1;
} else {
@@ -233,13 +225,13 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count,
return 0;
}
-static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w,
- int rows, int cols)
-{
- if (!m || !w || rows <= 0 || cols <= 0) return -1;
+static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) {
+ if (!m || !w || rows <= 0 || cols <= 0)
+ return -1;
size_t count = (size_t)rows * (size_t)cols;
float *tmp = (float *)malloc(count * sizeof(float));
- if (!tmp) return -1;
+ if (!tmp)
+ return -1;
if (copy_weight_host_f32(tmp, w, count) != 0) {
free(tmp);
return -1;
@@ -249,17 +241,17 @@ static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w,
return r;
}
-static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src,
- const embed_config_t *c)
-{
- if (!m || !src || !c) return -1;
+static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+ if (!m || !src || !c)
+ return -1;
int rows = c->q_dim + 2 * c->kv_dim;
int cols = c->hidden_size;
size_t q_count = (size_t)c->q_dim * cols;
size_t kv_count = (size_t)c->kv_dim * cols;
size_t total = q_count + 2 * kv_count;
float *tmp = (float *)malloc(total * sizeof(float));
- if (!tmp) return -1;
+ if (!tmp)
+ return -1;
if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 ||
copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 ||
copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) {
@@ -271,16 +263,17 @@ static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src,
return r;
}
-static int load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src,
- const embed_config_t *c)
-{
- if (!m || !src || !c) return -1;
+static int
+load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+ if (!m || !src || !c)
+ return -1;
int rows = 2 * c->intermediate_size;
int cols = c->hidden_size;
size_t proj_count = (size_t)c->intermediate_size * cols;
size_t total = 2 * proj_count;
float *tmp = (float *)malloc(total * sizeof(float));
- if (!tmp) return -1;
+ if (!tmp)
+ return -1;
if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 ||
copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) {
free(tmp);
@@ -291,9 +284,9 @@ static int load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src,
return r;
}
-static int load_vector(float **out, const float *src, int n)
-{
- if (!out || !src || n <= 0) return -1;
+static int load_vector(float **out, const float *src, int n) {
+ if (!out || !src || n <= 0)
+ return -1;
cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float));
if (e != cudaSuccess) {
fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e));
@@ -309,13 +302,12 @@ static int load_vector(float **out, const float *src, int n)
return 0;
}
-__global__ static void embed_lookup_kernel(float *x, const int *ids,
- const void *emb, int emb_bf16,
- int total, int hidden, int vocab)
-{
+__global__ static void embed_lookup_kernel(
+ float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int count = total * hidden;
- if (idx >= count) return;
+ if (idx >= count)
+ return;
int tok = idx / hidden;
int d = idx - tok * hidden;
int id = ids[tok];
@@ -324,20 +316,18 @@ __global__ static void embed_lookup_kernel(float *x, const int *ids,
return;
}
size_t off = (size_t)id * hidden + d;
- x[idx] = emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off])
- : ((const float *)emb)[off];
+ x[idx] =
+ emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) : ((const float *)emb)[off];
}
// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs).
-__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out,
- const float *in, size_t n)
-{
+__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float *in, size_t n) {
size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- if (i < n) out[i] = __float2bfloat16(in[i]);
+ if (i < n)
+ out[i] = __float2bfloat16(in[i]);
}
-__device__ static float block_sum(float v)
-{
+__device__ static float block_sum(float v) {
__shared__ float partial[8];
__shared__ float total;
int lane = threadIdx.x & 31;
@@ -346,36 +336,32 @@ __device__ static float block_sum(float v)
for (int offset = 16; offset > 0; offset >>= 1)
v += __shfl_down_sync(0xffffffff, v, offset);
- if (lane == 0) partial[warp] = v;
+ if (lane == 0)
+ partial[warp] = v;
__syncthreads();
if (warp == 0) {
v = lane < warps ? partial[lane] : 0.0f;
for (int offset = 16; offset > 0; offset >>= 1)
v += __shfl_down_sync(0xffffffff, v, offset);
- if (lane == 0) total = v;
+ if (lane == 0)
+ total = v;
}
__syncthreads();
return total;
}
-__device__ static inline void store_act(float *out, size_t i, float v)
-{
- out[i] = v;
-}
+__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; }
-__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v)
-{
+__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) {
out[i] = __float2bfloat16(v);
}
// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
// projection GEMM directly (same rounding as a separate cast kernel).
template <typename OUT_T>
-__global__ static void rms_norm_kernel(OUT_T *out, const float *x,
- const float *weight, int rows,
- int dim, float eps)
-{
+__global__ static void
+rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int dim, float eps) {
int row = blockIdx.x;
int tid = threadIdx.x;
float sum = 0.0f;
@@ -386,24 +372,25 @@ __global__ static void rms_norm_kernel(OUT_T *out, const float *x,
float total = block_sum(sum);
float inv = rsqrtf(total / (float)dim + eps);
for (int d = tid; d < dim; d += blockDim.x)
- store_act(out, (size_t)row * dim + d,
- x[(size_t)row * dim + d] * inv * weight[d]);
+ store_act(out, (size_t)row * dim + d, x[(size_t)row * dim + d] * inv * weight[d]);
}
// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with
// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset.
-__global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm,
+__global__ static void rms_norm_rope_qk_kernel(float *x,
+ const float *q_norm,
const float *k_norm,
const int *positions,
const float *cosv,
const float *sinv,
- int rows, int n_heads,
+ int rows,
+ int n_heads,
int n_kv_heads,
int head_dim,
int row_stride,
- int q_offset, int k_offset,
- float eps)
-{
+ int q_offset,
+ int k_offset,
+ float eps) {
__shared__ float partial[4];
__shared__ float total;
int row = blockIdx.x;
@@ -416,7 +403,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm,
float sign = tid < half ? -1.0f : 1.0f;
const float *weight = head < n_heads ? q_norm : k_norm;
int base_offset = head < n_heads ? q_offset : k_offset;
- if (head >= n_heads) head -= n_heads;
+ if (head >= n_heads)
+ head -= n_heads;
size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
float v = tid < head_dim ? x[base + tid] : 0.0f;
float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
@@ -424,13 +412,15 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm,
for (int offset = 16; offset > 0; offset >>= 1)
sum += __shfl_down_sync(0xffffffff, sum, offset);
- if (lane == 0) partial[warp] = sum;
+ if (lane == 0)
+ partial[warp] = sum;
__syncthreads();
if (warp == 0) {
sum = lane < 4 ? partial[lane] : 0.0f;
for (int offset = 16; offset > 0; offset >>= 1)
sum += __shfl_down_sync(0xffffffff, sum, offset);
- if (lane == 0) total = sum;
+ if (lane == 0)
+ total = sum;
}
__syncthreads();
@@ -449,10 +439,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm,
// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16
// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM.
template <typename OUT_T>
-__global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
- const float *gate_up, int rows,
- int intermediate)
-{
+__global__ static void silu_mul_packed_kernel(
+ OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
int n = rows * intermediate;
if (i < n) {
@@ -461,8 +449,7 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
size_t base = (size_t)row * (2 * intermediate);
float g = gate_up[base + d];
float u = gate_up[base + intermediate + d];
- store_act(out, (size_t)row * out_stride + d,
- (g / (1.0f + expf(-g))) * u);
+ store_act(out, (size_t)row * out_stride + d, (g / (1.0f + expf(-g))) * u);
}
}
@@ -471,14 +458,21 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
// so the attention GEMMs can stride uniformly over query heads.
// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly.
template <typename V_T>
-__global__ static void attn_expand_kv_kernel(
- float *kexp, V_T *vexp, const float *qkv, int total, int n_heads,
- int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset)
-{
+__global__ static void attn_expand_kv_kernel(float *kexp,
+ V_T *vexp,
+ const float *qkv,
+ int total,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ int qkv_dim,
+ int k_offset,
+ int v_offset) {
int q_dim = n_heads * head_dim;
size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
size_t count = (size_t)total * q_dim;
- if (idx >= count) return;
+ if (idx >= count)
+ return;
int t = (int)(idx / q_dim);
int rem = (int)(idx - (size_t)t * q_dim);
int h = rem / head_dim;
@@ -495,19 +489,20 @@ __global__ static void attn_expand_kv_kernel(
// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
// probabilities feed a BF16 @V GEMM.
template <typename P_T>
-__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
-{
+__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) {
float *s = scores + (size_t)blockIdx.x * L;
P_T *p = probs + (size_t)blockIdx.x * L;
int tid = threadIdx.x;
__shared__ float red[256];
float m = -3.402823466e+38F;
- for (int j = tid; j < L; j += blockDim.x) m = fmaxf(m, s[j]);
+ for (int j = tid; j < L; j += blockDim.x)
+ m = fmaxf(m, s[j]);
red[tid] = m;
__syncthreads();
for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]);
+ if (tid < o)
+ red[tid] = fmaxf(red[tid], red[tid + o]);
__syncthreads();
}
m = red[0];
@@ -522,7 +517,8 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
red[tid] = sum;
__syncthreads();
for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o) red[tid] += red[tid + o];
+ if (tid < o)
+ red[tid] += red[tid + o];
__syncthreads();
}
float inv = 1.0f / red[0];
@@ -531,9 +527,7 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
}
template <typename P_T>
-__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs,
- int L)
-{
+__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int L) {
float *s = scores + (size_t)blockIdx.x * L;
P_T *p = probs + (size_t)blockIdx.x * L;
int query = blockIdx.x % L;
@@ -542,11 +536,13 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs,
__shared__ float red[256];
float m = -3.402823466e+38F;
- for (int j = tid; j < keys; j += blockDim.x) m = fmaxf(m, s[j]);
+ for (int j = tid; j < keys; j += blockDim.x)
+ m = fmaxf(m, s[j]);
red[tid] = m;
__syncthreads();
for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]);
+ if (tid < o)
+ red[tid] = fmaxf(red[tid], red[tid + o]);
__syncthreads();
}
m = red[0];
@@ -561,7 +557,8 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs,
red[tid] = sum;
__syncthreads();
for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o) red[tid] += red[tid + o];
+ if (tid < o)
+ red[tid] += red[tid + o];
__syncthreads();
}
float inv = 1.0f / red[0];
@@ -569,10 +566,8 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs,
store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
}
-__global__ static void mean_pool_kernel(float *out, const float *x,
- const int *offsets, int batch,
- int hidden)
-{
+__global__ static void
+mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
int b = blockIdx.x;
int tid = threadIdx.x;
int start = offsets[b];
@@ -586,10 +581,8 @@ __global__ static void mean_pool_kernel(float *out, const float *x,
}
}
-__global__ static void last_pool_kernel(float *out, const float *x,
- const int *offsets, int batch,
- int hidden)
-{
+__global__ static void
+last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
int b = blockIdx.x;
int tid = threadIdx.x;
int last = offsets[b + 1] - 1;
@@ -597,10 +590,8 @@ __global__ static void last_pool_kernel(float *out, const float *x,
out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d];
}
-__global__ static void span_pool_kernel(float *out, const float *x,
- const int *starts, const int *lens,
- int n_spans, int hidden)
-{
+__global__ static void span_pool_kernel(
+ float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) {
int s = blockIdx.x;
int tid = threadIdx.x;
int start = starts[s];
@@ -613,8 +604,7 @@ __global__ static void span_pool_kernel(float *out, const float *x,
}
}
-static int launch_check(void)
-{
+static int launch_check(void) {
cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e));
@@ -627,8 +617,7 @@ static int launch_check(void)
// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
// internally.
-static cublasComputeType_t gemm_compute(void)
-{
+static cublasComputeType_t gemm_compute(void) {
static int env_init = 0;
if (!g_gemm_compute_set && !env_init) {
const char *e = getenv("EMBED_CUDA_GEMM_MODE");
@@ -644,80 +633,100 @@ static cublasComputeType_t gemm_compute(void)
// already produced in BF16 by the upstream kernel, otherwise the F32 operand
// is cast into ctx->act_bf16 first. With F32 weights this is the exact path,
// honoring gemm_compute().
-static int linear_ex(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w,
- const void *x, int x_is_bf16, float *y, int rows,
- int in_dim, int out_dim, int x_stride, float beta)
-{
+static int linear_ex(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const void *x,
+ int x_is_bf16,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim,
+ int x_stride,
+ float beta) {
const float alpha = 1.0f;
if (w->bf16) {
const void *xb = x;
if (!x_is_bf16) {
size_t n = (size_t)rows * (size_t)x_stride;
int threads = 256;
- cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0,
- ctx->stream>>>(
+ cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, ctx->stream>>>(
(__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n);
- if (launch_check() != 0) return -1;
+ if (launch_check() != 0)
+ return -1;
xb = ctx->act_bf16;
}
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N,
- out_dim, rows, in_dim,
- &alpha, w->d, CUDA_R_16BF, in_dim,
- xb, CUDA_R_16BF, x_stride,
- &beta, y, CUDA_R_32F, out_dim,
- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
+ &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride,
+ &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT));
return 0;
}
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N,
- out_dim, rows, in_dim,
- &alpha, w->d, CUDA_R_32F, in_dim,
- x, CUDA_R_32F, x_stride,
- &beta, y, CUDA_R_32F, out_dim,
- gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha,
+ w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y,
+ CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT));
return 0;
}
-static int linear(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w,
- const float *x, float *y, int rows, int in_dim, int out_dim)
-{
+static int linear(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const float *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim) {
return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f);
}
// x points at activations already stored as BF16 (e.g. ctx->act_bf16).
-static int linear_bf16x(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w,
- const void *x, float *y, int rows, int in_dim,
- int out_dim, int x_stride, float beta)
-{
+static int linear_bf16x(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const void *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim,
+ int x_stride,
+ float beta) {
return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta);
}
-static int linear_accum(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w,
- const float *x, float *y,
- int rows, int in_dim, int out_dim)
-{
+static int linear_accum(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const float *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim) {
return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f);
}
-static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa,
- cublasOperation_t transb, int m, int n, int k,
- const float *alpha, const float *A, int lda,
- long long strideA, const float *B, int ldb,
- long long strideB, const float *beta,
- float *C, int ldc, long long strideC,
- int batch_count)
-{
+static int gemm_strided_batched(cublasHandle_t blas,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const float *alpha,
+ const float *A,
+ int lda,
+ long long strideA,
+ const float *B,
+ int ldb,
+ long long strideB,
+ const float *beta,
+ float *C,
+ int ldc,
+ long long strideC,
+ int batch_count) {
cublasComputeType_t ct = gemm_compute();
if (ct == CUBLAS_COMPUTE_32F) {
- CUBLAS_CHECK(cublasSgemmStridedBatched(
- blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb,
- strideB, beta, C, ldc, strideC, batch_count));
+ CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda,
+ strideA, B, ldb, strideB, beta, C, ldc, strideC,
+ batch_count));
} else {
CUBLAS_CHECK(cublasGemmStridedBatchedEx(
- blas, transa, transb, m, n, k,
- alpha, A, CUDA_R_32F, lda, strideA,
- B, CUDA_R_32F, ldb, strideB,
- beta, C, CUDA_R_32F, ldc, strideC,
- batch_count, ct, CUBLAS_GEMM_DEFAULT));
+ blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb,
+ strideB, beta, C, CUDA_R_32F, ldc, strideC, batch_count, ct, CUBLAS_GEMM_DEFAULT));
}
return 0;
}
@@ -733,22 +742,26 @@ static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa,
// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
// batches, single sequences, or when one sequence's scores exceed the
// budget.
-static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset,
- int qkv_dim)
-{
+static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
const embed_config_t *c = &ctx->config;
ctx->attn_G = 0;
- if (batch < 2) return 0;
+ if (batch < 2)
+ return 0;
int L = ctx->offsets_host[1] - ctx->offsets_host[0];
- if (L > CUDA_ATTN_BATCHED_MAX_LEN) return 0;
+ if (L > CUDA_ATTN_BATCHED_MAX_LEN)
+ return 0;
for (int b = 1; b < batch; b++)
- if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L) return 0;
+ if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L)
+ return 0;
int H = c->n_heads, hd = c->head_dim, q_dim = c->q_dim;
long long per = (long long)H * L * L;
- if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2) return 0;
+ if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2)
+ return 0;
int G = (int)(CUDA_ATTN_SCORES_BUDGET / per);
- if (G > batch) G = batch;
- if (G < 2) return 0;
+ if (G > batch)
+ G = batch;
+ if (G < 2)
+ return 0;
int bf16 = g_weights_bf16;
long long want = (long long)G * per;
@@ -756,16 +769,14 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset,
cudaFree(ctx->attn_scores);
ctx->attn_scores = NULL;
ctx->attn_scores_elems = 0;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores,
- (size_t)want * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)want * sizeof(float)));
ctx->attn_scores_elems = want;
}
if (bf16 && want > ctx->attn_probs_elems) {
cudaFree(ctx->attn_probs);
ctx->attn_probs = NULL;
ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs,
- (size_t)want * sizeof(__nv_bfloat16)));
+ CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)want * sizeof(__nv_bfloat16)));
ctx->attn_probs_elems = want;
}
@@ -775,11 +786,10 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset,
free((void *)ctx->attn_ptrs_host);
ctx->attn_ptrs = NULL;
ctx->attn_ptrs_cap = 0;
- ctx->attn_ptrs_host =
- (const void **)malloc((size_t)entries * sizeof(void *));
- if (!ctx->attn_ptrs_host) return -1;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs,
- (size_t)entries * sizeof(void *)));
+ ctx->attn_ptrs_host = (const void **)malloc((size_t)entries * sizeof(void *));
+ if (!ctx->attn_ptrs_host)
+ return -1;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, (size_t)entries * sizeof(void *)));
ctx->attn_ptrs_cap = entries;
}
@@ -813,8 +823,7 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset,
}
}
CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host,
- (size_t)entries * sizeof(void *),
- cudaMemcpyHostToDevice));
+ (size_t)entries * sizeof(void *), cudaMemcpyHostToDevice));
ctx->attn_G = G;
ctx->attn_L = L;
return 0;
@@ -823,11 +832,15 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset,
// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
-static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
- int batch, int q_offset, int k_offset,
- int v_offset, int qkv_dim, float scale,
- __nv_bfloat16 *out_bf16)
-{
+static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
+ const int *h_offsets,
+ int batch,
+ int q_offset,
+ int k_offset,
+ int v_offset,
+ int qkv_dim,
+ float scale,
+ __nv_bfloat16 *out_bf16) {
const embed_config_t *c = &ctx->config;
int q_dim = c->q_dim;
int hd = c->head_dim;
@@ -839,17 +852,16 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
int threads = 256;
size_t exp_count = (size_t)total * q_dim;
if (out_bf16) {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
- 0, ctx->stream>>>(
- ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H,
- c->n_kv_heads, hd, qkv_dim, k_offset, v_offset);
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim,
+ k_offset, v_offset);
} else {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
- 0, ctx->stream>>>(
- ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd,
- qkv_dim, k_offset, v_offset);
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset,
+ v_offset);
}
- if (launch_check() != 0) return -1;
+ if (launch_check() != 0)
+ return -1;
if (ctx->attn_G >= 2) {
int G = ctx->attn_G, L = ctx->attn_L;
@@ -860,44 +872,37 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
const void *const *P = V + (size_t)batch * H;
void *const *O = (void *const *)(P + (size_t)batch * H);
cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F;
- cublasComputeType_t av_ct =
- out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute();
+ cublasComputeType_t av_ct = out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute();
for (int s = 0; s < batch; s += G) {
int g = batch - s < G ? batch - s : G;
int n = g * H;
size_t off = (size_t)s * H;
/* scores = scale * Q @ K^T, batched over (seq, head) */
- CUBLAS_CHECK(cublasGemmBatchedEx(
- ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
- &alpha, K + off, CUDA_R_32F, q_dim,
- Q + off, CUDA_R_32F, qkv_dim,
- &beta, S + off, CUDA_R_32F, L,
- n, gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha,
+ K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F,
+ qkv_dim, &beta, S + off, CUDA_R_32F, L, n,
+ gemm_compute(), CUBLAS_GEMM_DEFAULT));
if (out_bf16) {
if (causal)
attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores,
- (__nv_bfloat16 *)ctx->attn_probs, L);
+ ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
else
attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores,
- (__nv_bfloat16 *)ctx->attn_probs, L);
+ ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
} else {
if (causal)
- attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
+ attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
else
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
}
- if (launch_check() != 0) return -1;
+ if (launch_check() != 0)
+ return -1;
/* O = P @ V, batched over (seq, head) */
- CUBLAS_CHECK(cublasGemmBatchedEx(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V + off, pv, q_dim,
- P + off, pv, L,
- &beta, O + off, pv, q_dim,
- n, av_ct, CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one,
+ V + off, pv, q_dim, P + off, pv, L, &beta, O + off, pv,
+ q_dim, n, av_ct, CUBLAS_GEMM_DEFAULT));
}
return 0;
}
@@ -908,53 +913,47 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
const float *K = ctx->kexp + (size_t)start * q_dim;
/* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
- if (gemm_strided_batched(
- ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
- &alpha, K, q_dim, hd, Q, qkv_dim, hd,
- &beta, ctx->attn_scores, L, (long long)L * L, H) != 0)
+ if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim,
+ hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L,
+ H) != 0)
return -1;
if (out_bf16) {
__nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs;
- const __nv_bfloat16 *V =
- (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
+ const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
__nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
if (causal)
- attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, P, L);
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
else
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, P, L);
- if (launch_check() != 0) return -1;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
+ if (launch_check() != 0)
+ return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
CUBLAS_CHECK(cublasGemmStridedBatchedEx(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, CUDA_R_16BF, q_dim, hd,
- P, CUDA_R_16BF, L, (long long)L * L,
- &beta, O, CUDA_R_16BF, q_dim, hd,
- H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P,
+ CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
} else {
const float *V = ctx->vexp + (size_t)start * q_dim;
float *O = ctx->attn_out + (size_t)start * q_dim;
if (causal)
- attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
else
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
- if (launch_check() != 0) return -1;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
+ if (launch_check() != 0)
+ return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- if (gemm_strided_batched(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
- &one, V, q_dim, hd, ctx->attn_scores, L,
- (long long)L * L, &beta, O, q_dim, hd, H) != 0)
+ if (gemm_strided_batched(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, q_dim,
+ hd, ctx->attn_scores, L, (long long)L * L, &beta, O, q_dim, hd,
+ H) != 0)
return -1;
}
}
return 0;
}
-static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq)
-{
+static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
const embed_config_t *c = &ctx->config;
if (total > ctx->seq_cap) {
cudaFree(ctx->x);
@@ -972,9 +971,11 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
ctx->token_ids = ctx->positions = NULL;
ctx->kexp = ctx->vexp = NULL;
ctx->act_bf16 = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->qkv,
+ (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up,
(size_t)total * 2 * c->intermediate_size * sizeof(float)));
@@ -983,8 +984,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float)));
if (g_weights_bf16)
- CUDA_CHECK(cudaMalloc(&ctx->act_bf16,
- (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16)));
+ CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size *
+ sizeof(__nv_bfloat16)));
ctx->seq_cap = total;
}
if (batch + 1 > ctx->batch_cap) {
@@ -992,7 +993,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
free(ctx->offsets_host);
CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int));
- if (!ctx->offsets_host) return -1;
+ if (!ctx->offsets_host)
+ return -1;
ctx->batch_cap = batch + 1;
}
if (max_seq > ctx->max_seq_cap) {
@@ -1003,17 +1005,14 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
cudaFree(ctx->attn_scores);
ctx->attn_scores = NULL;
ctx->attn_scores_elems = 0;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores,
- (size_t)score_elems * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float)));
ctx->attn_scores_elems = score_elems;
}
if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) {
cudaFree(ctx->attn_probs);
ctx->attn_probs = NULL;
ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs,
- (size_t)score_elems *
- sizeof(__nv_bfloat16)));
+ CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(__nv_bfloat16)));
ctx->attn_probs_elems = score_elems;
}
float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
@@ -1026,8 +1025,7 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
int half = c->head_dim / 2;
for (int pos = 0; pos < max_seq; pos++) {
for (int d = 0; d < half; d++) {
- float inv = 1.0f / powf(c->rope_theta,
- (float)(2 * d) / (float)c->head_dim);
+ float inv = 1.0f / powf(c->rope_theta, (float)(2 * d) / (float)c->head_dim);
float angle = (float)pos * inv;
float cv = cosf(angle);
float sv = sinf(angle);
@@ -1037,10 +1035,10 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
hsin[(size_t)pos * c->head_dim + half + d] = sv;
}
}
- CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos,
- (size_t)max_seq * c->head_dim * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin,
- (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float)));
CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
(size_t)max_seq * c->head_dim * sizeof(float),
cudaMemcpyHostToDevice, ctx->stream));
@@ -1054,50 +1052,43 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
return 0;
}
-static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows)
-{
- if (rows <= ctx->pooled_rows_cap) return 0;
+static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) {
+ if (rows <= ctx->pooled_rows_cap)
+ return 0;
const embed_config_t *c = &ctx->config;
cudaFree(ctx->pooled_out);
ctx->pooled_out = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out,
- (size_t)rows * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float)));
ctx->pooled_rows_cap = rows;
return 0;
}
-static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans)
-{
- if (n_spans <= ctx->span_cap) return 0;
+static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
+ if (n_spans <= ctx->span_cap)
+ return 0;
cudaFree(ctx->span_starts);
cudaFree(ctx->span_lens);
ctx->span_starts = NULL;
ctx->span_lens = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts,
- (size_t)n_spans * sizeof(int)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens,
- (size_t)n_spans * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, (size_t)n_spans * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, (size_t)n_spans * sizeof(int)));
ctx->span_cap = n_spans;
return 0;
}
-static int load_layer(cuda_layer_t *dst, const embed_layer_t *src,
- const embed_config_t *c)
-{
- return
- load_qkv_matrix(&dst->qkv, src, c) ||
- load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
- load_vector(&dst->q_norm, src->q_norm, c->head_dim) ||
- load_vector(&dst->k_norm, src->k_norm, c->head_dim) ||
- load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
- load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
- load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
- load_matrix(&dst->down_proj, &src->down_proj,
- c->hidden_size, c->intermediate_size);
+static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
+ return load_qkv_matrix(&dst->qkv, src, c) ||
+ load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
+ load_vector(&dst->q_norm, src->q_norm, c->head_dim) ||
+ load_vector(&dst->k_norm, src->k_norm, c->head_dim) ||
+ load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
+ load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
+ load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
+ load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size);
}
-static void free_layer(cuda_layer_t *l)
-{
+static void free_layer(cuda_layer_t *l) {
cuda_matrix_free(&l->qkv);
cuda_matrix_free(&l->wo);
cudaFree(l->q_norm);
@@ -1108,10 +1099,10 @@ static void free_layer(cuda_layer_t *l)
cuda_matrix_free(&l->down_proj);
}
-embed_cuda_ctx_t *embed_cuda_load(const char *model_dir)
-{
+embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
embed_model_t *cpu = embed_model_load(model_dir);
- if (!cpu) return NULL;
+ if (!cpu)
+ return NULL;
embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx));
if (!ctx) {
@@ -1144,8 +1135,8 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir)
embed_cuda_free(ctx);
return NULL;
}
- if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens,
- c->vocab_size, c->hidden_size) != 0 ||
+ if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size,
+ c->hidden_size) != 0 ||
load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
embed_cuda_free(ctx);
return NULL;
@@ -1161,9 +1152,9 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir)
return ctx;
}
-void embed_cuda_free(embed_cuda_ctx_t *ctx)
-{
- if (!ctx) return;
+void embed_cuda_free(embed_cuda_ctx_t *ctx) {
+ if (!ctx)
+ return;
cuda_matrix_free(&ctx->embed_tokens);
if (ctx->layers) {
for (int i = 0; i < ctx->config.n_layers; i++)
@@ -1192,24 +1183,25 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx)
free((void *)ctx->attn_ptrs_host);
cudaFree(ctx->span_starts);
cudaFree(ctx->span_lens);
- if (ctx->blas) cublasDestroy(ctx->blas);
- if (ctx->stream) cudaStreamDestroy(ctx->stream);
+ if (ctx->blas)
+ cublasDestroy(ctx->blas);
+ if (ctx->stream)
+ cudaStreamDestroy(ctx->stream);
embed_model_free(ctx->cpu);
free(ctx);
}
-const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx)
-{
+const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) {
return ctx ? &ctx->config : NULL;
}
-static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
- int batch)
-{
- if (!ctx || !inputs || batch <= 0) return -1;
+static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
+ if (!ctx || !inputs || batch <= 0)
+ return -1;
const embed_config_t *c = &ctx->config;
int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
- if (!h_offsets) return -1;
+ if (!h_offsets)
+ return -1;
int total = 0, max_seq = 0;
for (int b = 0; b < batch; b++) {
if (!inputs[b].ids || inputs[b].n_tokens <= 0) {
@@ -1218,40 +1210,43 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
}
h_offsets[b] = total;
total += inputs[b].n_tokens;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
}
h_offsets[batch] = total;
int *h_ids = (int *)malloc((size_t)total * sizeof(int));
int *h_pos = (int *)malloc((size_t)total * sizeof(int));
if (!h_offsets || !h_ids || !h_pos) {
- free(h_offsets); free(h_ids); free(h_pos);
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
return -1;
}
for (int b = 0; b < batch; b++) {
int off = h_offsets[b];
- memcpy(h_ids + off, inputs[b].ids,
- (size_t)inputs[b].n_tokens * sizeof(int));
+ memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int));
for (int i = 0; i < inputs[b].n_tokens; i++)
h_pos[off + i] = i;
}
if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
- free(h_offsets); free(h_ids); free(h_pos);
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
return -1;
}
memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int));
- cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids,
- (size_t)total * sizeof(int),
+ cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, (size_t)total * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->positions, h_pos,
- (size_t)total * sizeof(int),
+ ce = cudaMemcpyAsync(ctx->positions, h_pos, (size_t)total * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->offsets, h_offsets,
- (size_t)(batch + 1) * sizeof(int),
+ ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
- free(h_offsets); free(h_ids); free(h_pos);
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
if (ce != cudaSuccess) {
fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
return -1;
@@ -1260,9 +1255,10 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
int threads = 256;
int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16,
- total, c->hidden_size, c->vocab_size);
- if (launch_check() != 0) return -1;
+ ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size,
+ c->vocab_size);
+ if (launch_check() != 0)
+ return -1;
float scale = 1.0f / sqrtf((float)c->head_dim);
int q_offset = 0;
@@ -1280,110 +1276,106 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
cuda_layer_t *l = &ctx->layers[layer];
int wbf16 = l->qkv.bf16;
if (wbf16) {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
- act, ctx->x, l->input_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
- if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv,
- total, c->hidden_size, qkv_dim,
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, total, c->hidden_size, qkv_dim,
c->hidden_size, 0.0f) != 0)
return -1;
} else {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
- ctx->x_norm, ctx->x, l->input_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
- if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv,
- total, c->hidden_size, qkv_dim) != 0)
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm,
+ total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0)
return -1;
}
- rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads),
- 128, 0, ctx->stream>>>(
- ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos,
- ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim,
- qkv_dim, q_offset, k_offset, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
+ rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>(
+ ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total,
+ c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
- if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset,
- k_offset, v_offset, qkv_dim, scale,
- wbf16 ? act : NULL) != 0)
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
+ qkv_dim, scale, wbf16 ? act : NULL) != 0)
return -1;
if (wbf16) {
- if (linear_bf16x(ctx, &l->wo, act, ctx->x,
- total, c->q_dim, c->hidden_size,
- c->q_dim, 1.0f) != 0)
+ if (linear_bf16x(ctx, &l->wo, act, ctx->x, total, c->q_dim, c->hidden_size, c->q_dim,
+ 1.0f) != 0)
return -1;
} else {
- if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
- total, c->q_dim, c->hidden_size) != 0)
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, c->hidden_size) !=
+ 0)
return -1;
}
int inter_count = total * c->intermediate_size;
if (wbf16) {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
- act, ctx->x, l->post_attn_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
- if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up,
- total, c->hidden_size, 2 * c->intermediate_size,
- c->hidden_size, 0.0f) != 0)
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->post_attn_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, total, c->hidden_size,
+ 2 * c->intermediate_size, c->hidden_size, 0.0f) != 0)
return -1;
- silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads,
- threads, 0, ctx->stream>>>(
- act, c->intermediate_size, ctx->ffn_gate_up, total,
- c->intermediate_size);
- if (launch_check() != 0) return -1;
- if (linear_bf16x(ctx, &l->down_proj, act, ctx->x,
- total, c->intermediate_size, c->hidden_size,
- c->intermediate_size, 1.0f) != 0)
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(act, c->intermediate_size, ctx->ffn_gate_up,
+ total, c->intermediate_size);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, total, c->intermediate_size,
+ c->hidden_size, c->intermediate_size, 1.0f) != 0)
return -1;
} else {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
- ctx->x_norm, ctx->x, l->post_attn_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
- if (linear(ctx, &l->gate_up_proj, ctx->x_norm,
- ctx->ffn_gate_up, total, c->hidden_size,
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->post_attn_norm,
+ total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear(ctx, &l->gate_up_proj, ctx->x_norm, ctx->ffn_gate_up, total, c->hidden_size,
2 * c->intermediate_size) != 0)
return -1;
- silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads,
- threads, 0, ctx->stream>>>(
- ctx->ffn_gate_up, 2 * c->intermediate_size, ctx->ffn_gate_up,
- total, c->intermediate_size);
- if (launch_check() != 0) return -1;
- if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x,
- total, c->intermediate_size, c->hidden_size,
- 2 * c->intermediate_size, 1.0f) != 0)
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(ctx->ffn_gate_up, 2 * c->intermediate_size,
+ ctx->ffn_gate_up, total, c->intermediate_size);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total,
+ c->intermediate_size, c->hidden_size, 2 * c->intermediate_size,
+ 1.0f) != 0)
return -1;
}
}
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
- ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0) return -1;
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
return 0;
}
-int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
- int batch, float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
const embed_config_t *c = &ctx->config;
if (cuda_forward_batch(ctx, inputs, batch) != 0)
return -1;
if (ensure_pooled_rows(ctx, batch) != 0)
return -1;
if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
- last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
- ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
+ last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
else
- mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
- ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
- if (launch_check() != 0) return -1;
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
+ if (launch_check() != 0)
+ return -1;
cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
(size_t)batch * c->hidden_size * sizeof(float),
cudaMemcpyDeviceToHost, ctx->stream);
@@ -1395,9 +1387,8 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
}
if (c->normalize_embeddings) {
for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(
- out_embeddings + (size_t)b * c->hidden_size,
- c->hidden_size) != 0)
+ if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) !=
+ 0)
return -1;
}
}
@@ -1405,25 +1396,24 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
}
int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch, float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
const embed_config_t *c = &ctx->config;
int total_spans = 0;
int total_tokens = 0;
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans ||
- input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans ||
- total_tokens > INT_MAX - n_tokens)
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens)
return -1;
for (int s = 0; s < input->n_spans; s++) {
int start = input->spans[s].start;
int len = input->spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens ||
- len > n_tokens - start)
+ if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start)
return -1;
}
total_spans += input->n_spans;
@@ -1457,23 +1447,19 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
cudaError_t ce;
if (cuda_forward_batch(ctx, packed, batch) != 0)
goto cleanup;
- if (ensure_pooled_rows(ctx, total_spans) != 0 ||
- ensure_span_buffers(ctx, total_spans) != 0)
+ if (ensure_pooled_rows(ctx, total_spans) != 0 || ensure_span_buffers(ctx, total_spans) != 0)
goto cleanup;
- ce = cudaMemcpyAsync(ctx->span_starts, h_starts,
- (size_t)total_spans * sizeof(int),
+ ce = cudaMemcpyAsync(ctx->span_starts, h_starts, (size_t)total_spans * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->span_lens, h_lens,
- (size_t)total_spans * sizeof(int),
+ ce = cudaMemcpyAsync(ctx->span_lens, h_lens, (size_t)total_spans * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
if (ce != cudaSuccess) {
fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce));
goto cleanup;
}
span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>(
- ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens,
- total_spans, c->hidden_size);
+ ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, total_spans, c->hidden_size);
if (launch_check() != 0)
goto cleanup;
ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
@@ -1482,8 +1468,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
if (ce == cudaSuccess)
ce = cudaStreamSynchronize(ctx->stream);
if (ce != cudaSuccess) {
- fprintf(stderr, "cuda span output copy failed: %s\n",
- cudaGetErrorString(ce));
+ fprintf(stderr, "cuda span output copy failed: %s\n", cudaGetErrorString(ce));
goto cleanup;
}
rc = 0;
@@ -1495,20 +1480,21 @@ cleanup:
return rc;
}
-int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_embedding)
-{
- embed_input_t input = { token_ids, n_tokens };
+int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
return embed_cuda_encode_batch(ctx, &input, 1, out_embedding);
}
-float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids,
- int n_tokens)
-{
- if (!ctx) return NULL;
+float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) {
+ if (!ctx)
+ return NULL;
int hidden = ctx->config.hidden_size;
float *out = (float *)malloc((size_t)hidden * sizeof(float));
- if (!out) return NULL;
+ if (!out)
+ return NULL;
if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) {
free(out);
return NULL;
diff --git a/embed_cuda.h b/embed_cuda.h
index b3d2c3c..b3c9582 100644
--- a/embed_cuda.h
+++ b/embed_cuda.h
@@ -28,14 +28,18 @@ EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx);
-EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_embedding);
-EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids,
- int n_tokens);
-EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
- int batch, float *out_embeddings);
+EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens);
+EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings);
EMBED_API int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch, float *out_embeddings);
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
#endif /* EMBED_CUDA_H */
diff --git a/embed_internal.h b/embed_internal.h
index 60fc581..d2c1a30 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -29,42 +29,50 @@ typedef struct {
const float *norm; /* [hidden] */
} embed_weights_t;
-typedef void (*embed_attention_fn)(
- float *out, const float *Q, const float *K, const float *V,
- const int *offsets, int batch, int n_heads, int n_kv_heads,
- int head_dim, float scale, float *scratch, size_t scratch_bytes);
+typedef void (*embed_attention_fn)(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
struct embed_model {
- embed_config_t config;
+ embed_config_t config;
embed_weights_t weights;
- void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
+ void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
embed_attention_fn attention;
};
struct embed_workspace {
- const embed_model_t *model; /* immutable model this workspace belongs to */
- int buf_seq_cap;
- float *x; /* [seq, hidden] */
- float *x_norm; /* [seq, hidden] */
- float *q; /* [seq, q_dim] */
- float *k; /* [seq, kv_dim] */
- float *v; /* [seq, kv_dim] */
- float *attn_out; /* [seq, q_dim] */
- float *proj_out; /* [seq, hidden] */
- float *ffn_gate; /* [seq, intermediate] */
- float *ffn_up; /* [seq, intermediate] */
- float *attn_scores; /* reusable BLAS attention score tiles */
+ const embed_model_t *model; /* immutable model this workspace belongs to */
+ int buf_seq_cap;
+ float *x; /* [seq, hidden] */
+ float *x_norm; /* [seq, hidden] */
+ float *q; /* [seq, q_dim] */
+ float *k; /* [seq, kv_dim] */
+ float *v; /* [seq, kv_dim] */
+ float *attn_out; /* [seq, q_dim] */
+ float *proj_out; /* [seq, hidden] */
+ float *ffn_gate; /* [seq, intermediate] */
+ float *ffn_up; /* [seq, intermediate] */
+ float *attn_scores; /* reusable BLAS attention score tiles */
size_t attn_scores_bytes;
- int *offsets; /* [batch + 1] packed sequence offsets */
- int offsets_cap;
+ int *offsets; /* [batch + 1] packed sequence offsets */
+ int offsets_cap;
/* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
float *rope_cos;
float *rope_sin;
- int rope_cache_cap;
+ int rope_cache_cap;
/* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
* largest projection. Workspace-owned so concurrent workspaces on one
diff --git a/embed_mlx.c b/embed_mlx.c
index cb3b737..9f20026 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -35,12 +35,12 @@ typedef struct {
struct embed_mlx_ctx {
embed_config_t config;
mlx_array embed_tokens;
- mlx_layer_t *layers; /* heap-allocated [n_layers] */
+ mlx_layer_t *layers; /* heap-allocated [n_layers] */
mlx_array norm;
mlx_stream stream;
multi_safetensors_t *ms;
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
int quantize_bits;
int quantize_group_size;
mlx_dtype weight_dtype;
@@ -55,7 +55,7 @@ struct embed_mlx_late_ctx {
struct embed_mlx_late_vectors {
embed_mlx_late_ctx_t *owner;
- mlx_array vectors; /* [tokens, dim] */
+ mlx_array vectors; /* [tokens, dim] */
int tokens;
int dim;
};
@@ -66,47 +66,46 @@ struct embed_mlx_late_vectors {
static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-static int mlx_mul_size(size_t a, size_t b, size_t *out)
-{
- if (a != 0 && b > SIZE_MAX / a) return -1;
+static int mlx_mul_size(size_t a, size_t b, size_t *out) {
+ if (a != 0 && b > SIZE_MAX / a)
+ return -1;
*out = a * b;
return 0;
}
-static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx)
-{
- return ctx->weight_dtype;
-}
+static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
-static size_t safetensor_dtype_size(safetensor_dtype_t dtype)
-{
+static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return sizeof(float);
- case DTYPE_BF16: return sizeof(uint16_t);
- default: return 0;
+ case DTYPE_F32:
+ return sizeof(float);
+ case DTYPE_BF16:
+ return sizeof(uint16_t);
+ default:
+ return 0;
}
}
static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
const safetensor_t *t,
- const char *name, const int *shape,
- int ndim)
-{
+ const char *name,
+ const int *shape,
+ int ndim) {
size_t elem_size = safetensor_dtype_size(t->dtype);
if (elem_size == 0) {
fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n",
- name, t->ndim, ndim);
+ fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
size_t numel = 1;
for (int i = 0; i < ndim; i++) {
if (shape[i] <= 0 || t->shape[i] != shape[i]) {
- fprintf(stderr, "mlx: bad shape for %s at dim %d: "
+ fprintf(stderr,
+ "mlx: bad shape for %s at dim %d: "
"got %lld, expected %d\n",
name, i, (long long)t->shape[i], shape[i]);
return 0;
@@ -123,15 +122,15 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
return 0;
}
if (t->data_size != bytes) {
- fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n",
- name, t->data_size, bytes);
+ fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
+ bytes);
return 0;
}
- if (sf->header_size > SIZE_MAX - 8) return 0;
+ if (sf->header_size > SIZE_MAX - 8)
+ return 0;
size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size ||
- t->data_offset > sf->file_size - data_start ||
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name);
return 0;
@@ -139,11 +138,10 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
- const int *shape, int ndim)
-{
+static mlx_array
+load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "mlx: tensor not found: %s\n", name);
return (mlx_array){0};
@@ -162,22 +160,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
}
-static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms,
- const char *name,
- const int *shape, int ndim)
-{
+static mlx_linear_t
+load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
mlx_linear_t l = {0};
l.w = load_tensor(ms, name, shape, ndim);
return l;
}
-static int linear_ok(mlx_linear_t l)
-{
- return arr_ok(l.w) || arr_ok(l.wt);
-}
+static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); }
-static void free_linear(mlx_linear_t *l)
-{
+static void free_linear(mlx_linear_t *l) {
mlx_array_free(l->w);
mlx_array_free(l->wt);
mlx_array_free(l->scales);
@@ -185,8 +177,7 @@ static void free_linear(mlx_linear_t *l)
memset(l, 0, sizeof(*l));
}
-static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
-{
+static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) {
if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
return 0;
@@ -214,15 +205,15 @@ static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
return 0;
}
-static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
- mlx_stream s)
-{
- if (!l || !arr_ok(l->w) || bits == 0) return 0;
+static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) {
+ if (!l || !arr_ok(l->w) || bits == 0)
+ return 0;
if (bits != 8) {
fprintf(stderr, "mlx: quantized weights support only 8 bits\n");
return -1;
}
- if (group_size <= 0) group_size = 64;
+ if (group_size <= 0)
+ group_size = 64;
mlx_optional_int gs = {.value = group_size, .has_value = true};
mlx_optional_int qb = {.value = bits, .has_value = true};
@@ -253,7 +244,8 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
}
mlx_array_eval(qw);
mlx_array_eval(scales);
- if (arr_ok(biases)) mlx_array_eval(biases);
+ if (arr_ok(biases))
+ mlx_array_eval(biases);
mlx_synchronize(s);
mlx_array_free(l->w);
@@ -269,16 +261,15 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
}
/* y = x @ W^T */
-static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
-{
+static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
if (W->quantized) {
- if (!arr_ok(W->w)) return (mlx_array){0};
+ if (!arr_ok(W->w))
+ return (mlx_array){0};
mlx_array res = mlx_array_new();
mlx_optional_int gs = {.value = W->group_size, .has_value = true};
mlx_optional_int bits = {.value = W->bits, .has_value = true};
mlx_array null_arr = (mlx_array){0};
- mlx_quantized_matmul(&res, x, W->w, W->scales,
- arr_ok(W->biases) ? W->biases : null_arr,
+ mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr,
true, gs, bits, "affine", s);
return res;
}
@@ -294,8 +285,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
return (mlx_array){0};
}
mlx_array res = mlx_array_new();
- if (mlx_array_dtype(Wt) == MLX_BFLOAT16 &&
- mlx_array_dtype(x) == MLX_FLOAT32) {
+ if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) {
/* F32 activations with BF16 weights: upcast the weight operand and run
* the matmul in F32 (so accumulation is F32). Only the weights are
* BF16-rounded; the residual stream stays F32 (see mlx_forward_layers),
@@ -307,21 +297,21 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
} else {
mlx_matmul(&res, x, Wt, s);
}
- if (free_wt) mlx_array_free(Wt);
+ if (free_wt)
+ mlx_array_free(Wt);
return res;
}
-const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx)
-{
+const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) {
return ctx ? &ctx->config : NULL;
}
/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
* tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
* residual stream in F32 while keeping the large matmul weights in BF16. */
-static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
-{
- if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return;
+static void upcast_f32_inplace(mlx_array *a, mlx_stream s) {
+ if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32)
+ return;
mlx_array f = mlx_array_new();
mlx_astype(&f, *a, MLX_FLOAT32, s);
mlx_array_free(*a);
@@ -335,14 +325,13 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
/* We duplicate a minimal config parser here so MLX can load standalone.
* (Same logic as embed.c:parse_config, kept inline for simplicity.) */
-static const char *skip_ws_m(const char *p)
-{
- while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+static const char *skip_ws_m(const char *p) {
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
+ p++;
return p;
}
-static const char *json_find_m(const char *json, const char *key)
-{
+static const char *json_find_m(const char *json, const char *key) {
size_t klen = strlen(key);
const char *p = json;
while ((p = strstr(p, key)) != NULL) {
@@ -350,7 +339,8 @@ static const char *json_find_m(const char *json, const char *key)
const char *a = p + klen;
if (*a == '"') {
a = skip_ws_m(a + 1);
- if (*a == ':') return skip_ws_m(a + 1);
+ if (*a == ':')
+ return skip_ws_m(a + 1);
}
}
p += klen;
@@ -358,18 +348,16 @@ static const char *json_find_m(const char *json, const char *key)
return NULL;
}
-static int json_string_equals_m(const char *json, const char *key,
- const char *expected)
-{
+static int json_string_equals_m(const char *json, const char *key, const char *expected) {
const char *v = json_find_m(json, key);
- if (!v || *v != '"') return 0;
+ if (!v || *v != '"')
+ return 0;
v++;
size_t len = strlen(expected);
return strncmp(v, expected, len) == 0 && v[len] == '"';
}
-static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
-{
+static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) {
char path[1024];
int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (path_len < 0 || (size_t)path_len >= sizeof(path)) {
@@ -377,8 +365,14 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
return -1;
}
FILE *f = fopen(path, "rb");
- if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; }
- if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; }
+ if (!f) {
+ fprintf(stderr, "mlx: cannot open %s\n", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
long sz = ftell(f);
if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
fclose(f);
@@ -386,22 +380,32 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
}
char *buf = (char *)malloc((size_t)sz + 1);
if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f); free(buf); return -1;
+ fclose(f);
+ free(buf);
+ return -1;
}
fclose(f);
buf[sz] = '\0';
-#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0)
-#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0)
- GI("hidden_size", hidden_size, 0);
- GI("num_hidden_layers", n_layers, 0);
- GI("num_attention_heads", n_heads, 0);
- GI("num_key_value_heads", n_kv_heads, 0);
- GI("head_dim", head_dim, EMBED_HEAD_DIM);
- GI("intermediate_size", intermediate_size, 0);
- GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE);
- GF("rms_norm_eps", rms_norm_eps, 1e-6f);
- GF("rope_theta", rope_theta, 1000000.0f);
+#define GI(k, fb, def) \
+ do { \
+ const char *v = json_find_m(buf, k); \
+ cfg->fb = v ? atoi(v) : def; \
+ } while (0)
+#define GF(k, fb, def) \
+ do { \
+ const char *v = json_find_m(buf, k); \
+ cfg->fb = v ? (float)atof(v) : def; \
+ } while (0)
+ GI("hidden_size", hidden_size, 0);
+ GI("num_hidden_layers", n_layers, 0);
+ GI("num_attention_heads", n_heads, 0);
+ GI("num_key_value_heads", n_kv_heads, 0);
+ GI("head_dim", head_dim, EMBED_HEAD_DIM);
+ GI("intermediate_size", intermediate_size, 0);
+ GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE);
+ GF("rms_norm_eps", rms_norm_eps, 1e-6f);
+ GF("rope_theta", rope_theta, 1000000.0f);
#undef GI
#undef GF
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
@@ -417,58 +421,53 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
cfg->q_dim = 0;
cfg->kv_dim = 0;
- if (cfg->n_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim)
cfg->q_dim = cfg->n_heads * cfg->head_dim;
- if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
free(buf);
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 ||
- cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
- cfg->head_dim <= 0 || cfg->intermediate_size <= 0 ||
- cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
- (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
- !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
- fprintf(stderr, "mlx: invalid config in %s "
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
+ cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
+ cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
+ cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
+ cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ fprintf(stderr,
+ "mlx: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
- path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
- cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size);
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
+ cfg->intermediate_size);
return -1;
}
if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "mlx: too many layers (%d > %d)\n",
- cfg->n_layers, EMBED_MAX_LAYERS);
+ fprintf(stderr, "mlx: too many layers (%d > %d)\n", cfg->n_layers, EMBED_MAX_LAYERS);
return -1;
}
return 0;
}
-static int mlx_model_dir_has_file(const char *model_dir, const char *rel)
-{
+static int mlx_model_dir_has_file(const char *model_dir, const char *rel) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path)) return 0;
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
FILE *f = fopen(path, "rb");
- if (!f) return 0;
+ if (!f)
+ return 0;
fclose(f);
return 1;
}
-static int mlx_model_dir_has_late_projection(const char *model_dir)
-{
+static int mlx_model_dir_has_late_projection(const char *model_dir) {
return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") &&
mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts)
-{
+static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) {
embed_mlx_options_t out = {0};
- if (opts) out = *opts;
- if (out.quantize_bits != 0 &&
- out.quantize_bits != 8) {
+ if (opts)
+ out = *opts;
+ if (out.quantize_bits != 0 && out.quantize_bits != 8) {
fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
out.quantize_bits = -1;
}
@@ -477,10 +476,9 @@ static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts
return out;
}
-static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
- mlx_stream s)
-{
- if (bits == 0) return 0;
+static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) {
+ if (bits == 0)
+ return 0;
return quantize_linear(&l->wq, bits, group_size, s) ||
quantize_linear(&l->wk, bits, group_size, s) ||
quantize_linear(&l->wv, bits, group_size, s) ||
@@ -490,45 +488,45 @@ static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
quantize_linear(&l->down_proj, bits, group_size, s);
}
-static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s)
-{
- return prepare_linear_transpose(&l->wq, s) ||
- prepare_linear_transpose(&l->wk, s) ||
- prepare_linear_transpose(&l->wv, s) ||
- prepare_linear_transpose(&l->wo, s) ||
- prepare_linear_transpose(&l->gate_proj, s) ||
- prepare_linear_transpose(&l->up_proj, s) ||
+static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) {
+ return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) ||
+ prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) ||
+ prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) ||
prepare_linear_transpose(&l->down_proj, s);
}
static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start, int layer_end,
- const embed_mlx_options_t *opts,
- int allow_late)
-{
+ int layer_start,
+ int layer_end,
+ const embed_mlx_options_t *opts,
+ int allow_late) {
embed_mlx_options_t options = normalize_mlx_options(opts);
- if (options.quantize_bits < 0) return NULL;
+ if (options.quantize_bits < 0)
+ return NULL;
if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx: late-interaction models require token-level "
- "MaxSim support and are not valid pooled embedding models "
- "yet\n");
+ "MaxSim support and are not valid pooled embedding models "
+ "yet\n");
return NULL;
}
embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t));
- if (!ctx) return NULL;
+ if (!ctx)
+ return NULL;
ctx->quantize_bits = options.quantize_bits;
ctx->quantize_group_size = options.quantize_group_size;
- if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
+ if (mlx_parse_config(&ctx->config, model_dir) != 0) {
+ free(ctx);
+ return NULL;
+ }
const embed_config_t *c = &ctx->config;
if (layer_end == -1)
layer_end = c->n_layers;
- if (layer_start < 0 || layer_start >= layer_end ||
- layer_end > c->n_layers) {
- fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n",
- layer_start, layer_end, c->n_layers);
+ if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) {
+ fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start,
+ layer_end, c->n_layers);
free(ctx);
return NULL;
}
@@ -538,7 +536,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
ctx->ms = multi_safetensors_open(model_dir);
if (!ctx->ms) {
fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx); return NULL;
+ free(ctx);
+ return NULL;
}
int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
@@ -547,64 +546,73 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
/* Only the first stage needs token embeddings. */
if (layer_start == 0) {
int emb_shape[] = {c->vocab_size, h};
- ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight",
- emb_shape, 2);
- if (!arr_ok(ctx->embed_tokens)) goto fail;
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens))
+ goto fail;
}
/* Layers */
ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers) goto fail;
+ if (!ctx->layers)
+ goto fail;
char name[256];
for (int i = layer_start; i < layer_end; i++) {
mlx_layer_t *l = &ctx->layers[i];
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!linear_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
- LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
- LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
- LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
+ LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
+ LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
+ LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
+ LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
#undef LD
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!arr_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!arr_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
- LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
- LD(input_norm, "layers.%d.input_layernorm.weight", h);
+ LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
+ LD(input_norm, "layers.%d.input_layernorm.weight", h);
LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h);
#undef LD
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!linear_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
- LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
- LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
+ LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
+ LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
+ LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
#undef LD
- if (embed_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
+ if (embed_verbose >= 2)
+ fprintf(stderr, " mlx layer %d loaded\n", i);
}
if (layer_end == c->n_layers) {
int ns[] = {h};
ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
- if (!arr_ok(ctx->norm)) goto fail;
+ if (!arr_ok(ctx->norm))
+ goto fail;
}
ctx->stream = mlx_default_gpu_stream_new();
@@ -624,13 +632,14 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
}
if (ctx->quantize_bits) {
for (int i = layer_start; i < layer_end; i++) {
- if (quantize_layer(&ctx->layers[i], ctx->quantize_bits,
- ctx->quantize_group_size, ctx->stream) != 0)
+ if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size,
+ ctx->stream) != 0)
goto fail;
}
mlx_clear_cache();
if (embed_verbose >= 1)
- fprintf(stderr, "mlx: quantized linear weights to %d-bit "
+ fprintf(stderr,
+ "mlx: quantized linear weights to %d-bit "
"(group_size=%d)\n",
ctx->quantize_bits, ctx->quantize_group_size);
} else {
@@ -645,7 +654,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
mlx_get_active_memory(&active);
mlx_get_cache_memory(&cache);
mlx_get_peak_memory(&peak);
- fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB "
+ fprintf(stderr,
+ "mlx: memory active=%.2f GiB cache=%.2f GiB "
"peak=%.2f GiB\n",
(double)active / (1024.0 * 1024.0 * 1024.0),
(double)cache / (1024.0 * 1024.0 * 1024.0),
@@ -653,8 +663,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
}
if (embed_verbose >= 1)
- fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n",
- layer_start, layer_end, h);
+ fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h);
return ctx;
@@ -663,14 +672,12 @@ fail:
return NULL;
}
-embed_mlx_ctx_t *embed_mlx_load(const char *model_dir)
-{
+embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) {
return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
}
embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts)
-{
+ const embed_mlx_options_t *opts) {
return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
}
@@ -678,19 +685,23 @@ embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
* Free
* ======================================================================== */
-static void free_mlx_layer(mlx_layer_t *l)
-{
- free_linear(&l->wq); free_linear(&l->wk);
- free_linear(&l->wv); free_linear(&l->wo);
- mlx_array_free(l->q_norm); mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm);
- free_linear(&l->gate_proj); free_linear(&l->up_proj);
+static void free_mlx_layer(mlx_layer_t *l) {
+ free_linear(&l->wq);
+ free_linear(&l->wk);
+ free_linear(&l->wv);
+ free_linear(&l->wo);
+ mlx_array_free(l->q_norm);
+ mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm);
+ mlx_array_free(l->post_attn_norm);
+ free_linear(&l->gate_proj);
+ free_linear(&l->up_proj);
free_linear(&l->down_proj);
}
-void embed_mlx_free(embed_mlx_ctx_t *ctx)
-{
- if (!ctx) return;
+void embed_mlx_free(embed_mlx_ctx_t *ctx) {
+ if (!ctx)
+ return;
mlx_array_free(ctx->embed_tokens);
if (ctx->layers) {
for (int i = 0; i < ctx->config.n_layers; i++)
@@ -698,24 +709,27 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx)
free(ctx->layers);
}
mlx_array_free(ctx->norm);
- if (ctx->stream.ctx) mlx_stream_free(ctx->stream);
- if (ctx->ms) multi_safetensors_close(ctx->ms);
+ if (ctx->stream.ctx)
+ mlx_stream_free(ctx->stream);
+ if (ctx->ms)
+ multi_safetensors_close(ctx->ms);
free(ctx);
}
-embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
- const char *model_dir, const embed_mlx_options_t *opts)
-{
+embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
+ const embed_mlx_options_t *opts) {
if (!mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx late: missing 1_Dense projection\n");
return NULL;
}
embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
- if (!late) return NULL;
+ if (!late)
+ return NULL;
late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1);
- if (!late->base) goto fail;
+ if (!late->base)
+ goto fail;
char dense_dir[1024];
int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir);
@@ -726,13 +740,11 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
late->dense_ms = multi_safetensors_open(dense_dir);
if (!late->dense_ms) {
- fprintf(stderr, "mlx late: failed to open safetensors in %s\n",
- dense_dir);
+ fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir);
goto fail;
}
- const safetensor_t *t = multi_safetensors_find(late->dense_ms,
- "linear.weight", NULL);
+ const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL);
if (!t) {
fprintf(stderr, "mlx late: missing linear.weight\n");
goto fail;
@@ -743,17 +755,16 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
goto fail;
}
- int shape[] = { (int)t->shape[0], late->base->config.hidden_size };
- late->projection = load_linear_tensor(late->dense_ms, "linear.weight",
- shape, 2);
- if (!linear_ok(late->projection)) goto fail;
+ int shape[] = {(int)t->shape[0], late->base->config.hidden_size};
+ late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2);
+ if (!linear_ok(late->projection))
+ goto fail;
if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
goto fail;
late->token_dim = shape[0];
if (embed_verbose >= 1)
- fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n",
- late->token_dim);
+ fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim);
return late;
fail:
@@ -761,39 +772,38 @@ fail:
return NULL;
}
-embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir)
-{
+embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) {
return embed_mlx_late_load_with_options(model_dir, NULL);
}
-void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx)
-{
- if (!ctx) return;
+void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) {
+ if (!ctx)
+ return;
free_linear(&ctx->projection);
- if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms);
+ if (ctx->dense_ms)
+ multi_safetensors_close(ctx->dense_ms);
embed_mlx_free(ctx->base);
free(ctx);
}
-const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx)
-{
+const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) {
return ctx && ctx->base ? &ctx->base->config : NULL;
}
-int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx)
-{
- return ctx ? ctx->token_dim : 0;
-}
+int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
/* ========================================================================
* Forward pass
* ======================================================================== */
-static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
- int batch, int max_seq, int has_padding,
+static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
+ mlx_array x,
+ int batch,
+ int max_seq,
+ int has_padding,
mlx_array attn_mask,
- int layer_start, int layer_end)
-{
+ int layer_start,
+ int layer_end) {
const embed_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int n_heads = c->n_heads;
@@ -802,8 +812,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
int q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
- const char *mask_mode =
- c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
+ const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
/* For non-quantized BF16 weights, run the residual stream, norms, and
* attention in F32; matmuls downcast their operands to BF16 internally
@@ -812,8 +821,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
* models are unaffected. */
mlx_array local_mask = attn_mask;
int free_mask = 0;
- int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 &&
- !ctx->layers[layer_start].wq.quantized;
+ int mixed_bf16 =
+ mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
if (mixed_bf16) {
if (mlx_array_dtype(x) != MLX_FLOAT32) {
mlx_array xf = mlx_array_new();
@@ -821,8 +830,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array_free(x);
x = xf;
}
- if (has_padding && arr_ok(attn_mask) &&
- mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
+ if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
local_mask = mlx_array_new();
mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S);
free_mask = 1;
@@ -840,7 +848,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array v_flat = linear(xn, &l->wv, S);
mlx_array_free(xn);
- int q_shape[] = {batch, max_seq, n_heads, head_dim};
+ int q_shape[] = {batch, max_seq, n_heads, head_dim};
int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
mlx_array q = mlx_array_new();
mlx_array k = mlx_array_new();
@@ -848,33 +856,40 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_reshape(&q, q_flat, q_shape, 4, S);
mlx_reshape(&k, k_flat, k_shape, 4, S);
mlx_reshape(&v, v_flat, k_shape, 4, S);
- mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
+ mlx_array_free(q_flat);
+ mlx_array_free(k_flat);
+ mlx_array_free(v_flat);
mlx_array qn = mlx_array_new();
mlx_array kn = mlx_array_new();
mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q); mlx_array_free(k);
+ mlx_array_free(q);
+ mlx_array_free(k);
/* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
int perm[] = {0, 2, 1, 3};
mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
mlx_transpose_axes(&qt, qn, perm, 4, S);
mlx_transpose_axes(&kt, kn, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v);
+ mlx_transpose_axes(&vt, v, perm, 4, S);
+ mlx_array_free(qn);
+ mlx_array_free(kn);
+ mlx_array_free(v);
mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
mlx_array qr = mlx_array_new(), kr = mlx_array_new();
mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_array_free(qt); mlx_array_free(kt);
+ mlx_array_free(qt);
+ mlx_array_free(kt);
mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale, mask_mode,
- has_padding ? local_mask : null_arr, null_arr, S);
- mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
+ mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode,
+ has_padding ? local_mask : null_arr, null_arr, S);
+ mlx_array_free(qr);
+ mlx_array_free(kr);
+ mlx_array_free(vt);
mlx_array attn_t = mlx_array_new();
mlx_transpose_axes(&attn_t, attn, perm, 4, S);
@@ -889,44 +904,49 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array_free(attn_flat);
mlx_array x2 = mlx_array_new();
mlx_add(&x2, x, proj, S);
- mlx_array_free(x); mlx_array_free(proj);
+ mlx_array_free(x);
+ mlx_array_free(proj);
x = x2;
mlx_array xn2 = mlx_array_new();
mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
mlx_array gate = linear(xn2, &l->gate_proj, S);
- mlx_array up = linear(xn2, &l->up_proj, S);
+ mlx_array up = linear(xn2, &l->up_proj, S);
mlx_array_free(xn2);
mlx_array gate_sig = mlx_array_new();
mlx_sigmoid(&gate_sig, gate, S);
mlx_array silu_gate = mlx_array_new();
mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate); mlx_array_free(gate_sig);
+ mlx_array_free(gate);
+ mlx_array_free(gate_sig);
mlx_array mid = mlx_array_new();
mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate); mlx_array_free(up);
+ mlx_array_free(silu_gate);
+ mlx_array_free(up);
mlx_array ffn = linear(mid, &l->down_proj, S);
mlx_array_free(mid);
mlx_array x3 = mlx_array_new();
mlx_add(&x3, x, ffn, S);
- mlx_array_free(x); mlx_array_free(ffn);
+ mlx_array_free(x);
+ mlx_array_free(ffn);
x = x3;
}
- if (free_mask) mlx_array_free(local_mask);
+ if (free_mask)
+ mlx_array_free(local_mask);
return x;
}
-embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
- embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
- int normalize)
-{
- if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 ||
- ctx->token_dim <= 0 || !linear_ok(ctx->projection))
+embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize) {
+ if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
+ !linear_ok(ctx->projection))
return NULL;
embed_mlx_ctx_t *base = ctx->base;
@@ -938,8 +958,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_stream S = base->stream;
for (int i = 0; i < n_tokens; i++) {
if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at position %d\n",
- token_ids[i], i);
+ fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i);
return NULL;
}
}
@@ -949,13 +968,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
return NULL;
int *ids_data = (int *)malloc(id_bytes);
- if (!ids_data) return NULL;
+ if (!ids_data)
+ return NULL;
memcpy(ids_data, token_ids, id_bytes);
int ids_shape[] = {1, n_tokens};
mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
free(ids_data);
- if (!arr_ok(ids)) return NULL;
+ if (!arr_ok(ids))
+ return NULL;
embed_mlx_late_vectors_t *result = NULL;
mlx_array x = mlx_array_new();
@@ -965,22 +986,25 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
mlx_array_free(ids);
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
- x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0},
- 0, c->n_layers);
- if (!arr_ok(x)) goto cleanup;
+ x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers);
+ if (!arr_ok(x))
+ goto cleanup;
x_normed = mlx_array_new();
mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
mlx_array_free(x);
x = (mlx_array){0};
- if (!arr_ok(x_normed)) goto cleanup;
+ if (!arr_ok(x_normed))
+ goto cleanup;
projected = linear(x_normed, &ctx->projection, S);
mlx_array_free(x_normed);
x_normed = (mlx_array){0};
- if (!arr_ok(projected)) goto cleanup;
+ if (!arr_ok(projected))
+ goto cleanup;
if (normalize) {
mlx_array sq = mlx_array_new();
@@ -1011,16 +1035,19 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_array_free(inv_norm);
mlx_array_free(projected);
projected = normalized;
- if (!arr_ok(projected)) goto cleanup;
+ if (!arr_ok(projected))
+ goto cleanup;
}
int vec_shape[] = {n_tokens, ctx->token_dim};
reshaped = mlx_array_new();
mlx_reshape(&reshaped, projected, vec_shape, 2, S);
- if (!arr_ok(reshaped)) goto cleanup;
+ if (!arr_ok(reshaped))
+ goto cleanup;
result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result) goto cleanup;
+ if (!result)
+ goto cleanup;
result->owner = ctx;
result->vectors = reshaped;
result->tokens = n_tokens;
@@ -1035,28 +1062,23 @@ cleanup:
return result;
}
-void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs)
-{
- if (!vecs) return;
+void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) {
+ if (!vecs)
+ return;
mlx_array_free(vecs->vectors);
free(vecs);
}
-int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs)
-{
+int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) {
return vecs ? vecs->tokens : 0;
}
-int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs)
-{
+int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) {
return vecs ? vecs->dim : 0;
}
-int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs,
- float *out_vectors)
-{
- if (!vecs || !out_vectors || !arr_ok(vecs->vectors) ||
- vecs->tokens <= 0 || vecs->dim <= 0)
+int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) {
+ if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
return -1;
size_t values, bytes;
@@ -1084,32 +1106,33 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs,
}
embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *const *items, int count)
-{
- if (!ctx || !ctx->base || !items || count <= 0) return NULL;
+ embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) {
+ if (!ctx || !ctx->base || !items || count <= 0)
+ return NULL;
int dim = 0;
int total = 0;
for (int i = 0; i < count; i++) {
const embed_mlx_late_vectors_t *v = items[i];
- if (!v || v->owner != ctx || !arr_ok(v->vectors) ||
- v->tokens <= 0 || v->dim <= 0)
+ if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0)
return NULL;
- if (i == 0) dim = v->dim;
+ if (i == 0)
+ dim = v->dim;
if (v->dim != dim || total > INT_MAX - v->tokens)
return NULL;
total += v->tokens;
}
mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
- if (!parts) return NULL;
+ if (!parts)
+ return NULL;
for (int i = 0; i < count; i++)
parts[i] = items[i]->vectors;
mlx_stream S = ctx->base->stream;
mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
free(parts);
- if (!vec.ctx) return NULL;
+ if (!vec.ctx)
+ return NULL;
mlx_array joined = mlx_array_new();
mlx_concatenate_axis(&joined, vec, 0, S);
@@ -1119,8 +1142,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return NULL;
}
- embed_mlx_late_vectors_t *out =
- (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(joined);
return NULL;
@@ -1132,13 +1154,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return out;
}
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs,
- const int *token_indices, int count)
-{
- if (!ctx || !ctx->base || !vecs || vecs->owner != ctx ||
- !arr_ok(vecs->vectors) || !token_indices || count <= 0 ||
- vecs->tokens <= 0 || vecs->dim <= 0)
+embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count) {
+ if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
+ !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
return NULL;
for (int i = 0; i < count; i++) {
@@ -1150,13 +1171,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
return NULL;
int *idx_data = (int *)malloc(idx_bytes);
- if (!idx_data) return NULL;
+ if (!idx_data)
+ return NULL;
memcpy(idx_data, token_indices, idx_bytes);
int idx_shape[] = {count};
mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
free(idx_data);
- if (!arr_ok(idx)) return NULL;
+ if (!arr_ok(idx))
+ return NULL;
mlx_stream S = ctx->base->stream;
mlx_array selected = mlx_array_new();
@@ -1167,8 +1190,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
return NULL;
}
- embed_mlx_late_vectors_t *out =
- (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(selected);
return NULL;
@@ -1180,21 +1202,21 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
return out;
}
-int embed_mlx_late_maxsim_batch_device(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs, const int *doc_offsets,
- int docs_count, float *scores)
-{
- if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 ||
- !scores || query->owner != ctx || docs->owner != ctx ||
- query->dim <= 0 || query->dim != docs->dim ||
- query->tokens <= 0 || docs->tokens <= 0 ||
- !arr_ok(query->vectors) || !arr_ok(docs->vectors) ||
- doc_offsets[0] != 0)
+int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *query,
+ const embed_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores) {
+ if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
+ query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
+ query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
+ !arr_ok(docs->vectors) || doc_offsets[0] != 0)
return -1;
int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
- if (!lengths) return -1;
+ if (!lengths)
+ return -1;
int max_doc_tokens = 0;
for (int i = 0; i < docs_count; i++) {
if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
@@ -1204,7 +1226,8 @@ int embed_mlx_late_maxsim_batch_device(
}
int n = doc_offsets[i + 1] - doc_offsets[i];
lengths[i] = n;
- if (n > max_doc_tokens) max_doc_tokens = n;
+ if (n > max_doc_tokens)
+ max_doc_tokens = n;
}
if (max_doc_tokens <= 0) {
free(lengths);
@@ -1214,8 +1237,7 @@ int embed_mlx_late_maxsim_batch_device(
mlx_stream S = ctx->base->stream;
int dim = query->dim;
int rc = -1;
- mlx_array *doc_rows =
- (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
+ mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
if (!doc_rows) {
free(lengths);
return -1;
@@ -1250,95 +1272,114 @@ int embed_mlx_late_maxsim_batch_device(
mlx_array padded = (mlx_array){0};
mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
- if (!arr_ok(doc)) goto cleanup;
+ if (!arr_ok(doc))
+ goto cleanup;
if (n < max_doc_tokens) {
int axes[] = {0};
int low[] = {0};
int high[] = {max_doc_tokens - n};
padded = mlx_array_new();
- mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value,
- "constant", S);
+ mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S);
mlx_array_free(doc);
doc = padded;
padded = (mlx_array){0};
- if (!arr_ok(doc)) goto cleanup;
+ if (!arr_ok(doc))
+ goto cleanup;
}
doc_rows[i] = mlx_array_new();
mlx_expand_dims(&doc_rows[i], doc, 0, S);
mlx_array_free(doc);
- if (!arr_ok(doc_rows[i])) goto cleanup;
+ if (!arr_ok(doc_rows[i]))
+ goto cleanup;
}
doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
- if (!doc_vec.ctx) goto cleanup;
+ if (!doc_vec.ctx)
+ goto cleanup;
doc_batch = mlx_array_new();
mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
- if (!arr_ok(doc_batch)) goto cleanup;
+ if (!arr_ok(doc_batch))
+ goto cleanup;
query_3d = mlx_array_new();
mlx_expand_dims(&query_3d, query->vectors, 0, S);
- if (!arr_ok(query_3d)) goto cleanup;
+ if (!arr_ok(query_3d))
+ goto cleanup;
int q_shape[] = {docs_count, query->tokens, dim};
query_batched = mlx_array_new();
mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
- if (!arr_ok(query_batched)) goto cleanup;
+ if (!arr_ok(query_batched))
+ goto cleanup;
int perm[] = {0, 2, 1};
docs_t = mlx_array_new();
mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
- if (!arr_ok(docs_t)) goto cleanup;
+ if (!arr_ok(docs_t))
+ goto cleanup;
sims = mlx_array_new();
mlx_matmul(&sims, query_batched, docs_t, S);
- if (!arr_ok(sims)) goto cleanup;
+ if (!arr_ok(sims))
+ goto cleanup;
int len_shape[] = {docs_count};
lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
- if (!arr_ok(lengths_arr)) goto cleanup;
+ if (!arr_ok(lengths_arr))
+ goto cleanup;
positions = mlx_array_new();
mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
- if (!arr_ok(positions)) goto cleanup;
+ if (!arr_ok(positions))
+ goto cleanup;
lengths_2d = mlx_array_new();
mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
- if (!arr_ok(lengths_2d)) goto cleanup;
+ if (!arr_ok(lengths_2d))
+ goto cleanup;
positions_2d = mlx_array_new();
mlx_expand_dims(&positions_2d, positions, 0, S);
- if (!arr_ok(positions_2d)) goto cleanup;
+ if (!arr_ok(positions_2d))
+ goto cleanup;
valid = mlx_array_new();
mlx_less(&valid, positions_2d, lengths_2d, S);
- if (!arr_ok(valid)) goto cleanup;
+ if (!arr_ok(valid))
+ goto cleanup;
valid_3d = mlx_array_new();
mlx_expand_dims(&valid_3d, valid, 1, S);
- if (!arr_ok(valid_3d)) goto cleanup;
+ if (!arr_ok(valid_3d))
+ goto cleanup;
masked_sims = mlx_array_new();
mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
- if (!arr_ok(masked_sims)) goto cleanup;
+ if (!arr_ok(masked_sims))
+ goto cleanup;
maxes = mlx_array_new();
mlx_max_axis(&maxes, masked_sims, 2, false, S);
- if (!arr_ok(maxes)) goto cleanup;
+ if (!arr_ok(maxes))
+ goto cleanup;
sums = mlx_array_new();
mlx_sum_axis(&sums, maxes, 1, false, S);
- if (!arr_ok(sums)) goto cleanup;
+ if (!arr_ok(sums))
+ goto cleanup;
sums_f32 = mlx_array_new();
mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
- if (!arr_ok(sums_f32)) goto cleanup;
+ if (!arr_ok(sums_f32))
+ goto cleanup;
mlx_array_eval(sums_f32);
const float *data = mlx_array_data_float32(sums_f32);
- if (!data) goto cleanup;
+ if (!data)
+ goto cleanup;
memcpy(scores, data, (size_t)docs_count * sizeof(float));
rc = 0;
@@ -1359,7 +1400,8 @@ cleanup:
mlx_array_free(query_batched);
mlx_array_free(query_3d);
mlx_array_free(doc_batch);
- if (doc_vec.ctx) mlx_vector_array_free(doc_vec);
+ if (doc_vec.ctx)
+ mlx_vector_array_free(doc_vec);
mlx_array_free(pad_value);
for (int i = 0; i < docs_count; i++)
mlx_array_free(doc_rows[i]);
@@ -1369,31 +1411,32 @@ cleanup:
}
int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors)
-{
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
embed_mlx_late_vectors_t *vecs =
- embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens,
- normalize);
- if (!vecs) return -1;
+ embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
+ if (!vecs)
+ return -1;
int rc = embed_mlx_late_vectors_copy(vecs, out_vectors);
embed_mlx_late_vectors_free(vecs);
return rc;
}
static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs, int batch,
- float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
- ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers)
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
+ ctx->layer_end != ctx->config.n_layers)
return -1;
const embed_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int max_seq = 0;
+ int hidden = c->hidden_size;
+ int max_seq = 0;
int has_padding = 0;
int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN;
@@ -1401,13 +1444,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
return -1;
for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return -1;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
}
- if (max_seq <= 0) return -1;
+ if (max_seq <= 0)
+ return -1;
for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ if (inputs[b].n_tokens != max_seq)
+ has_padding = 1;
}
int needs_pool_mask = has_padding && !last_token_pool;
int needs_attn_mask = has_padding && !causal;
@@ -1420,24 +1467,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
- (last_token_pool &&
- mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
+ (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
return -1;
if (ids_bytes == 0 || out_bytes == 0 ||
(needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
- (needs_attn_mask && mask_bytes == 0) ||
- (last_token_pool && last_index_bytes == 0))
+ (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0))
return -1;
int *padded_ids = (int *)malloc(ids_bytes);
- float *pool_mask_data =
- needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
- float *attn_mask_data =
- needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
- int *last_index_data =
- last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
- if (!padded_ids ||
- (needs_pool_mask && (!pool_mask_data || !len_data)) ||
+ int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
+ if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) ||
(needs_attn_mask && !attn_mask_data)) {
free(padded_ids);
free(pool_mask_data);
@@ -1455,7 +1496,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
}
for (int b = 0; b < batch; b++) {
- if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens;
+ if (needs_pool_mask)
+ len_data[b] = (float)inputs[b].n_tokens;
if (last_token_pool) {
int index = inputs[b].n_tokens - 1;
for (int d = 0; d < hidden; d++)
@@ -1466,8 +1508,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (t < inputs[b].n_tokens) {
int id = inputs[b].ids[t];
if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- id, b, t);
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t);
free(padded_ids);
free(pool_mask_data);
free(attn_mask_data);
@@ -1498,25 +1539,26 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
mlx_array pool_mask = needs_pool_mask
- ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
- : (mlx_array){0};
+ ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
+ : (mlx_array){0};
mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array lengths = needs_pool_mask
- ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array last_indices = last_token_pool
- ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
- free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array lengths =
+ needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
+ mlx_array last_indices =
+ last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
+ : (mlx_array){0};
+ free(padded_ids);
+ free(pool_mask_data);
+ free(attn_mask_data);
+ free(len_data);
free(last_index_data);
- if (!arr_ok(ids) ||
- (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
- (needs_attn_mask && !arr_ok(attn_mask)) ||
- (last_token_pool && !arr_ok(last_indices))) {
- mlx_array_free(ids); mlx_array_free(pool_mask);
+ if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
+ (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) {
+ mlx_array_free(ids);
+ mlx_array_free(pool_mask);
mlx_array_free(attn_mask);
mlx_array_free(lengths);
mlx_array_free(last_indices);
@@ -1528,7 +1570,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(attn_mask);
attn_mask = attn_mask_bf16;
if (!arr_ok(attn_mask)) {
- mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ mlx_array_free(ids);
+ mlx_array_free(pool_mask);
+ mlx_array_free(lengths);
mlx_array_free(last_indices);
return -1;
}
@@ -1540,9 +1584,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(ids);
/* 2. Transformer layers. */
- x = mlx_forward_layers(ctx, x, batch, max_seq,
- needs_attn_mask, attn_mask,
- 0, c->n_layers);
+ x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
/* 3. Final RMSNorm. */
mlx_array x_normed = mlx_array_new();
@@ -1573,7 +1615,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(masked);
mlx_divide(&emb, emb_sum, lengths, S);
- mlx_array_free(emb_sum); mlx_array_free(lengths);
+ mlx_array_free(emb_sum);
+ mlx_array_free(lengths);
mlx_array_free(attn_mask);
} else {
/* Equal-length batches do not need masks or a separate divide. */
@@ -1595,8 +1638,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (c->normalize_embeddings) {
rc = 0;
for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(
- out_embeddings + (size_t)b * hidden, hidden) != 0) {
+ if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
rc = -1;
break;
}
@@ -1614,8 +1656,7 @@ typedef struct {
const embed_input_t *input;
} mlx_batch_item_t;
-static int mlx_batch_item_cmp(const void *a, const void *b)
-{
+static int mlx_batch_item_cmp(const void *a, const void *b) {
const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
int da = ia->input->n_tokens;
@@ -1623,27 +1664,33 @@ static int mlx_batch_item_cmp(const void *a, const void *b)
return (da > db) - (da < db);
}
-static int mlx_padding_exceeds_limit(size_t padded, size_t tokens)
-{
+static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) {
if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5)
return 1;
return padded * 5 > tokens * 11;
}
-static int mlx_batch_should_split(const embed_input_t *inputs, int batch,
- int *out_total_tokens, int *out_max_seq)
-{
+static int mlx_batch_should_split(const embed_input_t *inputs,
+ int batch,
+ int *out_total_tokens,
+ int *out_max_seq) {
int max_seq = 0;
int total_tokens = 0;
for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
- if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0;
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return 0;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
+ if (total_tokens > INT_MAX - inputs[b].n_tokens)
+ return 0;
total_tokens += inputs[b].n_tokens;
}
- if (out_total_tokens) *out_total_tokens = total_tokens;
- if (out_max_seq) *out_max_seq = max_seq;
- if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0;
+ if (out_total_tokens)
+ *out_total_tokens = total_tokens;
+ if (out_max_seq)
+ *out_max_seq = max_seq;
+ if (batch < 4 || max_seq < 64 || total_tokens <= 0)
+ return 0;
size_t padded;
if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
@@ -1653,9 +1700,9 @@ static int mlx_batch_should_split(const embed_input_t *inputs, int batch,
}
static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
- int group_count, int group_max,
- int next_tokens)
-{
+ int group_count,
+ int group_max,
+ int next_tokens) {
if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
return 1;
@@ -1674,9 +1721,10 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
return merged <= split_total;
}
-int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
- int batch, float *out_embeddings)
-{
+int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -1684,19 +1732,20 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
- mlx_batch_item_t *items = (mlx_batch_item_t *)malloc(
- (size_t)batch * sizeof(*items));
- embed_input_t *group_inputs = (embed_input_t *)malloc(
- (size_t)batch * sizeof(*group_inputs));
+ mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items));
+ embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items); free(group_inputs);
+ free(items);
+ free(group_inputs);
return -1;
}
float *group_out = (float *)malloc(group_bytes);
if (!items || !group_inputs || !group_out) {
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return -1;
}
for (int i = 0; i < batch; i++) {
@@ -1706,7 +1755,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
int rc = 0;
- for (int start = 0; start < batch && rc == 0; ) {
+ for (int start = 0; start < batch && rc == 0;) {
int group_max = 0;
int group_tokens = 0;
int end = start;
@@ -1717,12 +1766,9 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
int next_total = group_tokens + next_tokens;
if (end > start) {
size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max,
- &next_padded) != 0 ||
- mlx_padding_exceeds_limit(next_padded,
- (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start,
- group_max, next_tokens))
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
+ mlx_padding_exceeds_limit(next_padded, (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
break;
}
group_max = next_max;
@@ -1733,26 +1779,28 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count,
- group_out);
- if (rc != 0) break;
+ rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
+ if (rc != 0)
+ break;
for (int i = 0; i < group_count; i++) {
int dst = items[start + i].index;
- memcpy(out_embeddings + (size_t)dst * hidden,
- group_out + (size_t)i * hidden,
+ memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden,
(size_t)hidden * sizeof(float));
}
start = end;
}
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return rc;
}
-int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_embedding)
-{
- embed_input_t input = { token_ids, n_tokens };
+int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
return embed_mlx_encode_batch(ctx, &input, 1, out_embedding);
}
@@ -1762,8 +1810,7 @@ typedef struct {
const embed_context_input_t *input;
} mlx_context_batch_item_t;
-static int mlx_context_batch_item_cmp(const void *a, const void *b)
-{
+static int mlx_context_batch_item_cmp(const void *a, const void *b) {
const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a;
const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b;
int da = ia->input->input.n_tokens;
@@ -1771,33 +1818,33 @@ static int mlx_context_batch_item_cmp(const void *a, const void *b)
return (da > db) - (da < db);
}
-static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens)
-{
+static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4)
return 1;
return padded * 4 > tokens * 5;
}
static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
- int batch, int *out_total_spans)
-{
+ int batch,
+ int *out_total_spans) {
int max_seq = 0;
int total_tokens = 0;
int total_spans = 0;
for (int b = 0; b < batch; b++) {
int n_tokens = inputs[b].input.n_tokens;
- if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans ||
- inputs[b].n_spans <= 0)
+ if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0)
return 0;
- if (n_tokens > max_seq) max_seq = n_tokens;
- if (total_tokens > INT_MAX - n_tokens ||
- total_spans > INT_MAX - inputs[b].n_spans)
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
+ if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans)
return 0;
total_tokens += n_tokens;
total_spans += inputs[b].n_spans;
}
- if (out_total_spans) *out_total_spans = total_spans;
- if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0;
+ if (out_total_spans)
+ *out_total_spans = total_spans;
+ if (batch < 2 || max_seq < 64 || total_tokens <= 0)
+ return 0;
size_t padded;
if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
@@ -1806,13 +1853,11 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
}
static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch, float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
- ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers ||
- !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
+ ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
return -1;
const embed_config_t *c = &ctx->config;
@@ -1824,29 +1869,29 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans ||
- input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans)
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_spans > INT_MAX - input->n_spans)
return -1;
total_spans += input->n_spans;
- if (n_tokens > max_seq) max_seq = n_tokens;
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
for (int t = 0; t < n_tokens; t++) {
- if (input->input.ids[t] < 0 ||
- input->input.ids[t] >= c->vocab_size) {
+ if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) {
fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
input->input.ids[t], b, t);
return -1;
}
}
for (int s = 0; s < input->n_spans; s++) {
- if (input->spans[s].start < 0 ||
- input->spans[s].start > n_tokens ||
+ if (input->spans[s].start < 0 || input->spans[s].start > n_tokens ||
input->spans[s].n_tokens <= 0 ||
input->spans[s].n_tokens > n_tokens - input->spans[s].start)
return -1;
}
}
for (int b = 0; b < batch; b++)
- if (inputs[b].input.n_tokens != max_seq) has_padding = 1;
+ if (inputs[b].input.n_tokens != max_seq)
+ has_padding = 1;
size_t rows, ids_bytes, mask_bytes, index_bytes;
size_t span_float_bytes, out_values, out_bytes;
@@ -1858,10 +1903,10 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 ||
mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
return -1;
- if (rows > INT_MAX) return -1;
+ if (rows > INT_MAX)
+ return -1;
int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = has_padding
- ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
if (!padded_ids || (has_padding && !attn_mask_data)) {
free(padded_ids);
free(attn_mask_data);
@@ -1873,10 +1918,12 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
size_t row = (size_t)b * max_seq + t;
if (t < input->n_tokens) {
padded_ids[row] = input->ids[t];
- if (has_padding) attn_mask_data[row] = 0.0f;
+ if (has_padding)
+ attn_mask_data[row] = 0.0f;
} else {
padded_ids[row] = 0;
- if (has_padding) attn_mask_data[row] = -1.0e9f;
+ if (has_padding)
+ attn_mask_data[row] = -1.0e9f;
}
}
}
@@ -1908,44 +1955,49 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int ids_shape[] = {batch, max_seq};
int attn_mask_shape[] = {batch, 1, 1, max_seq};
ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- attn_mask = has_padding
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
+ attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
free(padded_ids);
free(attn_mask_data);
- if (!arr_ok(ids)) goto cleanup;
- if (has_padding && !arr_ok(attn_mask)) goto cleanup;
+ if (!arr_ok(ids))
+ goto cleanup;
+ if (has_padding && !arr_ok(attn_mask))
+ goto cleanup;
if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
mlx_array attn_mask_bf16 = mlx_array_new();
mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
mlx_array_free(attn_mask);
attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) goto cleanup;
+ if (!arr_ok(attn_mask))
+ goto cleanup;
}
x = mlx_array_new();
mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
mlx_array_free(ids);
ids = (mlx_array){0};
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
- 0, c->n_layers);
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers);
if (has_padding) {
mlx_array_free(attn_mask);
attn_mask = (mlx_array){0};
}
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
x_normed = mlx_array_new();
mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
mlx_array_free(x);
x = (mlx_array){0};
- if (!arr_ok(x_normed)) goto cleanup;
+ if (!arr_ok(x_normed))
+ goto cleanup;
if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
- if (!pooled) goto cleanup;
+ if (!pooled)
+ goto cleanup;
int pooled_count = 0;
for (int b = 0; b < batch; b++) {
@@ -1964,23 +2016,25 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
pooled[pooled_count] = mlx_array_new();
mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
mlx_array_free(slice);
- if (!arr_ok(pooled[pooled_count])) goto cleanup;
+ if (!arr_ok(pooled[pooled_count]))
+ goto cleanup;
pooled_count++;
}
}
pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
- if (!pooled_vec.ctx) goto cleanup;
+ if (!pooled_vec.ctx)
+ goto cleanup;
embeddings = mlx_array_new();
mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
- if (!arr_ok(embeddings)) goto cleanup;
+ if (!arr_ok(embeddings))
+ goto cleanup;
} else {
end_idx_data = (int *)malloc(index_bytes);
start_idx_data = (int *)malloc(index_bytes);
lengths_data = (float *)malloc(span_float_bytes);
start_mask_data = (float *)malloc(span_float_bytes);
- if (!end_idx_data || !start_idx_data || !lengths_data ||
- !start_mask_data)
+ if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data)
goto cleanup;
int span_count = 0;
@@ -2001,57 +2055,65 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int index_shape[] = {total_spans};
int scale_shape[] = {total_spans, 1};
end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32);
- start_idx = mlx_array_new_data(start_idx_data, index_shape, 1,
- MLX_INT32);
- lengths = mlx_array_new_data(lengths_data, scale_shape, 2,
- MLX_FLOAT32);
- start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2,
- MLX_FLOAT32);
- free(end_idx_data); end_idx_data = NULL;
- free(start_idx_data); start_idx_data = NULL;
- free(lengths_data); lengths_data = NULL;
- free(start_mask_data); start_mask_data = NULL;
- if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) ||
- !arr_ok(start_mask))
+ start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32);
+ lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32);
+ start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32);
+ free(end_idx_data);
+ end_idx_data = NULL;
+ free(start_idx_data);
+ start_idx_data = NULL;
+ free(lengths_data);
+ lengths_data = NULL;
+ free(start_mask_data);
+ start_mask_data = NULL;
+ if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask))
goto cleanup;
prefix = mlx_array_new();
mlx_cumsum(&prefix, x_normed, 1, false, true, S);
mlx_array_free(x_normed);
x_normed = (mlx_array){0};
- if (!arr_ok(prefix)) goto cleanup;
+ if (!arr_ok(prefix))
+ goto cleanup;
int flat_shape[] = {(int)rows, hidden};
flat_prefix = mlx_array_new();
mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S);
mlx_array_free(prefix);
prefix = (mlx_array){0};
- if (!arr_ok(flat_prefix)) goto cleanup;
+ if (!arr_ok(flat_prefix))
+ goto cleanup;
end_values = mlx_array_new();
mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S);
- if (!arr_ok(end_values)) goto cleanup;
+ if (!arr_ok(end_values))
+ goto cleanup;
start_values = mlx_array_new();
mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S);
- if (!arr_ok(start_values)) goto cleanup;
+ if (!arr_ok(start_values))
+ goto cleanup;
masked_start = mlx_array_new();
mlx_multiply(&masked_start, start_values, start_mask, S);
- if (!arr_ok(masked_start)) goto cleanup;
+ if (!arr_ok(masked_start))
+ goto cleanup;
span_sums = mlx_array_new();
mlx_subtract(&span_sums, end_values, masked_start, S);
- if (!arr_ok(span_sums)) goto cleanup;
+ if (!arr_ok(span_sums))
+ goto cleanup;
embeddings = mlx_array_new();
mlx_divide(&embeddings, span_sums, lengths, S);
- if (!arr_ok(embeddings)) goto cleanup;
+ if (!arr_ok(embeddings))
+ goto cleanup;
}
embeddings_f32 = mlx_array_new();
mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S);
- if (!arr_ok(embeddings_f32)) goto cleanup;
+ if (!arr_ok(embeddings_f32))
+ goto cleanup;
mlx_array_eval(embeddings_f32);
const float *data = mlx_array_data_float32(embeddings_f32);
@@ -2067,9 +2129,11 @@ cleanup:
free(end_idx_data);
mlx_array_free(embeddings_f32);
mlx_array_free(embeddings);
- if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec);
+ if (pooled_vec.ctx)
+ mlx_vector_array_free(pooled_vec);
if (pooled) {
- for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]);
+ for (int s = 0; s < total_spans; s++)
+ mlx_array_free(pooled[s]);
free(pooled);
}
mlx_array_free(span_sums);
@@ -2090,31 +2154,33 @@ cleanup:
}
int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings)
-{
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
int total_spans = 0;
if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
- return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch,
- out_embeddings);
+ return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
- mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc(
- (size_t)batch * sizeof(*items));
- embed_context_input_t *group_inputs = (embed_context_input_t *)malloc(
- (size_t)batch * sizeof(*group_inputs));
+ mlx_context_batch_item_t *items =
+ (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items));
+ embed_context_input_t *group_inputs =
+ (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items); free(group_inputs);
+ free(items);
+ free(group_inputs);
return -1;
}
float *group_out = (float *)malloc(group_bytes);
if (!items || !group_inputs || !group_out) {
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return -1;
}
@@ -2128,7 +2194,7 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp);
int rc = 0;
- for (int start = 0; start < batch && rc == 0; ) {
+ for (int start = 0; start < batch && rc == 0;) {
int group_max = 0;
int group_tokens = 0;
int end = start;
@@ -2139,12 +2205,9 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
int next_total = group_tokens + next_tokens;
if (end > start) {
size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max,
- &next_padded) != 0 ||
- mlx_context_padding_exceeds_limit(next_padded,
- (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start,
- group_max, next_tokens))
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
+ mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
break;
}
group_max = next_max;
@@ -2155,47 +2218,51 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs,
- group_count, group_out);
- if (rc != 0) break;
+ rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
+ if (rc != 0)
+ break;
int src_span = 0;
for (int i = 0; i < group_count; i++) {
int n_spans = group_inputs[i].n_spans;
int dst_span = items[start + i].span_offset;
memcpy(out_embeddings + (size_t)dst_span * hidden,
- group_out + (size_t)src_span * hidden,
- (size_t)n_spans * hidden * sizeof(float));
+ group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float));
src_span += n_spans;
}
start = end;
}
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return rc;
}
-int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, const embed_span_t *spans,
- int n_spans, float *out_embeddings)
-{
+int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
embed_context_input_t input = {
- .input = { token_ids, n_tokens },
+ .input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
}
-float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
-{
- if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
+ if (!ctx || !token_ids || n_tokens <= 0)
+ return NULL;
int hidden = ctx->config.hidden_size;
size_t out_bytes;
if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0)
return NULL;
float *out = (float *)malloc(out_bytes);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
free(out);
diff --git a/embed_mlx.h b/embed_mlx.h
index 5f6e2c9..87fbda6 100644
--- a/embed_mlx.h
+++ b/embed_mlx.h
@@ -38,7 +38,7 @@ typedef struct {
*/
EMBED_API embed_mlx_ctx_t *embed_mlx_load(const char *model_dir);
EMBED_API embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts);
+ const embed_mlx_options_t *opts);
EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx);
@@ -48,8 +48,8 @@ EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx);
* scoring, not one pooled document vector.
*/
EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir);
-EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
- const char *model_dir, const embed_mlx_options_t *opts);
+EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
+ const embed_mlx_options_t *opts);
EMBED_API void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx);
EMBED_API const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx);
@@ -62,8 +62,10 @@ EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx);
* before copying back to CPU memory.
*/
EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors);
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
/*
* Device-resident late-interaction token vectors.
@@ -72,24 +74,27 @@ EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
* outlive ctx. Copying is optional: MaxSim can consume these handles directly
* and copy back only final candidate scores.
*/
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
- embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
- int normalize);
+EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize);
EMBED_API void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs);
EMBED_API int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs);
EMBED_API int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs,
- float *out_vectors);
+EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors);
EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *const *items, int count);
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs,
- const int *token_indices, int count);
-EMBED_API int embed_mlx_late_maxsim_batch_device(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs, const int *doc_offsets,
- int docs_count, float *scores);
+ embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count);
+EMBED_API embed_mlx_late_vectors_t *
+embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count);
+EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *query,
+ const embed_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores);
/*
* Compute embedding for token_ids[0..n_tokens-1].
@@ -101,8 +106,10 @@ EMBED_API float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, in
* Compute one embedding into caller-provided out[hidden_size].
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_embedding);
+EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
/*
* Compute a true padded dense batch on MLX.
@@ -111,24 +118,30 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids,
* Pooling and attention masking follow the loaded model configuration.
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
- int batch, float *out_embeddings);
+EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/*
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, const embed_span_t *spans,
- int n_spans, float *out_embeddings);
+EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
/*
* Run a padded dense contextual document batch and pool every selected span.
* out_embeddings contains spans in document order.
*/
EMBED_API int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings);
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/* Get the config (so embed_cli.c can read hidden_size etc.) */
EMBED_API const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx);
diff --git a/embed_server.c b/embed_server.c
index 8c7743c..38da125 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -47,39 +47,39 @@
#include <cjson/cJSON.h>
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
+#define EMBED_HTTP_IO_BUF 8192
+#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
+#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define EMBED_HTTP_MAX_PATH 255u
+#define EMBED_HTTP_BACKLOG 128
+#define EMBED_HTTP_SETSIZE 10240
#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
+#define EMBED_HTTP_SWEEP_MS 1000
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
+#define EMBED_API_MAX_STANDARD_INPUTS 512
+#define EMBED_API_MAX_CONTEXT_DOCS 512
+#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
+#define EMBED_API_MAX_ITEM_TOKENS 32768
+#define EMBED_API_MAX_TOTAL_TOKENS 120000
+#define EMBED_LATE_QUERY_TOKENS 32
+#define EMBED_LATE_MASK_TOKEN_ID 151642
+#define EMBED_LATE_QUERY_PREFIX_ID 151669
+#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
+#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
/* Microseconds the worker waits for more requests before dispatching a batch.
* CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
* busy; MLX and CPU dispatch immediately, where waiting only adds latency.
* Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
+#define EMBED_SERVER_BATCH_WAIT_US 0
+#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
+#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
+#define EMBED_MLX_RESIDENT_MULTIPLIER 2
typedef struct {
char *ptr;
@@ -94,19 +94,22 @@ static void die_oom(void) {
static void *xmalloc(size_t n) {
void *p = malloc(n ? n : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
static void *xcalloc(size_t n, size_t sz) {
void *p = calloc(n ? n : 1, sz ? sz : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
static void *xrealloc(void *p, size_t n) {
p = realloc(p, n ? n : 1);
- if (!p) die_oom();
+ if (!p)
+ die_oom();
return p;
}
@@ -142,9 +145,7 @@ static uint64_t nstime(void) {
return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u;
}
-static double ns_to_ms(uint64_t ns) {
- return (double)ns / 1000000.0;
-}
+static double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; }
static void server_log(const char *fmt, ...) {
char ts[32];
@@ -161,12 +162,15 @@ static void server_log(const char *fmt, ...) {
}
static void sbuf_reserve(sbuf *b, size_t add) {
- if (add > SIZE_MAX - b->len - 1) die_oom();
+ if (add > SIZE_MAX - b->len - 1)
+ die_oom();
size_t need = b->len + add + 1;
- if (need <= b->cap) return;
+ if (need <= b->cap)
+ return;
size_t cap = b->cap ? b->cap * 2 : 256;
while (cap < need) {
- if (cap > SIZE_MAX / 2) die_oom();
+ if (cap > SIZE_MAX / 2)
+ die_oom();
cap *= 2;
}
b->ptr = xrealloc(b->ptr, cap);
@@ -185,7 +189,8 @@ static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); }
static void sbuf_clear(sbuf *b) {
b->len = 0;
- if (b->ptr) b->ptr[0] = '\0';
+ if (b->ptr)
+ b->ptr[0] = '\0';
}
static void sbuf_printf(sbuf *b, const char *fmt, ...) {
@@ -194,11 +199,13 @@ static void sbuf_printf(sbuf *b, const char *fmt, ...) {
va_copy(ap2, ap);
int n = vsnprintf(NULL, 0, fmt, ap);
va_end(ap);
- if (n < 0) die_oom();
+ if (n < 0)
+ die_oom();
sbuf_reserve(b, (size_t)n);
int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2);
va_end(ap2);
- if (n2 < 0 || n2 != n) die_oom();
+ if (n2 < 0 || n2 != n)
+ die_oom();
b->len += (size_t)n;
}
@@ -220,11 +227,7 @@ typedef enum {
MODEL_UNKNOWN
} model_slot;
-typedef enum {
- MODEL_KIND_STANDARD,
- MODEL_KIND_CONTEXTUAL,
- MODEL_KIND_LATE
-} model_kind;
+typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
* API: the canonical output is the tanh int8 quantization, so the default
@@ -251,28 +254,30 @@ typedef struct {
} model_info;
static const model_info k_models[] = {
- {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
- {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
- {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0,
- EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
{"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
- {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
};
static model_slot model_slot_for_id(const char *id) {
- if (!id) return MODEL_UNKNOWN;
+ if (!id)
+ return MODEL_UNKNOWN;
for (int i = 0; i < MODEL_COUNT; i++) {
- if (!strcmp(id, k_models[i].id)) return (model_slot)i;
+ if (!strcmp(id, k_models[i].id))
+ return (model_slot)i;
}
return MODEL_UNKNOWN;
}
@@ -398,7 +403,8 @@ static volatile sig_atomic_t g_signal_wfd = -1;
static void stop_signal_handler(int sig) {
(void)sig;
- if (g_stop_requested) _exit(130);
+ if (g_stop_requested)
+ _exit(130);
int saved = errno;
g_stop_requested = 1;
int fd = (int)g_signal_wfd;
@@ -411,7 +417,8 @@ static void stop_signal_handler(int sig) {
static int set_nonblock(int fd) {
int flags = fcntl(fd, F_GETFL, 0);
- if (flags < 0) return -1;
+ if (flags < 0)
+ return -1;
return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
@@ -422,7 +429,8 @@ static void http_request_free(http_request *r) {
}
static void client_free(client *c) {
- if (!c) return;
+ if (!c)
+ return;
http_request_free(&c->req);
sbuf_free(&c->in);
sbuf_free(&c->out);
@@ -438,34 +446,45 @@ static void client_incref(client *c) {
static void client_decref(client *c) {
int free_now = 0;
pthread_mutex_lock(&c->srv->mu);
- if (--c->refcount == 0) free_now = 1;
+ if (--c->refcount == 0)
+ free_now = 1;
pthread_mutex_unlock(&c->srv->mu);
- if (free_now) client_free(c);
+ if (free_now)
+ client_free(c);
}
static void client_decref_n(client *c, int n) {
int free_now = 0;
- if (n <= 0) return;
+ if (n <= 0)
+ return;
pthread_mutex_lock(&c->srv->mu);
c->refcount -= n;
- if (c->refcount <= 0) free_now = 1;
+ if (c->refcount <= 0)
+ free_now = 1;
pthread_mutex_unlock(&c->srv->mu);
- if (free_now) client_free(c);
+ if (free_now)
+ client_free(c);
}
static void client_unlink(client *c) {
http_server *s = c->srv;
- if (!c->linked) return;
- if (c->prev) c->prev->next = c->next;
- else s->clients = c->next;
- if (c->next) c->next->prev = c->prev;
+ if (!c->linked)
+ return;
+ if (c->prev)
+ c->prev->next = c->next;
+ else
+ s->clients = c->next;
+ if (c->next)
+ c->next->prev = c->prev;
c->prev = c->next = NULL;
c->linked = 0;
- if (s->n_clients > 0) s->n_clients--;
+ if (s->n_clients > 0)
+ s->n_clients--;
}
static int close_client_unlink(client *c) {
- if (!c || c->cancelled) return 0;
+ if (!c || c->cancelled)
+ return 0;
c->cancelled = 1;
if (c->fd >= 0) {
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
@@ -478,40 +497,52 @@ static int close_client_unlink(client *c) {
}
static void close_client(client *c) {
- if (!close_client_unlink(c)) return;
+ if (!close_client_unlink(c))
+ return;
client_decref(c);
}
-static void append_http_response_ex(client *c, int status, const char *ctype,
+static void append_http_response_ex(client *c,
+ int status,
+ const char *ctype,
const char *extra_headers,
- const char *body, size_t body_len) {
+ const char *body,
+ size_t body_len) {
const char *reason = "OK";
- if (status == 204) reason = "No Content";
- else if (status == 400) reason = "Bad Request";
- else if (status == 401) reason = "Unauthorized";
- else if (status == 404) reason = "Not Found";
- else if (status == 405) reason = "Method Not Allowed";
- else if (status == 422) reason = "Unprocessable Entity";
- else if (status == 503) reason = "Service Unavailable";
- else if (status >= 500) reason = "Internal Server Error";
+ if (status == 204)
+ reason = "No Content";
+ else if (status == 400)
+ reason = "Bad Request";
+ else if (status == 401)
+ reason = "Unauthorized";
+ else if (status == 404)
+ reason = "Not Found";
+ else if (status == 405)
+ reason = "Method Not Allowed";
+ else if (status == 422)
+ reason = "Unprocessable Entity";
+ else if (status == 503)
+ reason = "Service Unavailable";
+ else if (status >= 500)
+ reason = "Internal Server Error";
sbuf_printf(&c->out,
"HTTP/1.1 %d %s\r\n"
"Content-Length: %zu\r\n"
"Connection: %s\r\n",
- status, reason, body_len,
- c->close_after_write ? "close" : "keep-alive");
+ status, reason, body_len, c->close_after_write ? "close" : "keep-alive");
if (ctype)
sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype);
if (extra_headers)
sbuf_puts(&c->out, extra_headers);
sbuf_puts(&c->out, "\r\n");
- if (body_len) sbuf_append(&c->out, body, body_len);
+ if (body_len)
+ sbuf_append(&c->out, body, body_len);
}
-static void append_http_response(client *c, int status, const char *ctype,
- const char *body, size_t body_len) {
+static void
+append_http_response(client *c, int status, const char *ctype, const char *body, size_t body_len) {
append_http_response_ex(c, status, ctype, NULL, body, body_len);
}
@@ -534,13 +565,27 @@ static void append_json_string(sbuf *b, const char *s) {
sbuf_putc(b, '"');
for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) {
switch (*p) {
- case '"': sbuf_puts(b, "\\\""); break;
- case '\\': sbuf_puts(b, "\\\\"); break;
- case '\b': sbuf_puts(b, "\\b"); break;
- case '\f': sbuf_puts(b, "\\f"); break;
- case '\n': sbuf_puts(b, "\\n"); break;
- case '\r': sbuf_puts(b, "\\r"); break;
- case '\t': sbuf_puts(b, "\\t"); break;
+ case '"':
+ sbuf_puts(b, "\\\"");
+ break;
+ case '\\':
+ sbuf_puts(b, "\\\\");
+ break;
+ case '\b':
+ sbuf_puts(b, "\\b");
+ break;
+ case '\f':
+ sbuf_puts(b, "\\f");
+ break;
+ case '\n':
+ sbuf_puts(b, "\\n");
+ break;
+ case '\r':
+ sbuf_puts(b, "\\r");
+ break;
+ case '\t':
+ sbuf_puts(b, "\\t");
+ break;
default:
if (*p < 0x20)
sbuf_printf(b, "\\u%04x", (unsigned)*p);
@@ -566,8 +611,10 @@ static void job_set_error(job *j, int status, const char *message, const char *t
}
static bool auth_ok(http_server *s, const char *auth) {
- if (!s->api_key || !s->api_key[0]) return true;
- if (!auth) return false;
+ if (!s->api_key || !s->api_key[0])
+ return true;
+ if (!auth)
+ return false;
const char prefix[] = "Bearer ";
return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
!strcmp(auth + sizeof(prefix) - 1, s->api_key);
@@ -575,12 +622,12 @@ static bool auth_ok(http_server *s, const char *auth) {
static ssize_t find_header_end(const char *p, size_t n) {
for (size_t i = 3; i < n; i++) {
- if (p[i - 3] == '\r' && p[i - 2] == '\n' &&
- p[i - 1] == '\r' && p[i] == '\n')
+ if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n')
return (ssize_t)(i + 1);
}
for (size_t i = 1; i < n; i++) {
- if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1);
+ if (p[i - 1] == '\n' && p[i] == '\n')
+ return (ssize_t)(i + 1);
}
return -1;
}
@@ -590,18 +637,22 @@ static char *header_value_dup(const char *h, size_t n, const char *name) {
const char *p = h, *end = h + n;
while (p < end) {
const char *line = p;
- while (p < end && *p != '\n') p++;
+ while (p < end && *p != '\n')
+ p++;
size_t len = (size_t)(p - line);
- if (len && line[len - 1] == '\r') len--;
- if (len > name_len && !strncasecmp(line, name, name_len) &&
- line[name_len] == ':') {
+ if (len && line[len - 1] == '\r')
+ len--;
+ if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') {
const char *v = line + name_len + 1;
- while (v < line + len && isspace((unsigned char)*v)) v++;
+ while (v < line + len && isspace((unsigned char)*v))
+ v++;
const char *e = line + len;
- while (e > v && isspace((unsigned char)e[-1])) e--;
+ while (e > v && isspace((unsigned char)e[-1]))
+ e--;
return xstrndup(v, (size_t)(e - v));
}
- if (p < end) p++;
+ if (p < end)
+ p++;
}
return NULL;
}
@@ -610,13 +661,15 @@ static int header_has_token(const char *value, const char *token) {
size_t token_len = strlen(token);
const char *p = value;
while (p && *p) {
- while (*p == ',' || isspace((unsigned char)*p)) p++;
+ while (*p == ',' || isspace((unsigned char)*p))
+ p++;
const char *start = p;
- while (*p && *p != ',') p++;
+ while (*p && *p != ',')
+ p++;
const char *end = p;
- while (end > start && isspace((unsigned char)end[-1])) end--;
- if ((size_t)(end - start) == token_len &&
- !strncasecmp(start, token, token_len))
+ while (end > start && isspace((unsigned char)end[-1]))
+ end--;
+ if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len))
return 1;
}
return 0;
@@ -630,14 +683,14 @@ static int parse_request_headers(client *c) {
i++;
}
line[i] = '\0';
- int fields = sscanf(line, "%7s %255s %15s",
- c->req.method, c->req.path, c->req.version);
+ int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version);
if (fields < 2)
return -1;
if (fields < 3)
snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0");
char *q = strchr(c->req.path, '?');
- if (q) *q = '\0';
+ if (q)
+ *q = '\0';
c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0;
char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection");
@@ -669,15 +722,19 @@ static int complete_request(client *c) {
if (!c->header_done) {
ssize_t hend = find_header_end(c->in.ptr, c->in.len);
if (hend < 0) {
- if (c->in.len > EMBED_HTTP_MAX_HEADER) return -1;
+ if (c->in.len > EMBED_HTTP_MAX_HEADER)
+ return -1;
return 0;
}
c->header_done = 1;
c->header_len = (size_t)hend;
- if (parse_request_headers(c) != 0) return -1;
+ if (parse_request_headers(c) != 0)
+ return -1;
}
- if (c->content_length > EMBED_HTTP_MAX_BODY) return -1;
- if (c->in.len < c->header_len + c->content_length) return 0;
+ if (c->content_length > EMBED_HTTP_MAX_BODY)
+ return -1;
+ if (c->in.len < c->header_len + c->content_length)
+ return 0;
c->req.body_len = c->content_length;
c->req.body = xmalloc(c->content_length + 1);
memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length);
@@ -687,23 +744,25 @@ static int complete_request(client *c) {
static void send_bad_http_request(client *c) {
size_t len;
- char *body = json_error_body("bad HTTP request", "invalid_request_error",
- &len);
+ char *body = json_error_body("bad HTTP request", "invalid_request_error", &len);
c->close_after_write = 1;
append_http_response(c, 400, "application/json", body, len);
free(body);
aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
}
static void reset_client_for_next_request(client *c) {
size_t consumed = c->header_len + c->content_length;
- if (consumed > c->in.len) consumed = c->in.len;
+ if (consumed > c->in.len)
+ consumed = c->in.len;
size_t remain = c->in.len - consumed;
if (remain)
memmove(c->in.ptr, c->in.ptr + consumed, remain);
c->in.len = remain;
- if (c->in.ptr) c->in.ptr[remain] = '\0';
+ if (c->in.ptr)
+ c->in.ptr[remain] = '\0';
http_request_free(&c->req);
sbuf_clear(&c->out);
@@ -716,7 +775,8 @@ static void reset_client_for_next_request(client *c) {
static void arm_next_request(client *c) {
reset_client_for_next_request(c);
- if (c->cancelled || c->fd < 0) return;
+ if (c->cancelled || c->fd < 0)
+ return;
if (c->in.len) {
int done = complete_request(c);
@@ -735,7 +795,8 @@ static void arm_next_request(client *c) {
}
static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
client *c = clientData;
while (c->sent < c->out.len) {
ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent);
@@ -744,26 +805,33 @@ static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
c->last_active_ms = mstime();
continue;
}
- if (n < 0 && errno == EINTR) continue;
- if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return;
+ if (n < 0 && errno == EINTR)
+ continue;
+ if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
+ return;
close_client(c);
return;
}
aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE);
- if (c->close_after_write) close_client(c);
- else arm_next_request(c);
+ if (c->close_after_write)
+ close_client(c);
+ else
+ arm_next_request(c);
}
static int queue_write(client *c) {
- if (!c || c->cancelled || c->fd < 0) return -1;
+ if (!c || c->cancelled || c->fd < 0)
+ return -1;
return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c);
}
static void enqueue_done(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
- if (s->done_tail) s->done_tail->next = j;
- else s->done_head = j;
+ if (s->done_tail)
+ s->done_tail->next = j;
+ else
+ s->done_head = j;
s->done_tail = j;
pthread_mutex_unlock(&s->mu);
const unsigned char byte = 1;
@@ -774,16 +842,17 @@ static job *pop_job_locked(http_server *s) {
job *j = s->job_head;
if (j) {
s->job_head = j->next;
- if (!s->job_head) s->job_tail = NULL;
+ if (!s->job_head)
+ s->job_tail = NULL;
j->next = NULL;
}
return j;
}
-static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu,
- uint64_t deadline_ns) {
+static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) {
uint64_t now_ns = nstime();
- if (now_ns >= deadline_ns) return ETIMEDOUT;
+ if (now_ns >= deadline_ns)
+ return ETIMEDOUT;
uint64_t remaining_ns = deadline_ns - now_ns;
struct timespec abs;
@@ -803,7 +872,8 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu,
}
static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
- if (max_jobs <= 0) return 0;
+ if (max_jobs <= 0)
+ return 0;
pthread_mutex_lock(&s->mu);
while (!s->job_head && !s->stopping)
@@ -817,9 +887,8 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
int n = 1;
jobs[0] = first;
- int batchable = s->batch_size > 1 &&
- (!strcmp(first->path, "/v1/embeddings") ||
- !strcmp(first->path, "/v1/contextualizedembeddings"));
+ int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") ||
+ !strcmp(first->path, "/v1/contextualizedembeddings"));
if (!batchable) {
pthread_mutex_unlock(&s->mu);
return n;
@@ -828,12 +897,13 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
/* Exact item and token budgets are applied after worker-side tokenization;
* this queue stage only defines which arrival window may be grouped. */
uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
- uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns
- ? UINT64_MAX : first->created_ns + wait_ns;
+ uint64_t deadline_ns =
+ first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
while (n < max_jobs) {
while (!s->job_head && !s->stopping && wait_ns > 0) {
int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
- if (rc == ETIMEDOUT) break;
+ if (rc == ETIMEDOUT)
+ break;
}
if (!s->job_head)
break;
@@ -850,8 +920,10 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
static void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
- if (s->job_tail) s->job_tail->next = j;
- else s->job_head = j;
+ if (s->job_tail)
+ s->job_tail->next = j;
+ else
+ s->job_head = j;
s->job_tail = j;
pthread_cond_signal(&s->cv);
pthread_mutex_unlock(&s->mu);
@@ -862,7 +934,8 @@ static job *pop_done(http_server *s) {
job *j = s->done_head;
if (j) {
s->done_head = j->next;
- if (!s->done_head) s->done_tail = NULL;
+ if (!s->done_head)
+ s->done_tail = NULL;
j->next = NULL;
}
pthread_mutex_unlock(&s->mu);
@@ -870,7 +943,8 @@ static job *pop_done(http_server *s) {
}
static void job_free(job *j) {
- if (!j) return;
+ if (!j)
+ return;
free(j->body);
free(j->auth);
free(j->extra_headers);
@@ -879,18 +953,19 @@ static void job_free(job *j) {
}
static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
http_server *s = clientData;
unsigned char tmp[128];
- while (read(fd, tmp, sizeof(tmp)) > 0) {}
+ while (read(fd, tmp, sizeof(tmp)) > 0) {
+ }
job *j;
while ((j = pop_done(s)) != NULL) {
client *c = j->c;
int refs_to_drop = 1; /* job reference */
if (!c->cancelled) {
- append_http_response_ex(c, j->status, j->content_type,
- j->extra_headers,
+ append_http_response_ex(c, j->status, j->content_type, j->extra_headers,
j->response ? j->response : "",
j->response ? j->response_len : 0);
if (queue_write(c) != AE_OK)
@@ -903,8 +978,7 @@ static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask)
static bool route_is_inference(const char *method, const char *path) {
return !strcmp(method, "POST") &&
- (!strcmp(path, "/v1/embeddings") ||
- !strcmp(path, "/v1/contextualizedembeddings") ||
+ (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") ||
!strcmp(path, "/v1/rerank"));
}
@@ -913,7 +987,8 @@ static void dispatch_request(client *c) {
if (!strcmp(c->req.method, "OPTIONS")) {
append_http_response(c, 204, NULL, "", 0);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
@@ -922,17 +997,19 @@ static void dispatch_request(client *c) {
char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
append_http_response(c, 404, "application/json", body, len);
free(body);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
if (!auth_ok(c->srv, c->req.auth)) {
size_t len;
- char *body = json_error_body("missing or invalid bearer token",
- "authentication_error", &len);
+ char *body =
+ json_error_body("missing or invalid bearer token", "authentication_error", &len);
append_http_response(c, 401, "application/json", body, len);
free(body);
- if (queue_write(c) != AE_OK) close_client(c);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
return;
}
@@ -950,7 +1027,8 @@ static void dispatch_request(client *c) {
}
static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
client *c = clientData;
char tmp[EMBED_HTTP_IO_BUF];
for (;;) {
@@ -973,23 +1051,28 @@ static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
close_client(c);
return;
}
- if (errno == EINTR) continue;
- if (errno == EAGAIN || errno == EWOULDBLOCK) return;
+ if (errno == EINTR)
+ continue;
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
close_client(c);
return;
}
}
static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop; (void)mask;
+ (void)loop;
+ (void)mask;
http_server *s = clientData;
char cip[64];
int cport = 0;
for (;;) {
int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
if (cfd == ANET_ERR) {
- if (errno == EAGAIN || errno == EWOULDBLOCK) return;
- if (errno == EINTR) continue;
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
+ if (errno == EINTR)
+ continue;
return;
}
set_nonblock(cfd);
@@ -1000,7 +1083,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
c->last_active_ms = mstime();
c->linked = 1;
c->next = s->clients;
- if (s->clients) s->clients->prev = c;
+ if (s->clients)
+ s->clients->prev = c;
s->clients = c;
s->n_clients++;
if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
@@ -1009,7 +1093,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
}
static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
- (void)loop; (void)id;
+ (void)loop;
+ (void)id;
http_server *s = clientData;
uint64_t now = mstime();
client *c = s->clients;
@@ -1027,15 +1112,16 @@ static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
}
static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)fd; (void)mask;
+ (void)fd;
+ (void)mask;
http_server *s = clientData;
unsigned char tmp[128];
- while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {}
+ while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
+ }
aeStop(loop);
}
-static bool ve_add(cJSON *detail, const char *loc_json,
- const char *msg, const char *type) {
+static bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) {
cJSON *obj = cJSON_CreateObject();
cJSON *loc = cJSON_Parse(loc_json);
if (!obj || !loc) {
@@ -1067,21 +1153,19 @@ static void job_set_422(job *j, cJSON *detail) {
}
static void job_set_timing_header(job *j) {
- if (!j || j->extra_headers || !j->created_ns) return;
+ if (!j || j->extra_headers || !j->created_ns)
+ return;
uint64_t done_ns = nstime();
- uint64_t queue_ns = j->started_ns > j->created_ns
- ? j->started_ns - j->created_ns : 0;
- uint64_t worker_ns = j->started_ns && done_ns > j->started_ns
- ? done_ns - j->started_ns : 0;
+ uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0;
+ uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0;
char buf[384];
int n = snprintf(buf, sizeof(buf),
- "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
- "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
- "worker;dur=%.3f\r\n",
- ns_to_ms(queue_ns), ns_to_ms(j->parse_ns),
- ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns),
- ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
+ "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
+ "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
+ "worker;dur=%.3f\r\n",
+ ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns),
+ ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
if (n > 0 && (size_t)n < sizeof(buf))
j->extra_headers = xstrdup(buf);
}
@@ -1098,7 +1182,8 @@ static cJSON *parse_json_body(job *j, cJSON *detail) {
return NULL;
}
const char *body_end = j->body + j->body_len;
- while (end && end < body_end && isspace((unsigned char)*end)) end++;
+ while (end && end < body_end && isspace((unsigned char)*end))
+ end++;
if (!end || end != body_end) {
ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
cJSON_Delete(root);
@@ -1113,30 +1198,30 @@ static cJSON *parse_json_body(job *j, cJSON *detail) {
}
static bool cjson_is_integer(cJSON *item) {
- if (!cJSON_IsNumber(item)) return false;
+ if (!cJSON_IsNumber(item))
+ return false;
double d = item->valuedouble;
int i = item->valueint;
return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i;
}
-static const char *encoding_from_root(cJSON *root, cJSON *detail,
- embedding_api_t api) {
+static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
/* Default and accepted encodings follow the model's API family: Perplexity
* defaults to base64_int8 and accepts {base64_int8, base64_binary, float};
* OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */
const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8";
cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
- if (!encoding) return dflt;
+ if (!encoding)
+ return dflt;
if (!cJSON_IsString(encoding) || !encoding->valuestring) {
- ve_add(detail, "[\"body\",\"encoding_format\"]",
- "encoding_format must be a string", "type_error.string");
+ ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string",
+ "type_error.string");
return dflt;
}
const char *v = encoding->valuestring;
- int ok = api == EMBED_API_OPENAI
- ? (!strcmp(v, "float") || !strcmp(v, "base64"))
- : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
- !strcmp(v, "float"));
+ int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64"))
+ : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
+ !strcmp(v, "float"));
if (!ok) {
ve_add(detail, "[\"body\",\"encoding_format\"]",
api == EMBED_API_OPENAI
@@ -1149,8 +1234,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail,
return v;
}
-static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim,
- int max_dim, const char *encoding) {
+static int
+dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
int dims = max_dim;
cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
if (dimensions) {
@@ -1161,31 +1246,30 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim,
dims = dimensions->valueint;
if (dims < min_dim || dims > max_dim) {
char msg[96];
- snprintf(msg, sizeof(msg),
- "dimensions must be between %d and %d",
- min_dim, max_dim);
+ snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim,
+ max_dim);
ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
}
}
}
if (!strcmp(encoding, "base64_binary") && dims % 8 != 0)
ve_add(detail, "[\"body\",\"dimensions\"]",
- "dimensions must be divisible by 8 for base64_binary",
- "value_error.divisible");
+ "dimensions must be divisible by 8 for base64_binary", "value_error.divisible");
return dims;
}
static char *base64_encode(const unsigned char *src, size_t n) {
- static const char tbl[] =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
size_t out_len = ((n + 2) / 3) * 4;
char *out = xmalloc(out_len + 1);
size_t o = 0;
for (size_t i = 0; i < n; i += 3) {
unsigned v = (unsigned)src[i] << 16;
int remain = (int)(n - i);
- if (remain > 1) v |= (unsigned)src[i + 1] << 8;
- if (remain > 2) v |= (unsigned)src[i + 2];
+ if (remain > 1)
+ v |= (unsigned)src[i + 1] << 8;
+ if (remain > 2)
+ v |= (unsigned)src[i + 2];
out[o++] = tbl[(v >> 18) & 63];
out[o++] = tbl[(v >> 12) & 63];
out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '=';
@@ -1197,8 +1281,10 @@ static char *base64_encode(const unsigned char *src, size_t n) {
static signed char quantize_int8_tanh(float x) {
int v = (int)lrintf(tanhf(x) * 127.0f);
- if (v > 127) v = 127;
- if (v < -128) v = -128;
+ if (v > 127)
+ v = 127;
+ if (v < -128)
+ v = -128;
return (signed char)v;
}
@@ -1295,54 +1381,62 @@ typedef struct {
} rerank_request;
static void free_token_bufs(token_buf *t, int n) {
- if (!t) return;
- for (int i = 0; i < n; i++) free(t[i].ids);
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
}
static void embedding_request_free(embedding_request *r) {
- if (!r) return;
+ if (!r)
+ return;
free_token_bufs(r->tokens, r->n_inputs);
free(r->tokens);
free(r->inputs);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
static void contextual_request_free(contextual_request *r) {
- if (!r) return;
+ if (!r)
+ return;
for (int i = 0; i < r->n_docs; i++) {
free(r->docs[i].ids);
free(r->docs[i].spans);
}
free(r->docs);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
static void late_tokens_free(late_tokens *t) {
- if (!t) return;
+ if (!t)
+ return;
free(t->ids);
free(t->keep);
memset(t, 0, sizeof(*t));
}
static void rerank_request_free(rerank_request *r) {
- if (!r) return;
+ if (!r)
+ return;
late_tokens_free(&r->query);
for (int i = 0; i < r->n_documents; i++)
late_tokens_free(&r->documents[i]);
free(r->documents);
- if (r->root) cJSON_Delete(r->root);
+ if (r->root)
+ cJSON_Delete(r->root);
memset(r, 0, sizeof(*r));
}
-static int tokenize_one(loaded_model *m, job *j, const char *text,
- token_buf *out) {
+static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
memset(out, 0, sizeof(*out));
uint64_t t0 = nstime();
- out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws,
- text, &out->n_tokens);
- if (j) j->tokenize_ns += nstime() - t0;
+ out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
if (!out->ids || out->n_tokens <= 0) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -1354,8 +1448,7 @@ static int tokenize_one(loaded_model *m, job *j, const char *text,
memset(out, 0, sizeof(*out));
return -1;
}
- int *ids = realloc(
- out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
if (!ids) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -1369,15 +1462,17 @@ static int tokenize_one(loaded_model *m, job *j, const char *text,
static int late_id_is_skipped(const loaded_model *m, int id) {
for (int i = 0; i < m->n_late_skip_ids; i++) {
- if (m->late_skip_ids[i] == id) return 1;
+ if (m->late_skip_ids[i] == id)
+ return 1;
}
return 0;
}
-static int tokenize_late_text(loaded_model *m, job *j, const char *text,
- int is_query, late_tokens *out) {
+static int
+tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
token_buf raw = {0};
- if (tokenize_one(m, j, text, &raw) != 0) return -1;
+ if (tokenize_one(m, j, text, &raw) != 0)
+ return -1;
int raw_tokens = raw.n_tokens;
int target = raw_tokens + 1;
@@ -1386,11 +1481,9 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text,
out->ids = xmalloc((size_t)target * sizeof(*out->ids));
out->ids[0] = raw.ids[0];
- out->ids[1] = is_query ? m->late_query_prefix_id
- : m->late_document_prefix_id;
+ out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
if (raw_tokens > 1) {
- memcpy(out->ids + 2, raw.ids + 1,
- (size_t)(raw_tokens - 1) * sizeof(*out->ids));
+ memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
}
out->n_tokens = raw_tokens + 1;
if (is_query) {
@@ -1408,8 +1501,7 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text,
return out->n_keep > 0 ? 0 : -1;
}
-static int model_embed_batch(loaded_model *m, const embed_input_t *inputs,
- int batch, float *out) {
+static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
#ifdef USE_MLX
if (m->mlx_ctx)
return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
@@ -1423,7 +1515,8 @@ static int model_embed_batch(loaded_model *m, const embed_input_t *inputs,
static int model_embed_spans_batch(loaded_model *m,
const embed_context_input_t *inputs,
- int batch, float *out) {
+ int batch,
+ float *out) {
#ifdef USE_MLX
if (m->mlx_ctx)
return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
@@ -1432,8 +1525,7 @@ static int model_embed_spans_batch(loaded_model *m,
if (m->cuda_ctx)
return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
#endif
- return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch,
- out);
+ return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
}
static int model_uses_dense_batches(const loaded_model *m) {
@@ -1447,10 +1539,10 @@ static int model_uses_dense_batches(const loaded_model *m) {
/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
* pads each dense row to the longest input in the chunk. */
-static int inference_batch_accepts_input(const loaded_model *m, int batch,
- int packed_tokens, int next_tokens,
- int max_batch_tokens) {
- if (batch == 0) return 1;
+static int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
if (model_uses_dense_batches(m))
return next_tokens <= max_batch_tokens / (batch + 1);
return next_tokens <= max_batch_tokens - packed_tokens;
@@ -1466,17 +1558,14 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) {
* "float" renders a JSON array: the true float32 vector for OpenAI/DashScope
* (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every
* other encoding renders a base64 string via encode_embedding(). */
-static void append_embedding_value(sbuf *b, int index, const float *emb,
- int dims, const char *encoding,
- embedding_api_t api) {
+static void append_embedding_value(
+ sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) {
if (!strcmp(encoding, "float")) {
- sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[",
- index);
+ sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index);
for (int i = 0; i < dims; i++) {
- if (i) sbuf_putc(b, ',');
- float v = api == EMBED_API_OPENAI
- ? emb[i]
- : (float)quantize_int8_tanh(emb[i]) / 128.0f;
+ if (i)
+ sbuf_putc(b, ',');
+ float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f;
sbuf_printf(b, "%.9g", (double)v);
}
sbuf_puts(b, "]}");
@@ -1500,12 +1589,10 @@ static int render_embedding_response(embedding_request *r, const float *embs) {
const float *render_embs = embs;
float *truncated = NULL;
if (r->model->renormalize_truncated && r->dims < full_dim) {
- truncated = xmalloc(
- (size_t)r->n_inputs * r->dims * sizeof(*truncated));
+ truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated));
for (int i = 0; i < r->n_inputs; i++) {
float *dst = truncated + (size_t)i * r->dims;
- memcpy(dst, embs + (size_t)i * full_dim,
- (size_t)r->dims * sizeof(*dst));
+ memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst));
if (embed_l2_normalize(dst, r->dims) != 0) {
free(truncated);
return -1;
@@ -1519,25 +1606,23 @@ static int render_embedding_response(embedding_request *r, const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
for (int i = 0; i < r->n_inputs; i++) {
- if (i) sbuf_putc(&b, ',');
- append_embedding_value(&b, i, render_embs + (size_t)i * render_stride,
- r->dims, r->encoding, api);
+ if (i)
+ sbuf_putc(&b, ',');
+ append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding,
+ api);
}
sbuf_printf(&b,
- "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
+ "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
set_response_from_buf(r->j, &b);
free(truncated);
return 0;
}
-static int embedding_request_compatible(const embedding_request *a,
- const embedding_request *b) {
- return a->ready && b->ready &&
- a->model == b->model &&
- a->dims == b->dims &&
+static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
!strcmp(a->encoding, b->encoding);
}
@@ -1549,21 +1634,27 @@ typedef struct {
static int embedding_batch_item_cmp(const void *a, const void *b) {
const embedding_batch_item *ia = a;
const embedding_batch_item *ib = b;
- if (ia->input.n_tokens < ib->input.n_tokens) return -1;
- if (ia->input.n_tokens > ib->input.n_tokens) return 1;
+ if (ia->input.n_tokens < ib->input.n_tokens)
+ return -1;
+ if (ia->input.n_tokens > ib->input.n_tokens)
+ return 1;
return ia->output_index - ib->output_index;
}
static int embedding_request_fits_group(const embedding_request *r,
- int group_inputs, int group_tokens,
- int max_batch, int max_batch_tokens) {
- if (group_inputs == 0) return 1;
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_inputs == 0)
+ return 1;
return r->n_inputs <= max_batch - group_inputs &&
r->total_tokens <= max_batch_tokens - group_tokens;
}
static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
- if (n_reqs <= 0) return;
+ if (n_reqs <= 0)
+ return;
loaded_model *m = reqs[0]->model;
int dim = m->info->dim;
int total_inputs = 0;
@@ -1583,9 +1674,9 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
}
qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp);
- int max_batch = reqs[0]->j->srv->batch_size > 0
- ? reqs[0]->j->srv->batch_size : total_inputs;
- if (max_batch > total_inputs) max_batch = total_inputs;
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs;
+ if (max_batch > total_inputs)
+ max_batch = total_inputs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
@@ -1597,8 +1688,7 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
int tokens = 0;
while (start + cur < total_inputs && cur < max_batch) {
int n_tokens = items[start + cur].input.n_tokens;
- if (!inference_batch_accepts_input(m, cur, tokens, n_tokens,
- max_batch_tokens))
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens))
break;
inputs[cur] = items[start + cur].input;
tokens += n_tokens;
@@ -1612,8 +1702,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
break;
}
for (int i = 0; i < cur; i++)
- memcpy(embs + (size_t)items[start + i].output_index * dim,
- batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float));
+ memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim,
+ (size_t)dim * sizeof(float));
start += cur;
}
@@ -1624,10 +1714,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
} else {
uint64_t t0 = nstime();
- if (render_embedding_response(
- reqs[i], embs + (size_t)pos * dim) != 0)
- job_set_error(reqs[i]->j, 500, "embedding normalization failed",
- "server_error");
+ if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
+ job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
reqs[i]->j->encode_ns += nstime() - t0;
}
pos += reqs[i]->n_inputs;
@@ -1639,41 +1727,40 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
free(items);
}
-static void render_contextual_response(contextual_request *r,
- const float *embs) {
+static void render_contextual_response(contextual_request *r, const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
int span_offset = 0;
for (int di = 0; di < r->n_docs; di++) {
- if (di) sbuf_putc(&b, ',');
+ if (di)
+ sbuf_putc(&b, ',');
sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di);
for (int ci = 0; ci < r->docs[di].n_spans; ci++) {
- if (ci) sbuf_putc(&b, ',');
+ if (ci)
+ sbuf_putc(&b, ',');
const float *emb = embs + (size_t)span_offset * r->model->info->dim;
- append_embedding_value(&b, ci, emb, r->dims, r->encoding,
- r->model->info->api);
+ append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api);
span_offset++;
}
sbuf_puts(&b, "]}");
}
- sbuf_printf(&b,
- "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
+ sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
+ r->model->info->id, r->total_tokens, r->total_tokens);
set_response_from_buf(r->j, &b);
}
-static int contextual_request_compatible(const contextual_request *a,
- const contextual_request *b) {
- return a->ready && b->ready &&
- a->model == b->model &&
- a->dims == b->dims &&
+static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
!strcmp(a->encoding, b->encoding);
}
static int contextual_request_fits_group(const contextual_request *r,
- int group_docs, int group_tokens,
- int max_batch, int max_batch_tokens) {
- if (group_docs == 0) return 1;
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_docs == 0)
+ return 1;
return r->n_docs <= max_batch - group_docs &&
r->total_tokens <= max_batch_tokens - group_tokens;
}
@@ -1687,14 +1774,16 @@ typedef struct {
static int contextual_batch_item_cmp(const void *a, const void *b) {
const contextual_batch_item *ia = a;
const contextual_batch_item *ib = b;
- if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1;
- if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1;
+ if (ia->input.input.n_tokens < ib->input.input.n_tokens)
+ return -1;
+ if (ia->input.input.n_tokens > ib->input.input.n_tokens)
+ return 1;
return ia->order - ib->order;
}
-static void execute_contextual_request_list(contextual_request **reqs,
- int n_reqs) {
- if (n_reqs <= 0) return;
+static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
loaded_model *m = reqs[0]->model;
int dim = m->info->dim;
int total_docs = 0;
@@ -1703,8 +1792,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
if (total_docs > INT_MAX - reqs[i]->n_docs ||
total_chunks > INT_MAX - reqs[i]->total_chunks) {
for (int k = 0; k < n_reqs; k++)
- job_set_error(reqs[k]->j, 500, "contextual batch is too large",
- "server_error");
+ job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error");
return;
}
total_docs += reqs[i]->n_docs;
@@ -1712,13 +1800,11 @@ static void execute_contextual_request_list(contextual_request **reqs,
}
if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
for (int i = 0; i < n_reqs; i++)
- job_set_error(reqs[i]->j, 500, "contextual batch is too large",
- "server_error");
+ job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error");
return;
}
- contextual_batch_item *items =
- xmalloc((size_t)total_docs * sizeof(*items));
+ contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items));
float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
int pos = 0;
int span_offset = 0;
@@ -1737,12 +1823,11 @@ static void execute_contextual_request_list(contextual_request **reqs,
}
qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
- int max_batch = reqs[0]->j->srv->batch_size > 0
- ? reqs[0]->j->srv->batch_size : total_docs;
- if (max_batch > total_docs) max_batch = total_docs;
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs;
+ if (max_batch > total_docs)
+ max_batch = total_docs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_context_input_t *inputs =
- xmalloc((size_t)max_batch * sizeof(*inputs));
+ embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc(sizeof(*batch_embs));
size_t batch_emb_cap = 0;
@@ -1754,8 +1839,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
int chunks = 0;
while (start + cur < total_docs && cur < max_batch) {
const embed_context_input_t *input = &items[start + cur].input;
- if (!inference_batch_accepts_input(m, cur, tokens,
- input->input.n_tokens,
+ if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
max_batch_tokens))
break;
inputs[cur] = *input;
@@ -1779,8 +1863,7 @@ static void execute_contextual_request_list(contextual_request **reqs,
for (int i = 0; i < cur; i++) {
int n_spans = inputs[i].n_spans;
memcpy(embs + (size_t)items[start + i].output_span_index * dim,
- batch_embs + (size_t)batch_span * dim,
- (size_t)n_spans * dim * sizeof(*embs));
+ batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs));
batch_span += n_spans;
}
start += cur;
@@ -1790,12 +1873,10 @@ static void execute_contextual_request_list(contextual_request **reqs,
for (int i = 0; i < n_reqs; i++) {
reqs[i]->j->infer_ns += infer_ns;
if (failed) {
- job_set_error(reqs[i]->j, 500, "contextual embedding failed",
- "server_error");
+ job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error");
} else {
uint64_t t0 = nstime();
- render_contextual_response(reqs[i],
- embs + (size_t)span_offset * dim);
+ render_contextual_response(reqs[i], embs + (size_t)span_offset * dim);
reqs[i]->j->encode_ns += nstime() - t0;
}
span_offset += reqs[i]->total_chunks;
@@ -1807,9 +1888,13 @@ static void execute_contextual_request_list(contextual_request **reqs,
free(items);
}
-static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
- loaded_model **out_model, int *out_dims,
- const char **out_encoding, http_server *s) {
+static int validate_common(cJSON *root,
+ cJSON *detail,
+ model_kind expected_kind,
+ loaded_model **out_model,
+ int *out_dims,
+ const char **out_encoding,
+ http_server *s) {
cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
const char *model_id = NULL;
if (!model_item) {
@@ -1832,8 +1917,7 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
}
}
model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN;
- embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api
- : EMBED_API_PERPLEXITY;
+ embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY;
const char *enc = encoding_from_root(root, detail, api);
int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128;
int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560;
@@ -1841,11 +1925,12 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
*out_encoding = enc;
*out_dims = dims;
return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN &&
- !s->models[model_slot_for_id(model_id)].info ? 503 : 0;
+ !s->models[model_slot_for_id(model_id)].info
+ ? 503
+ : 0;
}
-static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
- embedding_request *out) {
+static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -1854,8 +1939,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD,
- &m, &dims, &encoding, s);
+ int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
@@ -1866,8 +1950,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (cJSON_IsString(input)) {
if (!input->valuestring || input->valuestring[0] == '\0')
- ve_add(detail, "[\"body\",\"input\"]", "input must not be empty",
- "value_error.empty");
+ ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty");
n_inputs = 1;
} else if (cJSON_IsArray(input)) {
n_inputs = cJSON_GetArraySize(input);
@@ -1890,13 +1973,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
i++;
}
} else {
- ve_add(detail, "[\"body\",\"input\"]",
- "input must be a string or an array of strings", "type_error");
+ ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings",
+ "type_error");
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -1916,8 +1998,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
out->inputs[0].n_tokens = out->tokens[0].n_tokens;
out->total_tokens = out->tokens[0].n_tokens;
if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]",
- "input exceeds 32768 token limit",
+ ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
"value_error.context_length");
}
} else {
@@ -1926,15 +2007,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, loc, "input item must be a string",
- "type_error.string");
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
idx++;
continue;
}
- if (tokenize_one(m, j, item->valuestring,
- &out->tokens[idx]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_one(m, j, item->valuestring, &out->tokens[idx]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
out->inputs[idx].ids = out->tokens[idx].ids;
out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
@@ -1962,8 +2040,8 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
cJSON_Delete(detail);
}
-static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
- contextual_request *out) {
+static void
+prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -1972,8 +2050,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
- int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL,
- &m, &dims, &encoding, s);
+ int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
out->model = m;
out->dims = dims;
out->encoding = encoding;
@@ -1984,8 +2061,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
if (!input) {
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (!cJSON_IsArray(input)) {
- ve_add(detail, "[\"body\",\"input\"]",
- "input must be an array of document chunk arrays", "type_error.array");
+ ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays",
+ "type_error.array");
} else {
n_docs = cJSON_GetArraySize(input);
if (n_docs < 1)
@@ -2000,8 +2077,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
if (!cJSON_IsArray(doc_arr)) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
- ve_add(detail, loc, "document must be an array of strings",
- "type_error.array");
+ ve_add(detail, loc, "document must be an array of strings", "type_error.array");
di++;
continue;
}
@@ -2036,8 +2112,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -2051,8 +2126,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(doc_arr, input) {
contextual_doc *doc = &out->docs[di];
int n_chunks = cJSON_GetArraySize(doc_arr);
- token_buf *chunk_tokens =
- xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
+ token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
doc->n_spans = n_chunks;
int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
@@ -2063,10 +2137,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(chunk, doc_arr) {
char loc[80];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
- if (tokenize_one(m, j, chunk->valuestring,
- &chunk_tokens[ci]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
doc_valid = 0;
} else {
doc_tokens += chunk_tokens[ci].n_tokens;
@@ -2103,8 +2175,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
di++;
}
if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) {
- ve_add(detail, "[\"body\",\"input\"]",
- "request exceeds 120000 token limit",
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
"value_error.context_length");
} else {
out->total_tokens = (int)request_tokens;
@@ -2122,30 +2193,28 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
cJSON_Delete(detail);
}
-static int validate_rerank_model(cJSON *root, cJSON *detail, http_server *s,
- loaded_model **out_model) {
+static int
+validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) {
cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model");
if (!item) {
ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
return 0;
}
if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, "[\"body\",\"model\"]", "model must be a string",
- "type_error.string");
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
return 0;
}
model_slot slot = model_slot_for_id(item->valuestring);
if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) {
- ve_add(detail, "[\"body\",\"model\"]",
- "value is not a valid enum member for this endpoint", "enum");
+ ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
+ "enum");
return 0;
}
*out_model = s->models[slot].info ? &s->models[slot] : NULL;
return *out_model ? 0 : 503;
}
-static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
- rerank_request *out) {
+static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
out->root = root;
@@ -2157,28 +2226,24 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
if (!query) {
ve_add(detail, "[\"body\",\"query\"]", "field required", "missing");
} else if (!cJSON_IsString(query)) {
- ve_add(detail, "[\"body\",\"query\"]", "query must be a string",
- "type_error.string");
+ ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string");
} else if (!query->valuestring || !query->valuestring[0]) {
- ve_add(detail, "[\"body\",\"query\"]", "query must not be empty",
- "value_error.empty");
+ ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty");
}
int n_documents = 0;
if (!documents) {
ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing");
} else if (!cJSON_IsArray(documents)) {
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must be an array of strings", "type_error.array");
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings",
+ "type_error.array");
} else {
n_documents = cJSON_GetArraySize(documents);
if (n_documents < 1)
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must contain at least 1 item",
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
"value_error.list.min_items");
else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS)
- ve_add(detail, "[\"body\",\"documents\"]",
- "documents must contain at most 1000 items",
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
"value_error.list.max_items");
cJSON *document;
int i = 0;
@@ -2186,51 +2251,43 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
char loc[72];
snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++);
if (!cJSON_IsString(document))
- ve_add(detail, loc, "document must be a string",
- "type_error.string");
+ ve_add(detail, loc, "document must be a string", "type_error.string");
else if (!document->valuestring || !document->valuestring[0])
- ve_add(detail, loc, "document must not be empty",
- "value_error.empty");
+ ve_add(detail, loc, "document must not be empty", "value_error.empty");
}
}
cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n");
cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k");
if (top_n && top_k) {
- ve_add(detail, "[\"body\",\"top_n\"]",
- "top_n and top_k are aliases; provide only one",
+ ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one",
"value_error.conflict");
}
cJSON *top = top_n ? top_n : top_k;
out->top_n = n_documents;
if (top) {
if (!cjson_is_integer(top)) {
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]"
- : "[\"body\",\"top_k\"]",
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
"top_n must be an integer", "type_error.integer");
} else {
out->top_n = top->valueint;
if (out->top_n < 1 || out->top_n > n_documents)
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]"
- : "[\"body\",\"top_k\"]",
- "top_n must be between 1 and the number of documents",
- "value_error.range");
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be between 1 and the number of documents", "value_error.range");
}
}
- cJSON *return_documents =
- cJSON_GetObjectItemCaseSensitive(root, "return_documents");
+ cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents");
if (return_documents) {
if (!cJSON_IsBool(return_documents))
- ve_add(detail, "[\"body\",\"return_documents\"]",
- "return_documents must be a boolean", "type_error.bool");
+ ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean",
+ "type_error.bool");
else
out->return_documents = cJSON_IsTrue(return_documents);
}
if (cJSON_GetArraySize(detail) == 0 && unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
cJSON_Delete(detail);
return;
}
@@ -2238,15 +2295,13 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
if (cJSON_GetArraySize(detail) == 0 && out->model) {
out->n_documents = n_documents;
out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
- if (tokenize_late_text(out->model, j, query->valuestring, 1,
- &out->query) != 0) {
+ if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) {
ve_add(detail, "[\"body\",\"query\"]", "tokenization failed",
"value_error.tokenization");
} else {
out->query_tokens = out->query.n_tokens;
if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"query\"]",
- "query exceeds 32768 token limit",
+ ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
"value_error.context_length");
}
@@ -2255,10 +2310,9 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
cJSON_ArrayForEach(document, documents) {
char loc[72];
snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i);
- if (tokenize_late_text(out->model, j, document->valuestring, 0,
- &out->documents[i]) != 0) {
- ve_add(detail, loc, "tokenization failed",
- "value_error.tokenization");
+ if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) !=
+ 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
int n = out->documents[i].n_tokens;
if (n > EMBED_API_MAX_ITEM_TOKENS)
@@ -2271,10 +2325,8 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s,
}
i++;
}
- if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS -
- out->document_tokens) {
- ve_add(detail, "[\"body\",\"documents\"]",
- "request exceeds 120000 token limit",
+ if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) {
+ ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
"value_error.context_length");
}
}
@@ -2297,8 +2349,10 @@ typedef struct {
static int rerank_result_cmp(const void *a, const void *b) {
const rerank_result *ra = a;
const rerank_result *rb = b;
- if (ra->score > rb->score) return -1;
- if (ra->score < rb->score) return 1;
+ if (ra->score > rb->score)
+ return -1;
+ if (ra->score < rb->score)
+ return 1;
return ra->index - rb->index;
}
@@ -2317,26 +2371,24 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) {
float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all));
int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
- int rc = embed_late_model_encode_tokens(
- m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens,
- 1, query);
+ int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
+ r->query.n_tokens, 1, query);
int pos = 0;
offsets[0] = 0;
for (int i = 0; rc == 0 && i < r->n_documents; i++) {
late_tokens *doc = &r->documents[i];
- rc = embed_late_model_encode_tokens(
- m->cpu_late_model, m->cpu_late_ws, doc->ids, doc->n_tokens, 1, all);
+ rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids,
+ doc->n_tokens, 1, all);
for (int k = 0; rc == 0 && k < doc->n_keep; k++) {
- memcpy(docs + (size_t)pos * dim,
- all + (size_t)doc->keep[k] * dim,
+ memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim,
(size_t)dim * sizeof(*docs));
pos++;
}
offsets[i + 1] = pos;
}
if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets,
- r->n_documents, dim, scores);
+ rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
+ scores);
free(offsets);
free(all);
free(docs);
@@ -2347,25 +2399,22 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) {
#ifdef USE_MLX
static int execute_rerank_mlx(rerank_request *r, float *scores) {
loaded_model *m = r->model;
- embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device(
- m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
- embed_mlx_late_vectors_t **docs =
- xcalloc((size_t)r->n_documents, sizeof(*docs));
- const embed_mlx_late_vectors_t **doc_refs =
- xcalloc((size_t)r->n_documents, sizeof(*doc_refs));
+ embed_mlx_late_vectors_t *query =
+ embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
+ embed_mlx_late_vectors_t **docs = xcalloc((size_t)r->n_documents, sizeof(*docs));
+ const embed_mlx_late_vectors_t **doc_refs = xcalloc((size_t)r->n_documents, sizeof(*doc_refs));
int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
int rc = query ? 0 : -1;
offsets[0] = 0;
for (int i = 0; rc == 0 && i < r->n_documents; i++) {
late_tokens *doc = &r->documents[i];
- embed_mlx_late_vectors_t *all = embed_mlx_late_encode_tokens_device(
- m->mlx_late_ctx, doc->ids, doc->n_tokens, 1);
+ embed_mlx_late_vectors_t *all =
+ embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, doc->ids, doc->n_tokens, 1);
if (!all) {
rc = -1;
break;
}
- docs[i] = embed_mlx_late_vectors_select(
- m->mlx_late_ctx, all, doc->keep, doc->n_keep);
+ docs[i] = embed_mlx_late_vectors_select(m->mlx_late_ctx, all, doc->keep, doc->n_keep);
embed_mlx_late_vectors_free(all);
if (!docs[i]) {
rc = -1;
@@ -2374,14 +2423,13 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) {
doc_refs[i] = docs[i];
offsets[i + 1] = offsets[i] + doc->n_keep;
}
- embed_mlx_late_vectors_t *packed = rc == 0
- ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs,
- r->n_documents)
- : NULL;
- if (!packed) rc = -1;
+ embed_mlx_late_vectors_t *packed =
+ rc == 0 ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, r->n_documents) : NULL;
+ if (!packed)
+ rc = -1;
if (rc == 0)
- rc = embed_mlx_late_maxsim_batch_device(
- m->mlx_late_ctx, query, packed, offsets, r->n_documents, scores);
+ rc = embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets,
+ r->n_documents, scores);
embed_mlx_late_vectors_free(packed);
for (int i = 0; i < r->n_documents; i++)
embed_mlx_late_vectors_free(docs[i]);
@@ -2396,29 +2444,24 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) {
static void execute_rerank_request(rerank_request *r) {
if (!r || !r->model) {
if (r && r->j)
- job_set_error(r->j, 500, "late-interaction model is unavailable",
- "server_error");
+ job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
return;
}
float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
uint64_t t0 = nstime();
#ifdef USE_MLX
- int rc = r->model->mlx_late_ctx
- ? execute_rerank_mlx(r, scores)
- : execute_rerank_cpu(r, scores);
+ int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
#else
int rc = execute_rerank_cpu(r, scores);
#endif
r->j->infer_ns += nstime() - t0;
if (rc != 0) {
free(scores);
- job_set_error(r->j, 500, "late-interaction reranking failed",
- "server_error");
+ job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
return;
}
- rerank_result *ranked =
- xmalloc((size_t)r->n_documents * sizeof(*ranked));
+ rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked));
for (int i = 0; i < r->n_documents; i++) {
ranked[i].index = i;
ranked[i].score = scores[i];
@@ -2427,14 +2470,13 @@ static void execute_rerank_request(rerank_request *r) {
t0 = nstime();
sbuf b = {0};
- sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[",
- r->model->info->id);
- cJSON *documents =
- cJSON_GetObjectItemCaseSensitive(r->root, "documents");
+ sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id);
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents");
for (int i = 0; i < r->top_n; i++) {
- if (i) sbuf_putc(&b, ',');
- sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g",
- ranked[i].index, (double)ranked[i].score);
+ if (i)
+ sbuf_putc(&b, ',');
+ sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index,
+ (double)ranked[i].score);
if (r->return_documents) {
cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index);
sbuf_puts(&b, ",\"document\":");
@@ -2443,10 +2485,9 @@ static void execute_rerank_request(rerank_request *r) {
sbuf_putc(&b, '}');
}
sbuf_printf(&b,
- "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->query_tokens, r->document_tokens,
- r->query_tokens + r->document_tokens);
+ "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens);
set_response_from_buf(r->j, &b);
r->j->encode_ns += nstime() - t0;
free(ranked);
@@ -2469,7 +2510,8 @@ static int parse_job_root(job *j, cJSON **out_root) {
static void process_unknown_job(job *j) {
cJSON *root = NULL;
- if (!j->started_ns) j->started_ns = nstime();
+ if (!j->started_ns)
+ j->started_ns = nstime();
uint64_t t0 = nstime();
int rc = parse_job_root(j, &root);
j->parse_ns += nstime() - t0;
@@ -2482,8 +2524,7 @@ static void process_unknown_job(job *j) {
static void process_job_group(http_server *s, job **jobs, int n_jobs) {
embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
- rerank_request *rerank_reqs =
- xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
+ rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
int *kind = xcalloc((size_t)n_jobs, sizeof(*kind));
int *done = xcalloc((size_t)n_jobs, sizeof(*done));
embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
@@ -2491,7 +2532,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int i = 0; i < n_jobs; i++) {
cJSON *root = NULL;
- if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime();
+ if (!jobs[i]->started_ns)
+ jobs[i]->started_ns = nstime();
if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
kind[i] = 1;
} else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
@@ -2526,8 +2568,7 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
continue;
}
- if ((kind[i] == 1 && !std_reqs[i].ready) ||
- (kind[i] == 2 && !ctx_reqs[i].ready) ||
+ if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) ||
(kind[i] == 3 && !rerank_reqs[i].ready)) {
job_set_timing_header(jobs[i]);
enqueue_done(jobs[i]);
@@ -2548,8 +2589,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
- embedding_request_fits_group(&std_reqs[k], group_inputs,
- group_tokens, s->batch_size, s->max_batch_tokens)) {
+ embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
std_group[group_n++] = &std_reqs[k];
group_inputs += std_reqs[k].n_inputs;
group_tokens += std_reqs[k].total_tokens;
@@ -2570,8 +2611,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
for (int k = i + 1; k < n_jobs; k++) {
if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
- contextual_request_fits_group(&ctx_reqs[k], group_docs,
- group_tokens, s->batch_size, s->max_batch_tokens)) {
+ contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens,
+ s->batch_size, s->max_batch_tokens)) {
ctx_group[group_n++] = &ctx_reqs[k];
group_docs += ctx_reqs[k].n_docs;
group_tokens += ctx_reqs[k].total_tokens;
@@ -2601,26 +2642,19 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
free(std_reqs);
}
-static int configure_loaded_model(loaded_model *m,
- const embed_config_t *config,
- int token_dim)
-{
- if (!m || !m->info || !config ||
- config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim ||
- config->attention_mode != m->info->attention_mode ||
+static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size != m->info->dim ||
+ token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode ||
config->pooling_mode != m->info->pooling_mode ||
config->normalize_embeddings != m->info->normalize_embeddings) {
if (m && m->info && config)
- server_log("embed-server: model %s has incompatible config",
- m->info->id);
+ server_log("embed-server: model %s has incompatible config", m->info->id);
return -1;
}
m->append_terminal_token = config->append_terminal_token;
/* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
m->renormalize_truncated =
- config->normalize_embeddings &&
- config->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
return 0;
}
@@ -2631,28 +2665,27 @@ static void *worker_main(void *arg) {
int rc = 0;
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
- if (!m->info) continue;
+ if (!m->info)
+ continue;
embed_mlx_options_t mlx_opts = {
.quantize_bits = s->mlx_quantize_bits,
.quantize_group_size = s->mlx_quantize_group_size,
};
if (m->info->kind == MODEL_KIND_LATE)
- m->mlx_late_ctx =
- embed_mlx_late_load_with_options(m->path, &mlx_opts);
+ m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts);
else
m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts);
if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) ||
(!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load MLX model on worker: %s",
- m->path);
+ server_log("embed-server: failed to load MLX model on worker: %s", m->path);
rc = 1;
break;
}
const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_config(m->mlx_late_ctx)
- : embed_mlx_config(m->mlx_ctx);
- int token_dim = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
+ ? embed_mlx_late_config(m->mlx_late_ctx)
+ : embed_mlx_config(m->mlx_ctx);
+ int token_dim =
+ m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
if (configure_loaded_model(m, config, token_dim) != 0 ||
(m->info->kind == MODEL_KIND_LATE &&
(m->late_mask_id >= config->vocab_size ||
@@ -2665,7 +2698,8 @@ static void *worker_main(void *arg) {
pthread_mutex_lock(&s->mu);
s->worker_init_rc = rc;
s->worker_ready = 1;
- if (rc) s->stopping = 1;
+ if (rc)
+ s->stopping = 1;
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
@@ -2685,20 +2719,19 @@ static void *worker_main(void *arg) {
} else
#endif
#ifdef USE_CUDA
- if (s->use_cuda) {
+ if (s->use_cuda) {
int rc = 0;
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
- if (!m->info) continue;
+ if (!m->info)
+ continue;
m->cuda_ctx = embed_cuda_load(m->path);
if (!m->cuda_ctx) {
- server_log("embed-server: failed to load CUDA model on worker: %s",
- m->path);
+ server_log("embed-server: failed to load CUDA model on worker: %s", m->path);
rc = 1;
break;
}
- if (configure_loaded_model(
- m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
+ if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
rc = 1;
break;
}
@@ -2706,7 +2739,8 @@ static void *worker_main(void *arg) {
pthread_mutex_lock(&s->mu);
s->worker_init_rc = rc;
s->worker_ready = 1;
- if (rc) s->stopping = 1;
+ if (rc)
+ s->stopping = 1;
pthread_cond_broadcast(&s->cv);
pthread_mutex_unlock(&s->mu);
if (rc) {
@@ -2730,9 +2764,9 @@ static void *worker_main(void *arg) {
for (;;) {
job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS];
- int n_jobs = collect_job_batch(
- s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
- if (n_jobs == 0) break;
+ int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
+ if (n_jobs == 0)
+ break;
process_job_group(s, jobs, n_jobs);
}
#ifdef USE_MLX
@@ -2792,8 +2826,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
m->tok_ws = qwen_tokenizer_workspace_new();
if (!m->tok_ws) {
- server_log("embed-server: failed to allocate tokenizer workspace: %s",
- path);
+ server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
return -1;
}
/* The contextual chunk separator is the tokenizer's <|endoftext|>. The
@@ -2802,8 +2835,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
* token (e.g. small test models) takes precedence so the id stays
* inside that model's embedding table. */
int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>");
- m->context_separator_id = sep_id >= 0
- ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
/* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
* suffix - the same token as the separator, not the model's chat
* eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
@@ -2812,16 +2844,13 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
int id = qwen_tokenizer_token_id(m->tok, "[MASK]");
m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
id = qwen_tokenizer_token_id(m->tok, "[Q]");
- m->late_query_prefix_id =
- id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
+ m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
id = qwen_tokenizer_token_id(m->tok, "[D]");
- m->late_document_prefix_id =
- id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
+ m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) /
- sizeof(m->late_skip_ids[0]));
+ for (const char *p = punct; *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) /
+ sizeof(m->late_skip_ids[0]));
p++) {
char text[2] = {*p, '\0'};
int n_ids = 0;
@@ -2854,8 +2883,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
if (!m->cpu_late_ws) {
- server_log("embed-server: failed to allocate late workspace: %s",
- path);
+ server_log("embed-server: failed to allocate late workspace: %s", path);
return -1;
}
config = embed_late_model_config(m->cpu_late_model);
@@ -2877,8 +2905,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
return -1;
}
if (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
+ (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
m->late_document_prefix_id >= config->vocab_size)) {
server_log("embed-server: late special-token ids exceed model vocab");
return -1;
@@ -2890,18 +2917,27 @@ static void free_models(http_server *s) {
for (int i = 0; i < MODEL_COUNT; i++) {
loaded_model *m = &s->models[i];
free(m->path);
- if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws);
- if (m->tok) qwen_tokenizer_free(m->tok);
- if (m->cpu_ws) embed_workspace_free(m->cpu_ws);
- if (m->cpu_model) embed_model_free(m->cpu_model);
- if (m->cpu_late_ws) embed_late_workspace_free(m->cpu_late_ws);
- if (m->cpu_late_model) embed_late_model_free(m->cpu_late_model);
+ if (m->tok_ws)
+ qwen_tokenizer_workspace_free(m->tok_ws);
+ if (m->tok)
+ qwen_tokenizer_free(m->tok);
+ if (m->cpu_ws)
+ embed_workspace_free(m->cpu_ws);
+ if (m->cpu_model)
+ embed_model_free(m->cpu_model);
+ if (m->cpu_late_ws)
+ embed_late_workspace_free(m->cpu_late_ws);
+ if (m->cpu_late_model)
+ embed_late_model_free(m->cpu_late_model);
#ifdef USE_MLX
- if (m->mlx_ctx) embed_mlx_free(m->mlx_ctx);
- if (m->mlx_late_ctx) embed_mlx_late_free(m->mlx_late_ctx);
+ if (m->mlx_ctx)
+ embed_mlx_free(m->mlx_ctx);
+ if (m->mlx_late_ctx)
+ embed_mlx_late_free(m->mlx_late_ctx);
#endif
#ifdef USE_CUDA
- if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx);
+ if (m->cuda_ctx)
+ embed_cuda_free(m->cuda_ctx);
#endif
}
}
@@ -2910,14 +2946,12 @@ static uint64_t physical_memory_nbytes(void) {
#ifdef __APPLE__
uint64_t bytes = 0;
size_t len = sizeof(bytes);
- if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 &&
- len == sizeof(bytes) && bytes > 0)
+ if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0)
return bytes;
#endif
long pages = sysconf(_SC_PHYS_PAGES);
long page_size = sysconf(_SC_PAGESIZE);
- if (pages <= 0 || page_size <= 0 ||
- (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
+ if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
return 0;
return (uint64_t)pages * (uint64_t)page_size;
}
@@ -2928,13 +2962,11 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
const embed_server_model_spec_t *spec = &cfg->models[i];
model_slot slot = model_slot_for_id(spec->id);
if (slot == MODEL_UNKNOWN) {
- fprintf(stderr, "embed-server: unknown model id: %s\n",
- spec->id ? spec->id : "<null>");
+ fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : "<null>");
return -1;
}
if (!spec->path || !spec->path[0]) {
- fprintf(stderr, "embed-server: model %s has an empty path\n",
- spec->id);
+ fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id);
return -1;
}
if (seen[slot]) {
@@ -2942,8 +2974,7 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
return -1;
}
if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) {
- fprintf(stderr,
- "embed-server: model %s has no CUDA late-interaction backend\n",
+ fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n",
spec->id);
return -1;
}
@@ -2953,17 +2984,16 @@ static int validate_model_specs(const embed_server_config_t *cfg) {
}
static int mlx_memory_preflight(const embed_server_config_t *cfg) {
- if (!cfg->use_mlx) return 0;
+ if (!cfg->use_mlx)
+ return 0;
uint64_t payload = 0;
for (int i = 0; i < cfg->n_models; i++) {
- multi_safetensors_t *ms =
- multi_safetensors_open(cfg->models[i].path);
+ multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path);
size_t model_payload = 0;
if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
multi_safetensors_close(ms);
- fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n",
- cfg->models[i].path);
+ fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path);
return -1;
}
multi_safetensors_close(ms);
@@ -2985,22 +3015,21 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) {
"skipping MLX memory preflight");
return 0;
}
- double utilization = cfg->memory_utilization > 0.0
- ? cfg->memory_utilization
- : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
+ double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization
+ : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
uint64_t budget = (uint64_t)((double)physical * utilization);
const double gib = 1024.0 * 1024.0 * 1024.0;
server_log("embed-server: MLX memory preflight: %.1f GiB tensors, "
"%.1f GiB estimated resident, %.1f GiB budget "
"(%.2f of physical memory)",
- (double)payload / gib, (double)estimated / gib,
- (double)budget / gib, utilization);
+ (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization);
if (cfg->mlx_quantize_bits) {
server_log("embed-server: MLX %d-bit quantization enabled; preflight "
"uses source payload as a conservative peak estimate",
cfg->mlx_quantize_bits);
}
- if (estimated <= budget) return 0;
+ if (estimated <= budget)
+ return 0;
server_log("embed-server: refusing MLX model set above the host-memory "
"budget");
server_log("embed-server: use BF16 artifacts, load fewer models, or raise "
@@ -3020,24 +3049,23 @@ int embed_run_server(const embed_server_config_t *cfg) {
memset(&s, 0, sizeof(s));
s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
s.port = cfg->port > 0 ? cfg->port : 8000;
- s.batch_size = cfg->batch_size > 0
- ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
- s.max_batch_tokens = cfg->max_batch_tokens > 0
- ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
- s.batch_wait_us = cfg->batch_wait_us >= 0
- ? cfg->batch_wait_us
- : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
- : EMBED_SERVER_BATCH_WAIT_US);
+ s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
+ s.max_batch_tokens =
+ cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
+ s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us
+ : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
+ : EMBED_SERVER_BATCH_WAIT_US);
s.use_mlx = cfg->use_mlx;
s.use_cuda = cfg->use_cuda;
s.mlx_quantize_bits = cfg->mlx_quantize_bits;
- s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0
- ? cfg->mlx_quantize_group_size : 64;
+ s.mlx_quantize_group_size =
+ cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64;
if (cfg->api_key && cfg->api_key[0])
s.api_key = xstrdup(cfg->api_key);
else {
const char *env_key = getenv("EMBED_API_KEY");
- if (env_key && env_key[0]) s.api_key = xstrdup(env_key);
+ if (env_key && env_key[0])
+ s.api_key = xstrdup(env_key);
}
pthread_mutex_init(&s.mu, NULL);
@@ -3124,7 +3152,8 @@ int embed_run_server(const embed_server_config_t *cfg) {
aeMain(s.loop);
server_log("embed-server: shutdown requested");
- if (s.listen_fd >= 0) close(s.listen_fd);
+ if (s.listen_fd >= 0)
+ close(s.listen_fd);
client *c = s.clients;
while (c) {
client *next = c->next;
@@ -3155,59 +3184,58 @@ int embed_run_server(const embed_server_config_t *cfg) {
* ==================================================================== */
#ifndef EMBED_SERVER_TEST
-static void print_usage(const char *prog)
-{
+static void print_usage(const char *prog) {
fprintf(stderr,
- "Usage: %s --model ID=DIR [options]\n"
- "\n"
- "Options:\n"
- " --model ID=DIR Model to serve (repeatable)\n"
+ "Usage: %s --model ID=DIR [options]\n"
+ "\n"
+ "Options:\n"
+ " --model ID=DIR Model to serve (repeatable)\n"
#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
+ " --mlx Use Apple MLX GPU backend\n"
#endif
#ifdef USE_CUDA
- " --cuda Use NVIDIA CUDA GPU backend\n"
+ " --cuda Use NVIDIA CUDA GPU backend\n"
#endif
- " --host HOST Bind host (default: 127.0.0.1)\n"
- " --port N Bind port (default: 8000)\n"
- " --api-key K Require Authorization: Bearer K\n"
- " -b, --batch-size N Max texts or documents per inference batch\n"
- " (default: 32)\n"
- " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
- " --batch-wait-us N First-arrival micro-batch deadline in us\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 32)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N First-arrival micro-batch deadline in us\n"
#ifdef USE_CUDA
- " (default: CUDA 1000; CPU 0)\n"
+ " (default: CUDA 1000; CPU 0)\n"
#else
- " (default: 0)\n"
+ " (default: 0)\n"
#endif
#ifdef USE_MLX
- " --memory-utilization F Fraction of physical memory the MLX model-set\n"
- " preflight may plan for (default: 0.90; values\n"
- " above 1.0 overcommit)\n"
- " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
- " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
+ " --memory-utilization F Fraction of physical memory the MLX model-set\n"
+ " preflight may plan for (default: 0.90; values\n"
+ " above 1.0 overcommit)\n"
+ " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
+ " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
#endif
#ifdef USE_CUDA
- " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
- " (default: f32, exact)\n"
- " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
- " f32); bf16 halves weight memory\n"
+ " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
+ " (default: f32, exact)\n"
+ " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
+ " f32); bf16 halves weight memory\n"
#endif
- " -t, --threads N CPU threads (default: all cores)\n"
- " -v, --verbose Verbose (-vv for debug)\n"
- " -V, --version Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h, --help Show this help\n"
- "\n"
- "Examples:\n"
- " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " -t, --threads N CPU threads (default: all cores)\n"
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -V, --version Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
#ifdef USE_MLX
- " %s --mlx --mlx-quant-bits 8 \\\n"
- " --model pplx-embed-v1-4b=./model-4b-bf16\n",
- prog, prog, prog);
+ " %s --mlx --mlx-quant-bits 8 \\\n"
+ " --model pplx-embed-v1-4b=./model-4b-bf16\n",
+ prog, prog, prog);
#else
- ,
- prog, prog);
+ ,
+ prog, prog);
#endif
}
@@ -3217,8 +3245,7 @@ typedef struct {
int cap;
} model_specs_t;
-static int append_model_spec(model_specs_t *specs, const char *arg)
-{
+static int append_model_spec(model_specs_t *specs, const char *arg) {
const char *eq = strchr(arg, '=');
if (!eq || eq == arg || !eq[1]) {
fprintf(stderr, "--model expects MODEL_ID=DIR\n");
@@ -3227,7 +3254,8 @@ static int append_model_spec(model_specs_t *specs, const char *arg)
if (specs->n == specs->cap) {
int new_cap = specs->cap ? specs->cap * 2 : 4;
void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p) return -1;
+ if (!p)
+ return -1;
specs->v = p;
specs->cap = new_cap;
}
@@ -3246,9 +3274,9 @@ static int append_model_spec(model_specs_t *specs, const char *arg)
return 0;
}
-static void free_model_specs(model_specs_t *specs)
-{
- if (!specs) return;
+static void free_model_specs(model_specs_t *specs) {
+ if (!specs)
+ return;
for (int i = 0; i < specs->n; i++) {
free((char *)specs->v[i].id);
free((char *)specs->v[i].path);
@@ -3256,8 +3284,7 @@ static void free_model_specs(model_specs_t *specs)
free(specs->v);
}
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
int n_threads = 0;
int verbose = 0;
int batch_size = 0;
@@ -3287,37 +3314,34 @@ int main(int argc, char *argv[])
embed_print_version(prog);
free_model_specs(&model_specs);
return 0;
- }
- else if (!strcmp(f, "--build-info")) {
+ } else if (!strcmp(f, "--build-info")) {
embed_print_build_info(prog);
free_model_specs(&model_specs);
return 0;
- }
- else if (!strcmp(f, "--model")) {
+ } else if (!strcmp(f, "--model")) {
if (append_model_spec(&model_specs, argv[++arg]) != 0) {
free_model_specs(&model_specs);
return 1;
}
}
#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
- else if (!strcmp(f, "--mlx-quant-bits")) {
+ else if (!strcmp(f, "--mlx")) {
+ use_mlx = 1;
+ } else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg]);
if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
+ } else if (!strcmp(f, "--mlx-quant-group-size")) {
mlx_quantize_group_size = atoi(argv[++arg]);
if (mlx_quantize_group_size <= 0) {
fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--memory-utilization")) {
+ } else if (!strcmp(f, "--memory-utilization")) {
memory_utilization = atof(argv[++arg]);
if (memory_utilization <= 0.0) {
fprintf(stderr, "--memory-utilization must be > 0\n");
@@ -3327,47 +3351,47 @@ int main(int argc, char *argv[])
}
#endif
#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
- else if (!strcmp(f, "--cuda-gemm-mode")) {
+ else if (!strcmp(f, "--cuda")) {
+ use_cuda = 1;
+ } else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
+ } else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg];
}
#endif
- else if (!strcmp(f, "--host")) { host = argv[++arg]; }
- else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
- else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ else if (!strcmp(f, "--host")) {
+ host = argv[++arg];
+ } else if (!strcmp(f, "--port")) {
+ port = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--api-key")) {
+ api_key = argv[++arg];
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "--max-batch-tokens")) {
+ } else if (!strcmp(f, "--max-batch-tokens")) {
max_batch_tokens = atoi(argv[++arg]);
if (max_batch_tokens <= 0) {
fprintf(stderr, "--max-batch-tokens must be > 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "--batch-wait-us")) {
+ } else if (!strcmp(f, "--batch-wait-us")) {
batch_wait_us = atoi(argv[++arg]);
if (batch_wait_us < 0) {
fprintf(stderr, "--batch-wait-us must be >= 0\n");
free_model_specs(&model_specs);
return 1;
}
- }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
n_threads = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(prog);
free_model_specs(&model_specs);
return 0;
- }
- else {
+ } else {
fprintf(stderr, "unknown option: %s\n", f);
print_usage(prog);
free_model_specs(&model_specs);
@@ -3431,18 +3455,22 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (use_mlx) {
- if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ if (verbose >= 1)
+ fprintf(stderr, "Using MLX GPU backend\n");
} else
#endif
#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ if (use_cuda) {
+ if (verbose >= 1)
+ fprintf(stderr, "Using CUDA GPU backend\n");
} else
#endif
{
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ if (n_threads <= 0)
+ n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ if (verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
}
embed_server_config_t scfg = {
diff --git a/embed_server.h b/embed_server.h
index c10dddb..2a39b94 100644
--- a/embed_server.h
+++ b/embed_server.h
@@ -31,7 +31,7 @@ typedef struct {
int use_cuda;
int mlx_quantize_bits;
int mlx_quantize_group_size;
- double memory_utilization; /* fraction of physical memory the MLX
+ double memory_utilization; /* fraction of physical memory the MLX
preflight may plan for; 0 = default 0.90,
values above 1.0 allow overcommit */
const char *api_key;
diff --git a/qwen_kernels.c b/qwen_kernels.c
index dcd724c..c7f5762 100644
--- a/qwen_kernels.c
+++ b/qwen_kernels.c
@@ -5,7 +5,8 @@
#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
-#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
+#if (defined(__AVX512F__) || defined(__AVX2__)) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
#include <immintrin.h>
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
@@ -92,8 +93,10 @@ static void *worker_loop(void *arg) {
}
void qwen_set_threads(int n) {
- if (n < 1) n = 1;
- if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS;
+ if (n < 1)
+ n = 1;
+ if (n > QWEN_MAX_THREADS)
+ n = QWEN_MAX_THREADS;
#if defined(USE_OPENBLAS)
/* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the
@@ -119,8 +122,7 @@ void qwen_set_threads(int n) {
if (n <= 1) {
if (qwen_verbose >= 2) {
#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n",
- n, blas_threads);
+ fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
#else
fprintf(stderr, "Thread pool: %d threads\n", n);
#endif
@@ -135,8 +137,7 @@ void qwen_set_threads(int n) {
if (qwen_verbose >= 2) {
#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n",
- n, blas_threads);
+ fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
#else
fprintf(stderr, "Thread pool: %d threads\n", n);
#endif
@@ -183,7 +184,8 @@ static void parallel_for(parallel_fn_t fn, void *arg) {
* ======================================================================== */
void qwen_add_inplace(float *a, const float *b, int n) {
- for (int i = 0; i < n; i++) a[i] += b[i];
+ for (int i = 0; i < n; i++)
+ a[i] += b[i];
}
/* ========================================================================
@@ -192,8 +194,7 @@ void qwen_add_inplace(float *a, const float *b, int n) {
void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) {
#ifdef USE_BLAS
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
- M, N, K, 1.0f, A, K, B, K, 0.0f, C, N);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N);
#else
for (int m = 0; m < M; m++) {
for (int n = 0; n < N; n++) {
@@ -207,18 +208,20 @@ void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N
#endif
}
-void qwen_linear(float *y, const float *x, const float *W, const float *b,
- int seq_len, int in_dim, int out_dim) {
+void qwen_linear(float *y,
+ const float *x,
+ const float *W,
+ const float *b,
+ int seq_len,
+ int in_dim,
+ int out_dim) {
#ifdef USE_BLAS
if (seq_len == 1) {
- cblas_sgemv(CblasRowMajor, CblasNoTrans,
- out_dim, in_dim, 1.0f, W, in_dim, x, 1,
- 0.0f, y, 1);
+ cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y,
+ 1);
} else {
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
- seq_len, out_dim, in_dim,
- 1.0f, x, in_dim, W, in_dim,
- 0.0f, y, out_dim);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x,
+ in_dim, W, in_dim, 0.0f, y, out_dim);
}
if (b != NULL) {
for (int s = 0; s < seq_len; s++) {
@@ -243,8 +246,8 @@ void qwen_linear(float *y, const float *x, const float *W, const float *b,
#endif
}
-void qwen_linear_nobias(float *y, const float *x, const float *W,
- int seq_len, int in_dim, int out_dim) {
+void qwen_linear_nobias(
+ float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim);
}
@@ -273,8 +276,8 @@ void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
* Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias
* Processes 2 output rows at a time to amortize x vector loads.
*/
-static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+static void bf16_matvec_fused(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim);
}
@@ -293,22 +296,22 @@ static void matvec_worker(int tid, int n_threads, void *arg) {
int chunk = (t->out_dim + n_threads - 1) / n_threads;
int start = tid * chunk;
int end = start + chunk;
- if (end > t->out_dim) end = t->out_dim;
- if (start >= end) return;
+ if (end > t->out_dim)
+ end = t->out_dim;
+ if (start >= end)
+ return;
- bf16_matvec_fused(t->y + start, t->x,
- t->W_bf16 + (size_t)start * t->in_dim,
- t->bias ? t->bias + start : NULL,
- t->in_dim, end - start);
+ bf16_matvec_fused(t->y + start, t->x, t->W_bf16 + (size_t)start * t->in_dim,
+ t->bias ? t->bias + start : NULL, t->in_dim, end - start);
}
-static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+static void bf16_matvec_threaded(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
if (tp.n_threads <= 1) {
bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim);
return;
}
- matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim };
+ matvec_task_t task = {y, x, W_bf16, bias, in_dim, out_dim};
parallel_for(matvec_worker, &task);
}
@@ -327,18 +330,23 @@ static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) {
int chunk = (t->seq_len + n_threads - 1) / n_threads;
int start = tid * chunk;
int end = start + chunk;
- if (end > t->seq_len) end = t->seq_len;
- if (start >= end) return;
+ if (end > t->seq_len)
+ end = t->seq_len;
+ if (start >= end)
+ return;
for (int s = start; s < end; s++) {
- bf16_matvec_fused(t->y + (size_t)s * t->out_dim,
- t->x + (size_t)s * t->in_dim,
- t->W_bf16, t->bias, t->in_dim, t->out_dim);
+ bf16_matvec_fused(t->y + (size_t)s * t->out_dim, t->x + (size_t)s * t->in_dim, t->W_bf16,
+ t->bias, t->in_dim, t->out_dim);
}
}
-static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int seq_len, int in_dim,
+static void bf16_linear_rows(float *y,
+ const float *x,
+ const uint16_t *W_bf16,
+ const float *bias,
+ int seq_len,
+ int in_dim,
int out_dim) {
bf16_linear_rows_task_t task = {
.y = y,
@@ -369,8 +377,7 @@ typedef struct {
int total_dim;
} pair_matvec_task_t;
-static void pair_matvec_range(const pair_matvec_task_t *t, int row,
- int start, int end) {
+static void pair_matvec_range(const pair_matvec_task_t *t, int row, int start, int end) {
const float *x_row = t->x + (size_t)row * t->in_dim;
int a_end = t->a_dim;
int b_end = a_end + t->b_dim;
@@ -380,8 +387,7 @@ static void pair_matvec_range(const pair_matvec_task_t *t, int row,
int e = end < a_end ? end : a_end;
if (s < e) {
bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row,
- t->Wa_bf16 + (size_t)s * t->in_dim,
- NULL, t->in_dim, e - s);
+ t->Wa_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
}
}
@@ -391,8 +397,7 @@ static void pair_matvec_range(const pair_matvec_task_t *t, int row,
int e = e_abs - a_end;
if (s < e) {
bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row,
- t->Wb_bf16 + (size_t)s * t->in_dim,
- NULL, t->in_dim, e - s);
+ t->Wb_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
}
}
}
@@ -403,8 +408,10 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) {
int chunk = (total_work + n_threads - 1) / n_threads;
int start = tid * chunk;
int end = start + chunk;
- if (end > total_work) end = total_work;
- if (start >= end) return;
+ if (end > total_work)
+ end = total_work;
+ if (start >= end)
+ return;
while (start < end) {
int row = start / t->total_dim;
@@ -416,20 +423,23 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) {
}
}
-void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x,
+void qwen_linear_nobias_bf16_pair(float *a,
+ float *b,
+ const float *x,
const uint16_t *Wa_bf16,
const uint16_t *Wb_bf16,
- int seq_len, int in_dim, int a_dim,
+ int seq_len,
+ int in_dim,
+ int a_dim,
int b_dim) {
- if (seq_len <= 0) return;
+ if (seq_len <= 0)
+ return;
if (tp.n_threads <= 1) {
for (int s = 0; s < seq_len; s++) {
const float *x_row = x + (size_t)s * in_dim;
- bf16_matvec_fused(a + (size_t)s * a_dim, x_row,
- Wa_bf16, NULL, in_dim, a_dim);
- bf16_matvec_fused(b + (size_t)s * b_dim, x_row,
- Wb_bf16, NULL, in_dim, b_dim);
+ bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim);
+ bf16_matvec_fused(b + (size_t)s * b_dim, x_row, Wb_bf16, NULL, in_dim, b_dim);
}
return;
}
@@ -464,8 +474,7 @@ typedef struct {
int total_dim;
} qkv_matvec_task_t;
-static void qkv_matvec_range(const qkv_matvec_task_t *t, int row,
- int start, int end) {
+static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int start, int end) {
const float *x_row = t->x + (size_t)row * t->in_dim;
int q_end = t->q_dim;
int k_end = q_end + t->kv_dim;
@@ -476,8 +485,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row,
int e = end < q_end ? end : q_end;
if (s < e) {
bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row,
- t->Wq_bf16 + (size_t)s * t->in_dim,
- NULL, t->in_dim, e - s);
+ t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
}
}
@@ -487,8 +495,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row,
int e = e_abs - q_end;
if (s < e) {
bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row,
- t->Wk_bf16 + (size_t)s * t->in_dim,
- NULL, t->in_dim, e - s);
+ t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
}
}
@@ -498,8 +505,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row,
int e = e_abs - k_end;
if (s < e) {
bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row,
- t->Wv_bf16 + (size_t)s * t->in_dim,
- NULL, t->in_dim, e - s);
+ t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
}
}
}
@@ -510,8 +516,10 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
int chunk = (total_work + n_threads - 1) / n_threads;
int start = tid * chunk;
int end = start + chunk;
- if (end > total_work) end = total_work;
- if (start >= end) return;
+ if (end > total_work)
+ end = total_work;
+ if (start >= end)
+ return;
while (start < end) {
int row = start / t->total_dim;
@@ -523,23 +531,26 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
}
}
-void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
+void qwen_linear_nobias_bf16_qkv(float *q,
+ float *k,
+ float *v,
+ const float *x,
const uint16_t *Wq_bf16,
const uint16_t *Wk_bf16,
const uint16_t *Wv_bf16,
- int seq_len, int in_dim, int q_dim,
+ int seq_len,
+ int in_dim,
+ int q_dim,
int kv_dim) {
- if (seq_len <= 0) return;
+ if (seq_len <= 0)
+ return;
if (tp.n_threads <= 1) {
for (int s = 0; s < seq_len; s++) {
const float *x_row = x + (size_t)s * in_dim;
- bf16_matvec_fused(q + (size_t)s * q_dim, x_row,
- Wq_bf16, NULL, in_dim, q_dim);
- bf16_matvec_fused(k + (size_t)s * kv_dim, x_row,
- Wk_bf16, NULL, in_dim, kv_dim);
- bf16_matvec_fused(v + (size_t)s * kv_dim, x_row,
- Wv_bf16, NULL, in_dim, kv_dim);
+ bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim);
+ bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, Wk_bf16, NULL, in_dim, kv_dim);
+ bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, Wv_bf16, NULL, in_dim, kv_dim);
}
return;
}
@@ -561,8 +572,8 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
parallel_for(qkv_matvec_worker, &task);
}
-void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
- int seq_len, int in_dim, int out_dim) {
+void qwen_linear_nobias_bf16(
+ float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) {
if (seq_len == 1) {
bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
return;
@@ -573,7 +584,6 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
}
-
/* ========================================================================
* Normalization
* ======================================================================== */
@@ -582,9 +592,8 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
* for the dispatch. */
#define QWEN_RMS_NORM_PARALLEL_ELEMS 262144
-static void qwen_rms_norm_range(float *out, const float *x,
- const float *weight, int start, int end,
- int hidden, float eps) {
+static void qwen_rms_norm_range(
+ float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) {
for (int s = start; s < end; s++) {
const float *x_row = x + (size_t)s * hidden;
float *out_row = out + (size_t)s * hidden;
@@ -597,7 +606,8 @@ static void qwen_rms_norm_range(float *out, const float *x,
accv = _mm512_fmadd_ps(v, v, accv);
}
float sum_sq = _mm512_reduce_add_ps(accv);
- for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i];
+ for (; i < hidden; i++)
+ sum_sq += x_row[i] * x_row[i];
#elif defined(__AVX2__) && defined(__FMA__)
__m256 accv = _mm256_setzero_ps();
int i = 0;
@@ -609,7 +619,8 @@ static void qwen_rms_norm_range(float *out, const float *x,
acc128 = _mm_hadd_ps(acc128, acc128);
acc128 = _mm_hadd_ps(acc128, acc128);
float sum_sq = _mm_cvtss_f32(acc128);
- for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i];
+ for (; i < hidden; i++)
+ sum_sq += x_row[i] * x_row[i];
#else
float sum_sq = 0.0f;
for (int i = 0; i < hidden; i++) {
@@ -626,7 +637,8 @@ static void qwen_rms_norm_range(float *out, const float *x,
__m512 vw = _mm512_loadu_ps(weight + j);
_mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale));
}
- for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j];
+ for (; j < hidden; j++)
+ out_row[j] = x_row[j] * rms_inv * weight[j];
#elif defined(__AVX2__)
__m256 scale = _mm256_set1_ps(rms_inv);
int j = 0;
@@ -635,7 +647,8 @@ static void qwen_rms_norm_range(float *out, const float *x,
__m256 vw = _mm256_loadu_ps(weight + j);
_mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale));
}
- for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j];
+ for (; j < hidden; j++)
+ out_row[j] = x_row[j] * rms_inv * weight[j];
#else
for (int i = 0; i < hidden; i++) {
out_row[i] = x_row[i] * rms_inv * weight[i];
@@ -658,15 +671,17 @@ static void rms_norm_worker(int tid, int n_threads, void *arg) {
int chunk = (t->seq_len + n_threads - 1) / n_threads;
int start = tid * chunk;
int end = start + chunk;
- if (end > t->seq_len) end = t->seq_len;
- if (start >= end) return;
- qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden,
- t->eps);
+ if (end > t->seq_len)
+ end = t->seq_len;
+ if (start >= end)
+ return;
+ qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps);
}
-void qwen_rms_norm(float *out, const float *x, const float *weight,
- int seq_len, int hidden, float eps) {
- if (seq_len <= 0 || hidden <= 0) return;
+void qwen_rms_norm(
+ float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) {
+ if (seq_len <= 0 || hidden <= 0)
+ return;
long long elems = (long long)seq_len * hidden;
if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) {
@@ -684,8 +699,8 @@ void qwen_rms_norm(float *out, const float *x, const float *weight,
}
}
-void qwen_rms_norm_per_head(float *x, const float *weight,
- int seq_len, int n_heads, int head_dim, float eps) {
+void qwen_rms_norm_per_head(
+ float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) {
/* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
int hidden = n_heads * head_dim;
for (int s = 0; s < seq_len; s++) {
@@ -700,7 +715,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight,
accv = _mm512_fmadd_ps(v, v, accv);
}
float sum_sq = _mm512_reduce_add_ps(accv);
- for (; d < head_dim; d++) sum_sq += vec[d] * vec[d];
+ for (; d < head_dim; d++)
+ sum_sq += vec[d] * vec[d];
#elif defined(__AVX2__) && defined(__FMA__)
__m256 accv = _mm256_setzero_ps();
int d = 0;
@@ -708,11 +724,13 @@ void qwen_rms_norm_per_head(float *x, const float *weight,
__m256 v = _mm256_loadu_ps(vec + d);
accv = _mm256_fmadd_ps(v, v, accv);
}
- __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
+ __m128 acc128 =
+ _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
acc128 = _mm_hadd_ps(acc128, acc128);
acc128 = _mm_hadd_ps(acc128, acc128);
float sum_sq = _mm_cvtss_f32(acc128);
- for (; d < head_dim; d++) sum_sq += vec[d] * vec[d];
+ for (; d < head_dim; d++)
+ sum_sq += vec[d] * vec[d];
#else
float sum_sq = 0.0f;
for (int d = 0; d < head_dim; d++) {
@@ -729,7 +747,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight,
__m512 w = _mm512_loadu_ps(weight + j);
_mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale));
}
- for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j];
+ for (; j < head_dim; j++)
+ vec[j] = vec[j] * rms_inv * weight[j];
#elif defined(__AVX2__)
__m256 scale = _mm256_set1_ps(rms_inv);
int j = 0;
@@ -738,7 +757,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight,
__m256 w = _mm256_loadu_ps(weight + j);
_mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale));
}
- for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j];
+ for (; j < head_dim; j++)
+ vec[j] = vec[j] * rms_inv * weight[j];
#else
for (int d = 0; d < head_dim; d++) {
vec[d] = vec[d] * rms_inv * weight[d];
@@ -759,7 +779,8 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n) {
for (int base = 0; base < n; base += CHUNK) {
int len = n - base;
- if (len > CHUNK) len = CHUNK;
+ if (len > CHUNK)
+ len = CHUNK;
for (int i = 0; i < len; i++)
exp_neg[i] = -gate[base + i];
@@ -783,7 +804,8 @@ void qwen_softmax(float *x, int rows, int cols) {
float *row = x + r * cols;
float max_val = row[0];
for (int c = 1; c < cols; c++) {
- if (row[c] > max_val) max_val = row[c];
+ if (row[c] > max_val)
+ max_val = row[c];
}
float sum = 0.0f;
for (int c = 0; c < cols; c++) {
@@ -801,9 +823,7 @@ void qwen_softmax(float *x, int rows, int cols) {
* Attention Operations
* ======================================================================== */
-float qwen_dot_f32(const float *a, const float *b, int n) {
- return qwen_dot_f32_impl(a, b, n);
-}
+float qwen_dot_f32(const float *a, const float *b, int n) { return qwen_dot_f32_impl(a, b, n); }
static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) {
return qwen_dot_f32_impl(a, b, n);
@@ -824,13 +844,22 @@ static inline void qwen_vec_scale_add(float *dst, const float *src, float correc
qwen_vec_scale_add_impl(dst, src, correction, n);
}
-#define QWEN_PACKED_ATTN_QUERY_TILE 32
+#define QWEN_PACKED_ATTN_QUERY_TILE 32
#define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128
-static void qwen_bidirectional_gqa_attention_packed_online_rows(
- float *out, const float *Q, const float *K, const float *V,
- int start, int end, int n_heads, int n_kv_heads, int head_dim,
- float scale, int h, int query_start, int query_end) {
+static void qwen_bidirectional_gqa_attention_packed_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
int heads_per_kv = n_heads / n_kv_heads;
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
@@ -842,7 +871,8 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows(
float max_score = -1e30f;
float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o_row[d] = 0.0f;
for (int j = start; j < end; j++) {
const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
@@ -870,10 +900,20 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows(
}
#ifdef USE_BLAS
-static void qwen_bidirectional_gqa_attention_packed_blas_rows(
- float *out, const float *Q, const float *K, const float *V,
- int start, int end, int n_heads, int n_kv_heads, int head_dim,
- float scale, int h, int query_start, int query_end, float *scores) {
+static void qwen_bidirectional_gqa_attention_packed_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
int heads_per_kv = n_heads / n_kv_heads;
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
@@ -886,13 +926,11 @@ static void qwen_bidirectional_gqa_attention_packed_blas_rows(
const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
float *o = out + (size_t)query_start * q_hidden + h * head_dim;
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
- rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden,
- 0.0f, scores, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
+ k, kv_hidden, 0.0f, scores, keys);
qwen_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
- rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden,
- 0.0f, o, q_hidden);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
+ v, kv_hidden, 0.0f, o, q_hidden);
}
#endif
@@ -914,9 +952,8 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
int task_id = 0;
#ifdef USE_BLAS
- float *scores = t->scores
- ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq
- : NULL;
+ float *scores =
+ t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
#endif
for (int h = 0; h < t->n_heads; h++) {
@@ -925,41 +962,43 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
int end = t->offsets[b + 1];
for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) {
int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE;
- if (q1 > end) q1 = end;
- if (task_id++ % n_threads != tid) continue;
+ if (q1 > end)
+ q1 = end;
+ if (task_id++ % n_threads != tid)
+ continue;
#ifdef USE_BLAS
if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) {
qwen_bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0, q1,
- scores);
+ t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads,
+ t->head_dim, t->scale, h, q0, q1, scores);
continue;
}
#endif
qwen_bidirectional_gqa_attention_packed_online_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0, q1);
+ t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1);
}
}
}
}
-size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
- const int *offsets, int batch) {
+size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
#ifdef USE_BLAS
- if (!offsets || batch <= 0) return 0;
+ if (!offsets || batch <= 0)
+ return 0;
int max_seq = 0;
for (int b = 0; b < batch; b++) {
int len = offsets[b + 1] - offsets[b];
- if (len > max_seq) max_seq = len;
+ if (len > max_seq)
+ max_seq = len;
}
- if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0;
- if ((size_t)max_seq > SIZE_MAX / (sizeof(float) *
- QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads))
+ if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ)
+ return 0;
+ if ((size_t)max_seq >
+ SIZE_MAX / (sizeof(float) * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads))
return 0;
- return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE *
- (size_t)max_seq * sizeof(float);
+ return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
#else
(void)offsets;
(void)batch;
@@ -967,33 +1006,48 @@ size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
#endif
}
-void qwen_bidirectional_gqa_attention_packed_with_scratch(
- float *out, const float *Q, const float *K, const float *V,
- const int *offsets, int batch, int n_heads, int n_kv_heads,
- int head_dim, float scale, float *scratch, size_t scratch_bytes) {
+void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
long long qk_work = 0;
int max_seq = 0;
for (int b = 0; b < batch; b++) {
int len = offsets[b + 1] - offsets[b];
qk_work += (long long)len * len * n_heads;
- if (len > max_seq) max_seq = len;
+ if (len > max_seq)
+ max_seq = len;
}
float *scores = NULL;
#ifdef USE_BLAS
- size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(
- offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required) scores = scratch;
+ size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required)
+ scores = scratch;
#else
(void)scratch;
(void)scratch_bytes;
#endif
- packed_gqa_attn_task_t task = {
- .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
- .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads,
- .head_dim = head_dim, .scale = scale, .scores = scores,
- .max_seq = max_seq
- };
+ packed_gqa_attn_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .offsets = offsets,
+ .batch = batch,
+ .n_heads = n_heads,
+ .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores,
+ .max_seq = max_seq};
if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) {
parallel_for(packed_gqa_attn_worker, &task);
} else {
@@ -1001,24 +1055,35 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch(
}
}
-void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale) {
- size_t scratch_bytes =
- qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+void qwen_bidirectional_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale) {
+ size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
qwen_bidirectional_gqa_attention_packed_with_scratch(
- out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
- scores, scratch_bytes);
+ out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes);
free(scores);
}
-static void qwen_causal_gqa_attention_packed_online_rows(
- float *out, const float *Q, const float *K, const float *V,
- int start, int n_heads, int n_kv_heads, int head_dim,
- float scale, int h, int query_start, int query_end) {
+static void qwen_causal_gqa_attention_packed_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
int heads_per_kv = n_heads / n_kv_heads;
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
@@ -1030,7 +1095,8 @@ static void qwen_causal_gqa_attention_packed_online_rows(
float max_score = -1e30f;
float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o_row[d] = 0.0f;
for (int j = start; j <= i; j++) {
const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
@@ -1055,10 +1121,19 @@ static void qwen_causal_gqa_attention_packed_online_rows(
}
#ifdef USE_BLAS
-static void qwen_causal_gqa_attention_packed_blas_rows(
- float *out, const float *Q, const float *K, const float *V,
- int start, int n_heads, int n_kv_heads, int head_dim,
- float scale, int h, int query_start, int query_end, float *scores) {
+static void qwen_causal_gqa_attention_packed_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
int heads_per_kv = n_heads / n_kv_heads;
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
@@ -1071,18 +1146,17 @@ static void qwen_causal_gqa_attention_packed_blas_rows(
const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
float *o = out + (size_t)query_start * q_hidden + h * head_dim;
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
- rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden,
- 0.0f, scores, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
+ k, kv_hidden, 0.0f, scores, keys);
for (int r = 0; r < rows; r++) {
int allowed = query_start + r - start + 1;
float *row = scores + (size_t)r * keys;
- for (int j = allowed; j < keys; j++) row[j] = -INFINITY;
+ for (int j = allowed; j < keys; j++)
+ row[j] = -INFINITY;
}
qwen_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
- rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden,
- 0.0f, o, q_hidden);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
+ v, kv_hidden, 0.0f, o, q_hidden);
}
#endif
@@ -1090,9 +1164,8 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
int task_id = 0;
#ifdef USE_BLAS
- float *scores = t->scores
- ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq
- : NULL;
+ float *scores =
+ t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
#endif
for (int h = 0; h < t->n_heads; h++) {
@@ -1101,69 +1174,88 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
int end = t->offsets[b + 1];
for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) {
int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE;
- if (q1 > end) q1 = end;
- if (task_id++ % n_threads != tid) continue;
+ if (q1 > end)
+ q1 = end;
+ if (task_id++ % n_threads != tid)
+ continue;
#ifdef USE_BLAS
if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) {
qwen_causal_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0, q1,
- scores);
+ t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1, scores);
continue;
}
#endif
- qwen_causal_gqa_attention_packed_online_rows(
- t->out, t->Q, t->K, t->V, start, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0, q1);
+ qwen_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start,
+ t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1);
}
}
}
}
-void qwen_causal_gqa_attention_packed_with_scratch(
- float *out, const float *Q, const float *K, const float *V,
- const int *offsets, int batch, int n_heads, int n_kv_heads,
- int head_dim, float scale, float *scratch, size_t scratch_bytes) {
+void qwen_causal_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
long long qk_work = 0;
int max_seq = 0;
for (int b = 0; b < batch; b++) {
int len = offsets[b + 1] - offsets[b];
qk_work += (long long)len * (len + 1) / 2 * n_heads;
- if (len > max_seq) max_seq = len;
+ if (len > max_seq)
+ max_seq = len;
}
float *scores = NULL;
#ifdef USE_BLAS
- size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(
- offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required) scores = scratch;
+ size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required)
+ scores = scratch;
#else
(void)scratch;
(void)scratch_bytes;
#endif
- packed_gqa_attn_task_t task = {
- .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
- .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads,
- .head_dim = head_dim, .scale = scale, .scores = scores,
- .max_seq = max_seq
- };
+ packed_gqa_attn_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .offsets = offsets,
+ .batch = batch,
+ .n_heads = n_heads,
+ .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores,
+ .max_seq = max_seq};
if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096)
parallel_for(packed_causal_gqa_attn_worker, &task);
else
packed_causal_gqa_attn_worker(0, 1, &task);
}
-void qwen_causal_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale) {
- size_t scratch_bytes =
- qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+void qwen_causal_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale) {
+ size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- qwen_causal_gqa_attention_packed_with_scratch(
- out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
- scores, scratch_bytes);
+ qwen_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, scores, scratch_bytes);
free(scores);
}
@@ -1171,8 +1263,8 @@ void qwen_causal_gqa_attention_packed(float *out, const float *Q,
* Position Embeddings
* ======================================================================== */
-void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions,
- int seq, int head_dim, float theta) {
+void qwen_compute_rope_neox(
+ float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) {
int half = head_dim / 2;
for (int s = 0; s < seq; s++) {
@@ -1191,8 +1283,8 @@ void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions
}
}
-void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals,
- int seq, int n_heads, int head_dim) {
+void qwen_apply_rope_neox(
+ float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) {
/*
* NeoX split-half style:
* x1 = x[..., :half], x2 = x[..., half:]
@@ -1225,7 +1317,7 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#elif defined(__AVX2__) && defined(__FMA__)
@@ -1243,7 +1335,7 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
@@ -1261,14 +1353,14 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#else
for (int d = 0; d < half; d++) {
- float x1 = vec[d]; /* first half */
- float x2 = vec[half + d]; /* second half */
- vec[d] = x1 * c[d] + (-x2) * sn[d];
+ float x1 = vec[d]; /* first half */
+ float x2 = vec[half + d]; /* second half */
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#endif
diff --git a/qwen_kernels.h b/qwen_kernels.h
index c611194..3a6a4d6 100644
--- a/qwen_kernels.h
+++ b/qwen_kernels.h
@@ -35,34 +35,42 @@ void qwen_add_inplace(float *a, const float *b, int n);
void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */
-void qwen_linear(float *y, const float *x, const float *W, const float *b,
- int seq_len, int in_dim, int out_dim);
+void qwen_linear(
+ float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
-void qwen_linear_nobias(float *y, const float *x, const float *W,
- int seq_len, int in_dim, int out_dim);
+void qwen_linear_nobias(
+ float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
/* bf16 weight variants */
void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n);
-void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16,
- int seq_len, int in_dim, int out_dim);
+void qwen_linear_nobias_bf16(
+ float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim);
/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
-void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x,
+void qwen_linear_nobias_bf16_pair(float *a,
+ float *b,
+ const float *x,
const uint16_t *Wa_bf16,
const uint16_t *Wb_bf16,
- int seq_len, int in_dim, int a_dim,
+ int seq_len,
+ int in_dim,
+ int a_dim,
int b_dim);
/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
-void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
+void qwen_linear_nobias_bf16_qkv(float *q,
+ float *k,
+ float *v,
+ const float *x,
const uint16_t *Wq_bf16,
const uint16_t *Wk_bf16,
const uint16_t *Wv_bf16,
- int seq_len, int in_dim, int q_dim,
+ int seq_len,
+ int in_dim,
+ int q_dim,
int kv_dim);
-
/* Dot product using the best available local SIMD implementation. */
float qwen_dot_f32(const float *a, const float *b, int n);
@@ -71,14 +79,14 @@ float qwen_dot_f32(const float *a, const float *b, int n);
* ======================================================================== */
/* RMS Normalization: out = x / rms(x) * weight */
-void qwen_rms_norm(float *out, const float *x, const float *weight,
- int seq_len, int hidden, float eps);
+void qwen_rms_norm(
+ float *out, const float *x, const float *weight, int seq_len, int hidden, float eps);
/* Per-head RMS Normalization for Q/K norms in decoder
* x: [seq, n_heads, head_dim], weight: [head_dim]
* Normalizes each head independently */
-void qwen_rms_norm_per_head(float *x, const float *weight,
- int seq_len, int n_heads, int head_dim, float eps);
+void qwen_rms_norm_per_head(
+ float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps);
/* ========================================================================
* Activation Functions
@@ -102,33 +110,58 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n);
* Attention is block-diagonal: tokens only attend within their own
* offsets[b]..offsets[b+1] span.
*/
-void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale);
+void qwen_bidirectional_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
/* Optional reusable scratch for the BLAS-tiled packed attention path. */
-size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
- const int *offsets, int batch);
-void qwen_bidirectional_gqa_attention_packed_with_scratch(
- float *out, const float *Q, const float *K, const float *V,
- const int *offsets, int batch, int n_heads, int n_kv_heads,
- int head_dim, float scale, float *scratch, size_t scratch_bytes);
+size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
+void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/*
* Causal GQA attention over a packed/ragged batch. Query position i attends
* only to positions at or before i within its sequence.
*/
-void qwen_causal_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale);
-void qwen_causal_gqa_attention_packed_with_scratch(
- float *out, const float *Q, const float *K, const float *V,
- const int *offsets, int batch, int n_heads, int n_kv_heads,
- int head_dim, float scale, float *scratch, size_t scratch_bytes);
+void qwen_causal_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
+void qwen_causal_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/* ========================================================================
* Position Embeddings
@@ -139,16 +172,16 @@ void qwen_causal_gqa_attention_packed_with_scratch(
* cos_out, sin_out: [seq, head_dim]
* cos[d] and cos[half+d] are the same (duplicated for full head_dim).
*/
-void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions,
- int seq, int head_dim, float theta);
+void qwen_compute_rope_neox(
+ float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta);
/*
* Apply NeoX-style RoPE to Q or K.
* x: [seq, n_heads * head_dim] (in-place)
* cos_vals, sin_vals: [seq, head_dim]
*/
-void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals,
- int seq, int n_heads, int head_dim);
+void qwen_apply_rope_neox(
+ float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim);
/* ========================================================================
* Threading
diff --git a/qwen_kernels_avx.c b/qwen_kernels_avx.c
index 0fc1df8..42c358b 100644
--- a/qwen_kernels_avx.c
+++ b/qwen_kernels_avx.c
@@ -22,8 +22,8 @@
#if defined(__AVX512F__) && defined(__AVX512BW__)
-void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+void qwen_bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
/* Process 4 output rows at a time */
@@ -46,26 +46,34 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16
/* Row 0 */
__m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
__m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
- a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
- a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
/* Row 1 */
__m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
__m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
- a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
- a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
/* Row 2 */
__m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k));
__m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16));
- a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4);
- a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5);
+ a4 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4);
+ a5 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5);
/* Row 3 */
__m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k));
__m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16));
- a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6);
- a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7);
+ a6 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6);
+ a7 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7);
}
for (; k + 16 <= in_dim; k += 16) {
@@ -74,26 +82,39 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16
__m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
__m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k));
__m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k));
- a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
- a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
- a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4);
- a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6);
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ a4 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4);
+ a6 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6);
}
- float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
- float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f);
- float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f);
- float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f);
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
+ float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o + 2] : 0.0f);
+ float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o + 3] : 0.0f);
for (; k < in_dim; k++) {
float xk = x[k];
- uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
- uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16;
- float v0,v1,v2,v3;
- memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4);
- s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk;
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ uint32_t b2 = ((uint32_t)w2[k]) << 16, b3 = ((uint32_t)w3[k]) << 16;
+ float v0, v1, v2, v3;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ memcpy(&v2, &b2, 4);
+ memcpy(&v3, &b3, 4);
+ s0 += v0 * xk;
+ s1 += v1 * xk;
+ s2 += v2 * xk;
+ s3 += v3 * xk;
}
- y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3;
+ y[o] = s0;
+ y[o + 1] = s1;
+ y[o + 2] = s2;
+ y[o + 3] = s3;
}
/* Handle 2-row remainder */
@@ -111,28 +132,38 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16
__m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
__m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
__m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
- a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
- a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
- a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
- a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
}
for (; k + 16 <= in_dim; k += 16) {
__m512 xv = _mm512_loadu_ps(x + k);
__m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
__m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
- a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
- a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
}
- float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
- float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f);
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
for (; k < in_dim; k++) {
- uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
- float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4);
- s0+=v0*x[k]; s1+=v1*x[k];
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ float v0, v1;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ s0 += v0 * x[k];
+ s1 += v1 * x[k];
}
- y[o]=s0; y[o+1]=s1;
+ y[o] = s0;
+ y[o + 1] = s1;
}
/* Single remaining row */
@@ -142,13 +173,16 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16
int k = 0;
for (; k + 16 <= in_dim; k += 16) {
__m256i raw = _mm256_loadu_si256((const __m256i *)(w + k));
- acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
- _mm512_loadu_ps(x + k), acc);
+ acc = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
+ _mm512_loadu_ps(x + k), acc);
}
float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f);
for (; k < in_dim; k++) {
- uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
- sum += wv*x[k];
+ uint32_t bits = ((uint32_t)w[k]) << 16;
+ float wv;
+ memcpy(&wv, &bits, 4);
+ sum += wv * x[k];
}
y[o] = sum;
}
@@ -167,15 +201,15 @@ static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) {
*hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1));
}
-void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+void qwen_bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
for (; o + 1 < out_dim; o += 2) {
const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
const uint16_t *w1 = w0 + in_dim;
- __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps();
- __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps();
+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
int k = 0;
for (; k + 16 <= in_dim; k += 16) {
__m256 xlo = _mm256_loadu_ps(x + k);
@@ -188,36 +222,45 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16
a2 = _mm256_fmadd_ps(wlo, xlo, a2);
a3 = _mm256_fmadd_ps(whi, xhi, a3);
}
- a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3);
+ a0 = _mm256_add_ps(a0, a1);
+ a2 = _mm256_add_ps(a2, a3);
__m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
__m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1));
- r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0);
+ r0 = _mm_hadd_ps(r0, r1);
+ r0 = _mm_hadd_ps(r0, r0);
float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f);
- float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f);
+ float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0, r0, 1)) + (bias ? bias[o + 1] : 0.0f);
for (; k < in_dim; k++) {
- uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16;
- float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4);
- s0+=v0*x[k]; s1+=v1*x[k];
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ float v0, v1;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ s0 += v0 * x[k];
+ s1 += v1 * x[k];
}
- y[o]=s0; y[o+1]=s1;
+ y[o] = s0;
+ y[o + 1] = s1;
}
for (; o < out_dim; o++) {
const uint16_t *w = W_bf16 + (size_t)o * in_dim;
- __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps();
+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
int k = 0;
for (; k + 16 <= in_dim; k += 16) {
__m256 wlo, whi;
bf16x16_to_f32(w + k, &wlo, &whi);
- a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0);
- a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1);
+ a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x + k), a0);
+ a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x + k + 8), a1);
}
a0 = _mm256_add_ps(a0, a1);
__m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
- r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f);
for (; k < in_dim; k++) {
- uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4);
- sum+=wv*x[k];
+ uint32_t bits = ((uint32_t)w[k]) << 16;
+ float wv;
+ memcpy(&wv, &bits, 4);
+ sum += wv * x[k];
}
y[o] = sum;
}
@@ -238,7 +281,7 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) {
__m512 acc2 = _mm512_setzero_ps();
__m512 acc3 = _mm512_setzero_ps();
for (; i + 64 <= n; i += 64) {
- acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
+ acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
@@ -248,7 +291,8 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) {
acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
}
float sum = _mm512_reduce_add_ps(acc);
- for (; i < n; i++) sum += a[i] * b[i];
+ for (; i < n; i++)
+ sum += a[i] * b[i];
return sum;
#else
int i = 0;
@@ -257,20 +301,21 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) {
__m256 acc2 = _mm256_setzero_ps();
__m256 acc3 = _mm256_setzero_ps();
for (; i + 32 <= n; i += 32) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0);
- acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1);
- acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2);
- acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3);
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
+ acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
+ acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
+ acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
}
acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
for (; i + 8 <= n; i += 8) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0);
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
}
__m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
r = _mm_hadd_ps(r, r);
r = _mm_hadd_ps(r, r);
float sum = _mm_cvtss_f32(r);
- for (; i < n; i++) sum += a[i] * b[i];
+ for (; i < n; i++)
+ sum += a[i] * b[i];
return sum;
#endif
}
@@ -280,7 +325,7 @@ void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) {
int i = 0;
__m512 s = _mm512_set1_ps(scale);
for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
_mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
_mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
_mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
@@ -288,20 +333,22 @@ void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) {
for (; i + 16 <= n; i += 16) {
_mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
}
- for (; i < n; i++) dst[i] *= scale;
+ for (; i < n; i++)
+ dst[i] *= scale;
#else
int i = 0;
__m256 s = _mm256_set1_ps(scale);
for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s));
- _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s));
- _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s));
- _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s));
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
+ _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
+ _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
+ _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
}
for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s));
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
}
- for (; i < n; i++) dst[i] *= scale;
+ for (; i < n; i++)
+ dst[i] *= scale;
#endif
}
@@ -312,31 +359,38 @@ void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n)
for (; i + 64 <= n; i += 64) {
_mm512_storeu_ps(dst + i,
_mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- _mm512_storeu_ps(dst + i + 16,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16)));
- _mm512_storeu_ps(dst + i + 32,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32)));
- _mm512_storeu_ps(dst + i + 48,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
+ _mm512_loadu_ps(dst + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
+ _mm512_loadu_ps(dst + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
+ _mm512_loadu_ps(dst + i + 48)));
}
for (; i + 16 <= n; i += 16) {
_mm512_storeu_ps(dst + i,
_mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
}
- for (; i < n; i++) dst[i] += alpha * src[i];
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
#else
int i = 0;
__m256 a = _mm256_set1_ps(alpha);
for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i)));
- _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8)));
- _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16)));
- _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24)));
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
+ _mm256_loadu_ps(dst + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
+ _mm256_loadu_ps(dst + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
+ _mm256_loadu_ps(dst + i + 24)));
}
for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i)));
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
}
- for (; i < n; i++) dst[i] += alpha * src[i];
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
#endif
}
@@ -347,31 +401,38 @@ void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int
for (; i + 64 <= n; i += 64) {
_mm512_storeu_ps(dst + i,
_mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- _mm512_storeu_ps(dst + i + 16,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16)));
- _mm512_storeu_ps(dst + i + 32,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32)));
- _mm512_storeu_ps(dst + i + 48,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
+ _mm512_loadu_ps(src + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
+ _mm512_loadu_ps(src + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
+ _mm512_loadu_ps(src + i + 48)));
}
for (; i + 16 <= n; i += 16) {
_mm512_storeu_ps(dst + i,
_mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
}
- for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
#else
int i = 0;
__m256 c = _mm256_set1_ps(correction);
for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i)));
- _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8)));
- _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16)));
- _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24)));
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
+ _mm256_loadu_ps(src + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
+ _mm256_loadu_ps(src + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
+ _mm256_loadu_ps(src + i + 24)));
}
for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i)));
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
}
- for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
#endif
}
diff --git a/qwen_kernels_generic.c b/qwen_kernels_generic.c
index fd07455..149dfe0 100644
--- a/qwen_kernels_generic.c
+++ b/qwen_kernels_generic.c
@@ -6,8 +6,8 @@
#include <string.h>
-void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+void qwen_bf16_matvec_fused_generic(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
for (int o = 0; o < out_dim; o++) {
const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
float sum = bias ? bias[o] : 0.0f;
@@ -23,18 +23,22 @@ void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_
float qwen_dot_f32_generic(const float *a, const float *b, int n) {
float sum = 0.0f;
- for (int i = 0; i < n; i++) sum += a[i] * b[i];
+ for (int i = 0; i < n; i++)
+ sum += a[i] * b[i];
return sum;
}
void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) {
- for (int i = 0; i < n; i++) dst[i] *= scale;
+ for (int i = 0; i < n; i++)
+ dst[i] *= scale;
}
void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
- for (int i = 0; i < n; i++) dst[i] += alpha * src[i];
+ for (int i = 0; i < n; i++)
+ dst[i] += alpha * src[i];
}
void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
- for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i];
+ for (int i = 0; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
}
diff --git a/qwen_kernels_impl.h b/qwen_kernels_impl.h
index 43e4fbe..217f1e8 100644
--- a/qwen_kernels_impl.h
+++ b/qwen_kernels_impl.h
@@ -7,47 +7,47 @@
#include <stdint.h>
-void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim);
+void qwen_bf16_matvec_fused_generic(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
float qwen_dot_f32_generic(const float *a, const float *b, int n);
void qwen_vec_scale_inplace_generic(float *dst, float scale, int n);
void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n);
#ifdef __ARM_NEON
-void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim);
+void qwen_bf16_matvec_fused_neon(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
float qwen_dot_f32_neon(const float *a, const float *b, int n);
void qwen_vec_scale_inplace_neon(float *dst, float scale, int n);
void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n);
void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n);
#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon
-#define qwen_dot_f32_impl qwen_dot_f32_neon
+#define qwen_dot_f32_impl qwen_dot_f32_neon
#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon
-#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon
-#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon
#elif defined(__AVX2__) && defined(__FMA__)
-void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim);
+void qwen_bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
float qwen_dot_f32_avx(const float *a, const float *b, int n);
void qwen_vec_scale_inplace_avx(float *dst, float scale, int n);
void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n);
void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n);
#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx
-#define qwen_dot_f32_impl qwen_dot_f32_avx
+#define qwen_dot_f32_impl qwen_dot_f32_avx
#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx
-#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx
-#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx
#else
#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic
-#define qwen_dot_f32_impl qwen_dot_f32_generic
+#define qwen_dot_f32_impl qwen_dot_f32_generic
#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic
-#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic
-#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic
+#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic
+#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic
#endif
#endif /* QWEN_KERNELS_IMPL_H */
diff --git a/qwen_kernels_neon.c b/qwen_kernels_neon.c
index d656f1d..5d51fc6 100644
--- a/qwen_kernels_neon.c
+++ b/qwen_kernels_neon.c
@@ -9,8 +9,8 @@
#include <arm_neon.h>
#include <string.h>
-void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16,
- const float *bias, int in_dim, int out_dim) {
+void qwen_bf16_matvec_fused_neon(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
/* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */
@@ -128,7 +128,8 @@ float qwen_dot_f32_neon(const float *a, const float *b, int n) {
acc1 = vfmaq_f32(acc1, a1, b1);
}
float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
- for (; i < n; i++) sum += a[i] * b[i];
+ for (; i < n; i++)
+ sum += a[i] * b[i];
return sum;
}
@@ -141,7 +142,8 @@ void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) {
vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
}
- for (; i < n; i++) dst[i] *= scale;
+ for (; i < n; i++)
+ dst[i] *= scale;
}
void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) {
@@ -155,7 +157,8 @@ void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n
vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
}
- for (; i < n; i++) dst[i] += alpha * src[i];
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
}
void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) {
@@ -169,7 +172,8 @@ void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int
vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
}
- for (; i < n; i++) dst[i] = dst[i] * correction + src[i];
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
}
#endif /* __ARM_NEON */
diff --git a/qwen_safetensors.c b/qwen_safetensors.c
index 030798e..e1c0e87 100644
--- a/qwen_safetensors.c
+++ b/qwen_safetensors.c
@@ -13,29 +13,37 @@
* ======================================================================== */
static void skip_whitespace(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++;
+ while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
+ (*p)++;
}
static int parse_string(const char **p, char *out, size_t max_len) {
skip_whitespace(p);
- if (**p != '"') return -1;
+ if (**p != '"')
+ return -1;
(*p)++;
size_t i = 0;
while (**p && **p != '"' && i < max_len - 1) {
if (**p == '\\') {
(*p)++;
- if (**p == 'n') out[i++] = '\n';
- else if (**p == 't') out[i++] = '\t';
- else if (**p == '"') out[i++] = '"';
- else if (**p == '\\') out[i++] = '\\';
- else out[i++] = **p;
+ if (**p == 'n')
+ out[i++] = '\n';
+ else if (**p == 't')
+ out[i++] = '\t';
+ else if (**p == '"')
+ out[i++] = '"';
+ else if (**p == '\\')
+ out[i++] = '\\';
+ else
+ out[i++] = **p;
} else {
out[i++] = **p;
}
(*p)++;
}
out[i] = '\0';
- if (**p != '"') return -1;
+ if (**p != '"')
+ return -1;
(*p)++;
return 0;
}
@@ -44,8 +52,12 @@ static int parse_int(const char **p, int64_t *out) {
skip_whitespace(p);
int64_t val = 0;
int neg = 0;
- if (**p == '-') { neg = 1; (*p)++; }
- if (**p < '0' || **p > '9') return -1;
+ if (**p == '-') {
+ neg = 1;
+ (*p)++;
+ }
+ if (**p < '0' || **p > '9')
+ return -1;
while (**p >= '0' && **p <= '9') {
val = val * 10 + (**p - '0');
(*p)++;
@@ -55,18 +67,25 @@ static int parse_int(const char **p, int64_t *out) {
}
static safetensor_dtype_t parse_dtype(const char *s) {
- if (strcmp(s, "F32") == 0) return DTYPE_F32;
- if (strcmp(s, "F16") == 0) return DTYPE_F16;
- if (strcmp(s, "BF16") == 0) return DTYPE_BF16;
- if (strcmp(s, "I32") == 0) return DTYPE_I32;
- if (strcmp(s, "I64") == 0) return DTYPE_I64;
- if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL;
+ if (strcmp(s, "F32") == 0)
+ return DTYPE_F32;
+ if (strcmp(s, "F16") == 0)
+ return DTYPE_F16;
+ if (strcmp(s, "BF16") == 0)
+ return DTYPE_BF16;
+ if (strcmp(s, "I32") == 0)
+ return DTYPE_I32;
+ if (strcmp(s, "I64") == 0)
+ return DTYPE_I64;
+ if (strcmp(s, "BOOL") == 0)
+ return DTYPE_BOOL;
return DTYPE_UNKNOWN;
}
static int parse_tensor_entry(const char **p, safetensor_t *t) {
skip_whitespace(p);
- if (**p != '{') return -1;
+ if (**p != '{')
+ return -1;
(*p)++;
t->dtype = DTYPE_UNKNOWN;
@@ -76,79 +95,109 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) {
while (**p && **p != '}') {
skip_whitespace(p);
- if (**p == ',') { (*p)++; continue; }
+ if (**p == ',') {
+ (*p)++;
+ continue;
+ }
char key[64];
- if (parse_string(p, key, sizeof(key)) != 0) return -1;
+ if (parse_string(p, key, sizeof(key)) != 0)
+ return -1;
skip_whitespace(p);
- if (**p != ':') return -1;
+ if (**p != ':')
+ return -1;
(*p)++;
skip_whitespace(p);
if (strcmp(key, "dtype") == 0) {
char dtype_str[32];
- if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1;
+ if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
+ return -1;
t->dtype = parse_dtype(dtype_str);
} else if (strcmp(key, "shape") == 0) {
- if (**p != '[') return -1;
+ if (**p != '[')
+ return -1;
(*p)++;
t->ndim = 0;
while (**p && **p != ']') {
skip_whitespace(p);
- if (**p == ',') { (*p)++; continue; }
- if (t->ndim >= 8) return -1;
- if (parse_int(p, &t->shape[t->ndim]) != 0) return -1;
+ if (**p == ',') {
+ (*p)++;
+ continue;
+ }
+ if (t->ndim >= 8)
+ return -1;
+ if (parse_int(p, &t->shape[t->ndim]) != 0)
+ return -1;
t->ndim++;
}
- if (**p != ']') return -1;
+ if (**p != ']')
+ return -1;
(*p)++;
} else if (strcmp(key, "data_offsets") == 0) {
- if (**p != '[') return -1;
+ if (**p != '[')
+ return -1;
(*p)++;
skip_whitespace(p);
int64_t start_i;
- if (parse_int(p, &start_i) != 0) return -1;
+ if (parse_int(p, &start_i) != 0)
+ return -1;
skip_whitespace(p);
- if (**p != ',') return -1;
+ if (**p != ',')
+ return -1;
(*p)++;
skip_whitespace(p);
int64_t end_i;
- if (parse_int(p, &end_i) != 0) return -1;
- if (start_i < 0 || end_i < start_i) return -1;
+ if (parse_int(p, &end_i) != 0)
+ return -1;
+ if (start_i < 0 || end_i < start_i)
+ return -1;
t->data_offset = (size_t)start_i;
t->data_size = (size_t)(end_i - start_i);
skip_whitespace(p);
- if (**p != ']') return -1;
+ if (**p != ']')
+ return -1;
(*p)++;
} else {
/* Skip unknown value */
if (**p == '"') {
(*p)++;
while (**p && **p != '"') {
- if (**p == '\\') (*p)++;
- if (**p) (*p)++;
+ if (**p == '\\')
+ (*p)++;
+ if (**p)
+ (*p)++;
}
- if (**p == '"') (*p)++;
+ if (**p == '"')
+ (*p)++;
} else if (**p == '[') {
- int depth = 1; (*p)++;
+ int depth = 1;
+ (*p)++;
while (**p && depth > 0) {
- if (**p == '[') depth++;
- else if (**p == ']') depth--;
+ if (**p == '[')
+ depth++;
+ else if (**p == ']')
+ depth--;
(*p)++;
}
} else if (**p == '{') {
- int depth = 1; (*p)++;
+ int depth = 1;
+ (*p)++;
while (**p && depth > 0) {
- if (**p == '{') depth++;
- else if (**p == '}') depth--;
+ if (**p == '{')
+ depth++;
+ else if (**p == '}')
+ depth--;
(*p)++;
}
} else {
- while (**p && **p != ',' && **p != '}') (*p)++;
+ while (**p && **p != ',' && **p != '}')
+ (*p)++;
}
}
}
- if (**p != '}') return -1;
+ if (**p != '}')
+ return -1;
(*p)++;
return 0;
}
@@ -156,29 +205,39 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) {
static int parse_header(safetensors_file_t *sf) {
const char *p = sf->header_json;
skip_whitespace(&p);
- if (*p != '{') return -1;
+ if (*p != '{')
+ return -1;
p++;
sf->num_tensors = 0;
while (*p && *p != '}') {
skip_whitespace(&p);
- if (*p == ',') { p++; continue; }
- if (*p == '}') break;
+ if (*p == ',') {
+ p++;
+ continue;
+ }
+ if (*p == '}')
+ break;
char name[256];
- if (parse_string(&p, name, sizeof(name)) != 0) return -1;
+ if (parse_string(&p, name, sizeof(name)) != 0)
+ return -1;
skip_whitespace(&p);
- if (*p != ':') return -1;
+ if (*p != ':')
+ return -1;
p++;
if (strcmp(name, "__metadata__") == 0) {
skip_whitespace(&p);
if (*p == '{') {
- int depth = 1; p++;
+ int depth = 1;
+ p++;
while (*p && depth > 0) {
- if (*p == '{') depth++;
- else if (*p == '}') depth--;
+ if (*p == '{')
+ depth++;
+ else if (*p == '}')
+ depth--;
p++;
}
}
@@ -193,10 +252,12 @@ static int parse_header(safetensors_file_t *sf) {
safetensor_t *t = &sf->tensors[sf->num_tensors];
snprintf(t->name, sizeof(t->name), "%s", name);
- if (parse_tensor_entry(&p, t) != 0) return -1;
+ if (parse_tensor_entry(&p, t) != 0)
+ return -1;
sf->num_tensors++;
}
- if (*p != '}') return -1;
+ if (*p != '}')
+ return -1;
return 0;
}
@@ -206,24 +267,38 @@ static int parse_header(safetensors_file_t *sf) {
safetensors_file_t *safetensors_open(const char *path) {
int fd = open(path, O_RDONLY);
- if (fd < 0) return NULL;
+ if (fd < 0)
+ return NULL;
struct stat st;
- if (fstat(fd, &st) < 0) { close(fd); return NULL; }
+ if (fstat(fd, &st) < 0) {
+ close(fd);
+ return NULL;
+ }
size_t file_size = (size_t)st.st_size;
- if (file_size < 8) { close(fd); return NULL; }
+ if (file_size < 8) {
+ close(fd);
+ return NULL;
+ }
void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
- if (data == MAP_FAILED) return NULL;
+ if (data == MAP_FAILED)
+ return NULL;
uint64_t header_size = 0;
memcpy(&header_size, data, 8);
- if (header_size > file_size - 8) { munmap(data, file_size); return NULL; }
+ if (header_size > file_size - 8) {
+ munmap(data, file_size);
+ return NULL;
+ }
safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
- if (!sf) { munmap(data, file_size); return NULL; }
+ if (!sf) {
+ munmap(data, file_size);
+ return NULL;
+ }
sf->path = strdup(path);
sf->data = data;
@@ -231,18 +306,26 @@ safetensors_file_t *safetensors_open(const char *path) {
sf->header_size = (size_t)header_size;
sf->header_json = malloc(header_size + 1);
- if (!sf->header_json) { safetensors_close(sf); return NULL; }
+ if (!sf->header_json) {
+ safetensors_close(sf);
+ return NULL;
+ }
memcpy(sf->header_json, (char *)data + 8, header_size);
sf->header_json[header_size] = '\0';
- if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; }
+ if (parse_header(sf) != 0) {
+ safetensors_close(sf);
+ return NULL;
+ }
return sf;
}
void safetensors_close(safetensors_file_t *sf) {
- if (!sf) return;
- if (sf->data) munmap(sf->data, sf->file_size);
+ if (!sf)
+ return;
+ if (sf->data)
+ munmap(sf->data, sf->file_size);
free(sf->path);
free(sf->header_json);
free(sf);
@@ -254,7 +337,8 @@ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t
int64_t safetensor_numel(const safetensor_t *t) {
int64_t n = 1;
- for (int i = 0; i < t->ndim; i++) n *= t->shape[i];
+ for (int i = 0; i < t->ndim; i++)
+ n *= t->shape[i];
return n;
}
@@ -267,25 +351,28 @@ static float bf16_to_f32(uint16_t bf16) {
float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
int64_t n = safetensor_numel(t);
- if (n <= 0) return NULL;
+ if (n <= 0)
+ return NULL;
float *out = malloc(n * sizeof(float));
- if (!out) return NULL;
+ if (!out)
+ return NULL;
const void *data = safetensors_data(sf, t);
switch (t->dtype) {
- case DTYPE_F32:
- memcpy(out, data, n * sizeof(float));
- break;
- case DTYPE_BF16: {
- const uint16_t *src = (const uint16_t *)data;
- for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]);
- break;
- }
- default:
- free(out);
- return NULL;
+ case DTYPE_F32:
+ memcpy(out, data, n * sizeof(float));
+ break;
+ case DTYPE_BF16: {
+ const uint16_t *src = (const uint16_t *)data;
+ for (int64_t i = 0; i < n; i++)
+ out[i] = bf16_to_f32(src[i]);
+ break;
+ }
+ default:
+ free(out);
+ return NULL;
}
return out;
}
@@ -313,7 +400,8 @@ static int parse_shard_name(const char *name, int *idx, int *total) {
multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
- if (!ms) return NULL;
+ if (!ms)
+ return NULL;
char path[4096];
@@ -328,7 +416,10 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
/* Scan directory for shard files */
DIR *dir = opendir(model_dir);
- if (!dir) { free(ms); return NULL; }
+ if (!dir) {
+ free(ms);
+ return NULL;
+ }
struct dirent *entry;
char shard_names[SAFETENSORS_MAX_SHARDS][256];
@@ -341,15 +432,14 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
strstr(entry->d_name, ".safetensors") != NULL) {
int shard_idx = 0, shard_total = 0;
if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
- fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n",
- entry->d_name);
+ fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
closedir(dir);
free(ms);
return NULL;
}
if (shard_total > SAFETENSORS_MAX_SHARDS) {
- fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n",
- shard_total, SAFETENSORS_MAX_SHARDS);
+ fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
+ SAFETENSORS_MAX_SHARDS);
closedir(dir);
free(ms);
return NULL;
@@ -364,16 +454,14 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
return NULL;
}
if (seen[shard_idx - 1]) {
- fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n",
- shard_idx);
+ fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
closedir(dir);
free(ms);
return NULL;
}
seen[shard_idx - 1] = 1;
- snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]),
- "%s", entry->d_name);
+ snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
n_shards++;
}
}
@@ -392,8 +480,8 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
}
for (int i = 0; i < expected_total; i++) {
if (!seen[i]) {
- fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n",
- i + 1, expected_total, model_dir);
+ fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
+ expected_total, model_dir);
free(ms);
return NULL;
}
@@ -417,22 +505,24 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
}
void multi_safetensors_close(multi_safetensors_t *ms) {
- if (!ms) return;
+ if (!ms)
+ return;
for (int i = 0; i < ms->num_shards; i++) {
safetensors_close(ms->shards[i]);
}
free(ms);
}
-int multi_safetensors_data_nbytes(const multi_safetensors_t *ms,
- size_t *out_nbytes) {
- if (!ms || !out_nbytes) return -1;
+int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes) {
+ if (!ms || !out_nbytes)
+ return -1;
size_t total = 0;
for (int s = 0; s < ms->num_shards; s++) {
const safetensors_file_t *sf = ms->shards[s];
for (int i = 0; i < sf->num_tensors; i++) {
size_t n = sf->tensors[i].data_size;
- if (n > SIZE_MAX - total) return -1;
+ if (n > SIZE_MAX - total)
+ return -1;
total += n;
}
}
@@ -441,13 +531,14 @@ int multi_safetensors_data_nbytes(const multi_safetensors_t *ms,
}
const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf) {
+ const char *name,
+ safetensors_file_t **out_sf) {
for (int s = 0; s < ms->num_shards; s++) {
safetensors_file_t *sf = ms->shards[s];
for (int i = 0; i < sf->num_tensors; i++) {
if (strcmp(sf->tensors[i].name, name) == 0) {
- if (out_sf) *out_sf = sf;
+ if (out_sf)
+ *out_sf = sf;
return &sf->tensors[i];
}
}
diff --git a/qwen_safetensors.h b/qwen_safetensors.h
index ba61217..5e839a9 100644
--- a/qwen_safetensors.h
+++ b/qwen_safetensors.h
@@ -10,7 +10,7 @@
#include <stdint.h>
#define SAFETENSORS_MAX_TENSORS 1024
-#define SAFETENSORS_MAX_SHARDS 8
+#define SAFETENSORS_MAX_SHARDS 8
typedef enum {
DTYPE_F32 = 0,
@@ -54,13 +54,12 @@ void safetensors_close(safetensors_file_t *sf);
/* Open model from directory (auto-detects single file or multi-shard) */
multi_safetensors_t *multi_safetensors_open(const char *model_dir);
void multi_safetensors_close(multi_safetensors_t *ms);
-int multi_safetensors_data_nbytes(const multi_safetensors_t *ms,
- size_t *out_nbytes);
+int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes);
/* Find a tensor by name across all shards */
const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf);
+ const char *name,
+ safetensors_file_t **out_sf);
/* Get raw pointer to tensor data (within mmap'd region) */
const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c
index 59f8784..e813071 100644
--- a/qwen_tokenizer.c
+++ b/qwen_tokenizer.c
@@ -15,24 +15,31 @@ static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */
static int gpt2_mapping_initialized = 0;
static void init_gpt2_mapping(void) {
- if (gpt2_mapping_initialized) return;
+ if (gpt2_mapping_initialized)
+ return;
memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte));
int n = 0;
for (int b = 0; b < 256; b++) {
int is_normal = 0;
- if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */
- if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */
- if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */
+ if (b >= 33 && b <= 126)
+ is_normal = 1; /* '!'..'~' */
+ if (b >= 161 && b <= 172)
+ is_normal = 1; /* '¡'..'¬' */
+ if (b >= 174 && b <= 255)
+ is_normal = 1; /* '®'..'ÿ' */
- if (is_normal) gpt2_byte_to_unicode[b] = b;
- else gpt2_byte_to_unicode[b] = 256 + n++;
+ if (is_normal)
+ gpt2_byte_to_unicode[b] = b;
+ else
+ gpt2_byte_to_unicode[b] = 256 + n++;
}
for (int b = 0; b < 256; b++) {
int cp = gpt2_byte_to_unicode[b];
- if (cp < 512) gpt2_unicode_to_byte[cp] = b;
+ if (cp < 512)
+ gpt2_unicode_to_byte[cp] = b;
}
gpt2_mapping_initialized = 1;
@@ -64,7 +71,8 @@ static char *decode_gpt2_token(const char *token_str) {
size_t len = strlen(token_str);
unsigned char *bytes = (unsigned char *)malloc(len + 1);
- if (!bytes) return NULL;
+ if (!bytes)
+ return NULL;
int byte_count = 0;
const unsigned char *p = (const unsigned char *)token_str;
@@ -79,18 +87,24 @@ static char *decode_gpt2_token(const char *token_str) {
nbytes = 1;
} else if ((*p & 0xE0) == 0xC0) {
cp = (*p & 0x1F) << 6;
- if (p + 1 < end) cp |= (p[1] & 0x3F);
+ if (p + 1 < end)
+ cp |= (p[1] & 0x3F);
nbytes = 2;
} else if ((*p & 0xF0) == 0xE0) {
cp = (*p & 0x0F) << 12;
- if (p + 1 < end) cp |= (p[1] & 0x3F) << 6;
- if (p + 2 < end) cp |= (p[2] & 0x3F);
+ if (p + 1 < end)
+ cp |= (p[1] & 0x3F) << 6;
+ if (p + 2 < end)
+ cp |= (p[2] & 0x3F);
nbytes = 3;
} else if ((*p & 0xF8) == 0xF0) {
cp = (*p & 0x07) << 18;
- if (p + 1 < end) cp |= (p[1] & 0x3F) << 12;
- if (p + 2 < end) cp |= (p[2] & 0x3F) << 6;
- if (p + 3 < end) cp |= (p[3] & 0x3F);
+ if (p + 1 < end)
+ cp |= (p[1] & 0x3F) << 12;
+ if (p + 2 < end)
+ cp |= (p[2] & 0x3F) << 6;
+ if (p + 3 < end)
+ cp |= (p[3] & 0x3F);
nbytes = 4;
} else {
cp = *p;
@@ -113,31 +127,41 @@ static char *decode_gpt2_token(const char *token_str) {
* ======================================================================== */
static void skip_ws(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++;
+ while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
+ (*p)++;
}
static int parse_json_string(const char **p, char *out, size_t max_len) {
skip_ws(p);
- if (**p != '"') return -1;
+ if (**p != '"')
+ return -1;
(*p)++;
size_t i = 0;
while (**p && **p != '"' && i < max_len - 1) {
if (**p == '\\') {
(*p)++;
- if (**p == 'n') out[i++] = '\n';
- else if (**p == 't') out[i++] = '\t';
- else if (**p == '"') out[i++] = '"';
- else if (**p == '\\') out[i++] = '\\';
- else if (**p == '/') out[i++] = '/';
+ if (**p == 'n')
+ out[i++] = '\n';
+ else if (**p == 't')
+ out[i++] = '\t';
+ else if (**p == '"')
+ out[i++] = '"';
+ else if (**p == '\\')
+ out[i++] = '\\';
+ else if (**p == '/')
+ out[i++] = '/';
else if (**p == 'u') {
(*p)++;
unsigned int cp = 0;
for (int j = 0; j < 4 && **p; j++, (*p)++) {
cp <<= 4;
char c = **p;
- if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0');
- else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10);
- else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10);
+ if (c >= '0' && c <= '9')
+ cp |= (unsigned)(c - '0');
+ else if (c >= 'a' && c <= 'f')
+ cp |= (unsigned)(c - 'a' + 10);
+ else if (c >= 'A' && c <= 'F')
+ cp |= (unsigned)(c - 'A' + 10);
}
if (cp < 0x80 && i + 1 < max_len) {
out[i++] = (char)cp;
@@ -159,7 +183,8 @@ static int parse_json_string(const char **p, char *out, size_t max_len) {
(*p)++;
}
out[i] = '\0';
- if (**p != '"') return -1;
+ if (**p != '"')
+ return -1;
(*p)++;
return 0;
}
@@ -167,7 +192,10 @@ static int parse_json_string(const char **p, char *out, size_t max_len) {
static int64_t parse_json_int(const char **p) {
skip_ws(p);
int neg = 0;
- if (**p == '-') { neg = 1; (*p)++; }
+ if (**p == '-') {
+ neg = 1;
+ (*p)++;
+ }
int64_t val = 0;
while (**p >= '0' && **p <= '9') {
val = val * 10 + (**p - '0');
@@ -211,23 +239,28 @@ static uint64_t fnv1a_hash_pair(const char *a, const char *b) {
static int pair_key_eq(const char *key, const char *a, const char *b) {
while (*a) {
- if (*key++ != *a++) return 0;
+ if (*key++ != *a++)
+ return 0;
}
- if (*key++ != ' ') return 0;
+ if (*key++ != ' ')
+ return 0;
while (*b) {
- if (*key++ != *b++) return 0;
+ if (*key++ != *b++)
+ return 0;
}
return *key == '\0';
}
static int next_pow2(int x) {
int p = 1;
- while (p < x) p <<= 1;
+ while (p < x)
+ p <<= 1;
return p;
}
static int map_insert(str_int_entry_t *map, int cap, char *key, int value) {
- if (!map || cap <= 0 || !key) return -1;
+ if (!map || cap <= 0 || !key)
+ return -1;
int mask = cap - 1;
int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
for (int i = 0; i < cap; i++) {
@@ -246,9 +279,11 @@ static int map_insert(str_int_entry_t *map, int cap, char *key, int value) {
}
static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) {
- if (!keyp) return -1;
+ if (!keyp)
+ return -1;
char *key = *keyp;
- if (!map || cap <= 0 || !key) return -1;
+ if (!map || cap <= 0 || !key)
+ return -1;
int mask = cap - 1;
int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
for (int i = 0; i < cap; i++) {
@@ -268,26 +303,31 @@ static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int valu
}
static int map_get(const str_int_entry_t *map, int cap, const char *key) {
- if (!map || cap <= 0 || !key) return -1;
+ if (!map || cap <= 0 || !key)
+ return -1;
int mask = cap - 1;
int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
for (int i = 0; i < cap; i++) {
int idx = (pos + i) & mask;
- if (!map[idx].key) return -1;
- if (strcmp(map[idx].key, key) == 0) return map[idx].value;
+ if (!map[idx].key)
+ return -1;
+ if (strcmp(map[idx].key, key) == 0)
+ return map[idx].value;
}
return -1;
}
-static int map_get_pair(const str_int_entry_t *map, int cap,
- const char *a, const char *b) {
- if (!map || cap <= 0 || !a || !b) return -1;
+static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, const char *b) {
+ if (!map || cap <= 0 || !a || !b)
+ return -1;
int mask = cap - 1;
int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask);
for (int i = 0; i < cap; i++) {
int idx = (pos + i) & mask;
- if (!map[idx].key) return -1;
- if (pair_key_eq(map[idx].key, a, b)) return map[idx].value;
+ if (!map[idx].key)
+ return -1;
+ if (pair_key_eq(map[idx].key, a, b))
+ return map[idx].value;
}
return -1;
}
@@ -297,35 +337,43 @@ static int map_get_pair(const str_int_entry_t *map, int cap,
* ======================================================================== */
static int utf8_char_len(unsigned char c) {
- if ((c & 0x80) == 0) return 1;
- if ((c & 0xE0) == 0xC0) return 2;
- if ((c & 0xF0) == 0xE0) return 3;
- if ((c & 0xF8) == 0xF0) return 4;
+ if ((c & 0x80) == 0)
+ return 1;
+ if ((c & 0xE0) == 0xC0)
+ return 2;
+ if ((c & 0xF0) == 0xE0)
+ return 3;
+ if ((c & 0xF8) == 0xF0)
+ return 4;
return 1;
}
static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) {
*out_syms = NULL;
*out_n = 0;
- if (!s || !*s) return 0;
+ if (!s || !*s)
+ return 0;
int n = 0;
- for (const unsigned char *p = (const unsigned char *)s; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)s; *p;) {
int l = utf8_char_len(*p);
p += l;
n++;
}
- if (n <= 0) return 0;
+ if (n <= 0)
+ return 0;
char **syms = (char **)calloc((size_t)n, sizeof(char *));
- if (!syms) return -1;
+ if (!syms)
+ return -1;
int i = 0;
- for (const unsigned char *p = (const unsigned char *)s; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)s; *p;) {
int l = utf8_char_len(*p);
syms[i] = (char *)malloc((size_t)l + 1);
if (!syms[i]) {
- for (int j = 0; j < i; j++) free(syms[j]);
+ for (int j = 0; j < i; j++)
+ free(syms[j]);
free(syms);
return -1;
}
@@ -343,7 +391,8 @@ static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) {
static char *str_concat2(const char *a, const char *b) {
size_t la = strlen(a), lb = strlen(b);
char *r = (char *)malloc(la + lb + 1);
- if (!r) return NULL;
+ if (!r)
+ return NULL;
memcpy(r, a, la);
memcpy(r + la, b, lb);
r[la + lb] = '\0';
@@ -355,21 +404,23 @@ static char *str_concat2(const char *a, const char *b) {
* ======================================================================== */
typedef struct {
- uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */
+ uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */
int rank;
int merged_id;
} pair_merge_entry_t;
static uint64_t pair_hash(uint64_t k) {
/* splitmix64 finalizer */
- k ^= k >> 30; k *= 0xbf58476d1ce4e5b9ULL;
- k ^= k >> 27; k *= 0x94d049bb133111ebULL;
+ k ^= k >> 30;
+ k *= 0xbf58476d1ce4e5b9ULL;
+ k ^= k >> 27;
+ k *= 0x94d049bb133111ebULL;
k ^= k >> 31;
return k;
}
-static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key,
- int rank, int merged_id) {
+static void
+pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) {
int mask = cap - 1;
int idx = (int)(pair_hash(key) & (uint64_t)mask);
for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) {
@@ -379,22 +430,21 @@ static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key,
map[idx].merged_id = merged_id;
return;
}
- if (map[idx].key == key) return; /* first (lowest) rank wins */
+ if (map[idx].key == key)
+ return; /* first (lowest) rank wins */
}
}
/* Returns the rank and sets *merged_id, or INT_MAX when the pair never
* merges. */
-static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right,
- int *merged_id) {
- const pair_merge_entry_t *map =
- (const pair_merge_entry_t *)tok->int_merges;
+static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, int *merged_id) {
+ const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges;
uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right;
int mask = tok->int_merges_cap - 1;
int idx = (int)(pair_hash(key) & (uint64_t)mask);
- for (int probe = 0; probe < tok->int_merges_cap;
- probe++, idx = (idx + 1) & mask) {
- if (map[idx].key == UINT64_MAX) return INT_MAX;
+ for (int probe = 0; probe < tok->int_merges_cap; probe++, idx = (idx + 1) & mask) {
+ if (map[idx].key == UINT64_MAX)
+ return INT_MAX;
if (map[idx].key == key) {
*merged_id = map[idx].merged_id;
return map[idx].rank;
@@ -404,10 +454,10 @@ static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right,
}
static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) {
- if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX;
+ if (!tok->merge_map || tok->merge_map_cap <= 0)
+ return INT_MAX;
- int rank = map_get_pair((const str_int_entry_t *)tok->merge_map,
- tok->merge_map_cap, a, b);
+ int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, a, b);
return rank >= 0 ? rank : INT_MAX;
}
@@ -415,7 +465,8 @@ static int append_id(int **arr, int *n, int *cap, int id) {
if (*n >= *cap) {
int new_cap = (*cap == 0) ? 16 : (*cap * 2);
int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int));
- if (!tmp) return -1;
+ if (!tmp)
+ return -1;
*arr = tmp;
*cap = new_cap;
}
@@ -429,25 +480,28 @@ static char *text_to_bpe_unicode(const char *text) {
size_t len = strlen(text);
/* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */
char *out = (char *)malloc(len * 3 + 1);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
size_t w = 0;
for (size_t i = 0; i < len; i++) {
int cp = gpt2_byte_to_unicode[(unsigned char)text[i]];
char tmp[4];
int n = utf8_encode_cp(cp, tmp);
- for (int j = 0; j < n; j++) out[w++] = tmp[j];
+ for (int j = 0; j < n; j++)
+ out[w++] = tmp[j];
}
out[w] = '\0';
return out;
}
-static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped,
- int **out_ids, int *out_n);
+static int
+encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n);
-static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end,
- int *cp_out, int *len_out) {
- if (p >= end || *p == '\0') return 0;
+static int
+utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) {
+ if (p >= end || *p == '\0')
+ return 0;
unsigned char c = *p;
if ((c & 0x80) == 0) {
@@ -455,27 +509,19 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end,
*len_out = 1;
return 1;
}
- if ((c & 0xE0) == 0xC0 && p + 1 < end &&
- (p[1] & 0xC0) == 0x80) {
+ if ((c & 0xE0) == 0xC0 && p + 1 < end && (p[1] & 0xC0) == 0x80) {
*cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F);
*len_out = 2;
return 1;
}
- if ((c & 0xF0) == 0xE0 && p + 2 < end &&
- (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) {
- *cp_out = ((c & 0x0F) << 12) |
- ((p[1] & 0x3F) << 6) |
- (p[2] & 0x3F);
+ if ((c & 0xF0) == 0xE0 && p + 2 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) {
+ *cp_out = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F);
*len_out = 3;
return 1;
}
- if ((c & 0xF8) == 0xF0 && p + 3 < end &&
- (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 &&
+ if ((c & 0xF8) == 0xF0 && p + 3 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 &&
(p[3] & 0xC0) == 0x80) {
- *cp_out = ((c & 0x07) << 18) |
- ((p[1] & 0x3F) << 12) |
- ((p[2] & 0x3F) << 6) |
- (p[3] & 0x3F);
+ *cp_out = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F);
*len_out = 4;
return 1;
}
@@ -494,47 +540,56 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end,
#define CPC_LETTER 4
static const unsigned char ascii_cp_class[128] = {
- ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE,
- ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE,
- ['0' ... '9'] = CPC_NUMBER,
- ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER,
+ ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE,
+ ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE,
+ ['0' ... '9'] = CPC_NUMBER, ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER,
};
static int is_number_cp(int cp) {
- if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_NUMBER;
- if (cp >= 0x0660 && cp <= 0x0669) return 1;
- if (cp >= 0x06F0 && cp <= 0x06F9) return 1;
- if (cp >= 0x0966 && cp <= 0x096F) return 1;
- if (cp >= 0xFF10 && cp <= 0xFF19) return 1;
+ if ((unsigned)cp < 0x80)
+ return ascii_cp_class[cp] & CPC_NUMBER;
+ if (cp >= 0x0660 && cp <= 0x0669)
+ return 1;
+ if (cp >= 0x06F0 && cp <= 0x06F9)
+ return 1;
+ if (cp >= 0x0966 && cp <= 0x096F)
+ return 1;
+ if (cp >= 0xFF10 && cp <= 0xFF19)
+ return 1;
return 0;
}
static int is_space_cp(int cp) {
- if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_SPACE;
- return cp == 0x85 || cp == 0xA0 ||
- cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) ||
- cp == 0x2028 || cp == 0x2029 || cp == 0x202F ||
- cp == 0x205F || cp == 0x3000;
+ if ((unsigned)cp < 0x80)
+ return ascii_cp_class[cp] & CPC_SPACE;
+ return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) ||
+ cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000;
}
static int is_symbolish_cp(int cp) {
- if (cp < 0x80) return 0;
- if (cp >= 0x0300 && cp <= 0x036F) return 1;
- if (cp >= 0x00A1 && cp <= 0x00BF &&
- cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA)
+ if (cp < 0x80)
+ return 0;
+ if (cp >= 0x0300 && cp <= 0x036F)
+ return 1;
+ if (cp >= 0x00A1 && cp <= 0x00BF && cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA)
+ return 1;
+ if (cp == 0x00D7 || cp == 0x00F7)
+ return 1;
+ if (cp >= 0x064B && cp <= 0x065F)
+ return 1;
+ if ((cp >= 0x0900 && cp <= 0x0903) || (cp >= 0x093A && cp <= 0x094F) ||
+ (cp >= 0x0951 && cp <= 0x0957) || (cp >= 0x0962 && cp <= 0x0963))
return 1;
- if (cp == 0x00D7 || cp == 0x00F7) return 1;
- if (cp >= 0x064B && cp <= 0x065F) return 1;
- if ((cp >= 0x0900 && cp <= 0x0903) ||
- (cp >= 0x093A && cp <= 0x094F) ||
- (cp >= 0x0951 && cp <= 0x0957) ||
- (cp >= 0x0962 && cp <= 0x0963))
+ if (cp >= 0x2000 && cp <= 0x206F)
+ return 1;
+ if (cp >= 0x2190 && cp <= 0x27BF)
+ return 1;
+ if (cp >= 0x2E00 && cp <= 0x2E7F)
+ return 1;
+ if (cp >= 0x3000 && cp <= 0x303F)
+ return 1;
+ if (cp >= 0x1F000 && cp <= 0x1FAFF)
return 1;
- if (cp >= 0x2000 && cp <= 0x206F) return 1;
- if (cp >= 0x2190 && cp <= 0x27BF) return 1;
- if (cp >= 0x2E00 && cp <= 0x2E7F) return 1;
- if (cp >= 0x3000 && cp <= 0x303F) return 1;
- if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1;
return 0;
}
@@ -542,63 +597,162 @@ static int compose_latin_mark(int base, int mark) {
switch (mark) {
case 0x0300:
switch (base) {
- case 'A': return 0x00C0; case 'E': return 0x00C8;
- case 'I': return 0x00CC; case 'O': return 0x00D2;
- case 'U': return 0x00D9; case 'a': return 0x00E0;
- case 'e': return 0x00E8; case 'i': return 0x00EC;
- case 'o': return 0x00F2; case 'u': return 0x00F9;
+ case 'A':
+ return 0x00C0;
+ case 'E':
+ return 0x00C8;
+ case 'I':
+ return 0x00CC;
+ case 'O':
+ return 0x00D2;
+ case 'U':
+ return 0x00D9;
+ case 'a':
+ return 0x00E0;
+ case 'e':
+ return 0x00E8;
+ case 'i':
+ return 0x00EC;
+ case 'o':
+ return 0x00F2;
+ case 'u':
+ return 0x00F9;
}
break;
case 0x0301:
switch (base) {
- case 'A': return 0x00C1; case 'C': return 0x0106;
- case 'E': return 0x00C9; case 'I': return 0x00CD;
- case 'L': return 0x0139; case 'N': return 0x0143;
- case 'O': return 0x00D3; case 'R': return 0x0154;
- case 'S': return 0x015A; case 'U': return 0x00DA;
- case 'Y': return 0x00DD; case 'Z': return 0x0179;
- case 'a': return 0x00E1; case 'c': return 0x0107;
- case 'e': return 0x00E9; case 'i': return 0x00ED;
- case 'l': return 0x013A; case 'n': return 0x0144;
- case 'o': return 0x00F3; case 'r': return 0x0155;
- case 's': return 0x015B; case 'u': return 0x00FA;
- case 'y': return 0x00FD; case 'z': return 0x017A;
+ case 'A':
+ return 0x00C1;
+ case 'C':
+ return 0x0106;
+ case 'E':
+ return 0x00C9;
+ case 'I':
+ return 0x00CD;
+ case 'L':
+ return 0x0139;
+ case 'N':
+ return 0x0143;
+ case 'O':
+ return 0x00D3;
+ case 'R':
+ return 0x0154;
+ case 'S':
+ return 0x015A;
+ case 'U':
+ return 0x00DA;
+ case 'Y':
+ return 0x00DD;
+ case 'Z':
+ return 0x0179;
+ case 'a':
+ return 0x00E1;
+ case 'c':
+ return 0x0107;
+ case 'e':
+ return 0x00E9;
+ case 'i':
+ return 0x00ED;
+ case 'l':
+ return 0x013A;
+ case 'n':
+ return 0x0144;
+ case 'o':
+ return 0x00F3;
+ case 'r':
+ return 0x0155;
+ case 's':
+ return 0x015B;
+ case 'u':
+ return 0x00FA;
+ case 'y':
+ return 0x00FD;
+ case 'z':
+ return 0x017A;
}
break;
case 0x0302:
switch (base) {
- case 'A': return 0x00C2; case 'E': return 0x00CA;
- case 'I': return 0x00CE; case 'O': return 0x00D4;
- case 'U': return 0x00DB; case 'a': return 0x00E2;
- case 'e': return 0x00EA; case 'i': return 0x00EE;
- case 'o': return 0x00F4; case 'u': return 0x00FB;
+ case 'A':
+ return 0x00C2;
+ case 'E':
+ return 0x00CA;
+ case 'I':
+ return 0x00CE;
+ case 'O':
+ return 0x00D4;
+ case 'U':
+ return 0x00DB;
+ case 'a':
+ return 0x00E2;
+ case 'e':
+ return 0x00EA;
+ case 'i':
+ return 0x00EE;
+ case 'o':
+ return 0x00F4;
+ case 'u':
+ return 0x00FB;
}
break;
case 0x0303:
switch (base) {
- case 'A': return 0x00C3; case 'N': return 0x00D1;
- case 'O': return 0x00D5; case 'a': return 0x00E3;
- case 'n': return 0x00F1; case 'o': return 0x00F5;
+ case 'A':
+ return 0x00C3;
+ case 'N':
+ return 0x00D1;
+ case 'O':
+ return 0x00D5;
+ case 'a':
+ return 0x00E3;
+ case 'n':
+ return 0x00F1;
+ case 'o':
+ return 0x00F5;
}
break;
case 0x0308:
switch (base) {
- case 'A': return 0x00C4; case 'E': return 0x00CB;
- case 'I': return 0x00CF; case 'O': return 0x00D6;
- case 'U': return 0x00DC; case 'Y': return 0x0178;
- case 'a': return 0x00E4; case 'e': return 0x00EB;
- case 'i': return 0x00EF; case 'o': return 0x00F6;
- case 'u': return 0x00FC; case 'y': return 0x00FF;
+ case 'A':
+ return 0x00C4;
+ case 'E':
+ return 0x00CB;
+ case 'I':
+ return 0x00CF;
+ case 'O':
+ return 0x00D6;
+ case 'U':
+ return 0x00DC;
+ case 'Y':
+ return 0x0178;
+ case 'a':
+ return 0x00E4;
+ case 'e':
+ return 0x00EB;
+ case 'i':
+ return 0x00EF;
+ case 'o':
+ return 0x00F6;
+ case 'u':
+ return 0x00FC;
+ case 'y':
+ return 0x00FF;
}
break;
case 0x030A:
switch (base) {
- case 'A': return 0x00C5; case 'a': return 0x00E5;
+ case 'A':
+ return 0x00C5;
+ case 'a':
+ return 0x00E5;
}
break;
case 0x0327:
switch (base) {
- case 'C': return 0x00C7; case 'c': return 0x00E7;
+ case 'C':
+ return 0x00C7;
+ case 'c':
+ return 0x00E7;
}
break;
}
@@ -609,13 +763,15 @@ static char *normalize_nfc_latin(const char *text) {
const unsigned char *p = (const unsigned char *)text;
const unsigned char *end = p + strlen(text);
char *out = (char *)malloc((size_t)(end - p) * 2 + 1);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
size_t w = 0;
while (p < end) {
const unsigned char *cur = p;
int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+ if (!utf8_decode_cp_len(p, end, &cp, &len))
+ break;
p += len;
if (p < end) {
@@ -625,7 +781,8 @@ static char *normalize_nfc_latin(const char *text) {
if (composed) {
char tmp[4];
int n = utf8_encode_cp(composed, tmp);
- for (int i = 0; i < n; i++) out[w++] = tmp[i];
+ for (int i = 0; i < n; i++)
+ out[w++] = tmp[i];
p += mark_len;
continue;
}
@@ -640,26 +797,22 @@ static char *normalize_nfc_latin(const char *text) {
}
static int is_letter_cp(int cp) {
- if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_LETTER;
+ if ((unsigned)cp < 0x80)
+ return ascii_cp_class[cp] & CPC_LETTER;
if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp))
return 0;
return 1;
}
-static int is_word_cp(int cp) {
- return is_letter_cp(cp) || is_number_cp(cp);
-}
+static int is_word_cp(int cp) { return is_letter_cp(cp) || is_number_cp(cp); }
-static int is_newline_cp(int cp) {
- return cp == '\n' || cp == '\r';
-}
+static int is_newline_cp(int cp) { return cp == '\n' || cp == '\r'; }
-static int ascii_lower(int c) {
- return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
-}
+static int ascii_lower(int c) { return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; }
static int match_contraction(const unsigned char *p, const unsigned char *end) {
- if (p >= end || *p != '\'') return 0;
+ if (p >= end || *p != '\'')
+ return 0;
int remain = (int)(end - p);
if (remain >= 2) {
int c1 = ascii_lower(p[1]);
@@ -669,27 +822,31 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) {
if (remain >= 3) {
int c1 = ascii_lower(p[1]);
int c2 = ascii_lower(p[2]);
- if ((c1 == 'r' && c2 == 'e') ||
- (c1 == 'v' && c2 == 'e') ||
- (c1 == 'l' && c2 == 'l'))
+ if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l'))
return 3;
}
return 0;
}
static int append_encoded_piece(const qwen_tokenizer_t *tok,
- const char *piece, size_t piece_len,
- int **ids, int *n_ids, int *cap) {
- if (piece_len == 0) return 0;
+ const char *piece,
+ size_t piece_len,
+ int **ids,
+ int *n_ids,
+ int *cap) {
+ if (piece_len == 0)
+ return 0;
char *tmp = (char *)malloc(piece_len + 1);
- if (!tmp) return -1;
+ if (!tmp)
+ return -1;
memcpy(tmp, piece, piece_len);
tmp[piece_len] = '\0';
char *mapped = text_to_bpe_unicode(tmp);
free(tmp);
- if (!mapped) return -1;
+ if (!mapped)
+ return -1;
int *piece_ids = NULL;
int n_piece = 0;
@@ -710,8 +867,8 @@ static int append_encoded_piece(const qwen_tokenizer_t *tok,
return 0;
}
-static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
- int **out_ids, int *out_n) {
+static int
+encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) {
const unsigned char *start = (const unsigned char *)text;
const unsigned char *p = start;
const unsigned char *end = start + strlen(text);
@@ -721,7 +878,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
while (p < end) {
const unsigned char *piece = p;
int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+ if (!utf8_decode_cp_len(p, end, &cp, &len))
+ break;
int clen = match_contraction(p, end);
if (clen > 0) {
@@ -731,13 +889,13 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
int next_cp = 0, next_len = 0;
int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len);
- if (!is_newline_cp(cp) && !is_word_cp(cp) &&
- has_next && is_letter_cp(next_cp)) {
+ if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) {
p = next + next_len;
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c)) break;
+ if (!is_letter_cp(c))
+ break;
p += l;
}
} else if (is_letter_cp(cp)) {
@@ -745,7 +903,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c)) break;
+ if (!is_letter_cp(c))
+ break;
p += l;
}
} else if (is_number_cp(cp)) {
@@ -756,14 +915,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
while (q < end) {
int c = 0, l = 0;
utf8_decode_cp_len(q, end, &c, &l);
- if (!is_space_cp(c)) break;
- if (is_newline_cp(c)) saw_newline = 1;
+ if (!is_space_cp(c))
+ break;
+ if (is_newline_cp(c))
+ saw_newline = 1;
q += l;
if (saw_newline) {
while (q < end) {
int c2 = 0, l2 = 0;
utf8_decode_cp_len(q, end, &c2, &l2);
- if (!is_newline_cp(c2)) break;
+ if (!is_newline_cp(c2))
+ break;
q += l2;
}
break;
@@ -772,19 +934,20 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
if (saw_newline) {
p = q;
- } else if (cp == ' ' && has_next && !is_space_cp(next_cp) &&
- !is_word_cp(next_cp)) {
+ } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) {
p = next + next_len;
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c)) break;
+ if (is_space_cp(c) || is_word_cp(c))
+ break;
p += l;
}
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c)) break;
+ if (!is_newline_cp(c))
+ break;
p += l;
}
} else {
@@ -794,7 +957,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
while (q2 < end) {
int c = 0, l = 0;
utf8_decode_cp_len(q2, end, &c, &l);
- if (!is_space_cp(c)) break;
+ if (!is_space_cp(c))
+ break;
last = q2;
q2 += l;
count++;
@@ -809,20 +973,22 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c)) break;
+ if (is_space_cp(c) || is_word_cp(c))
+ break;
p += l;
}
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c)) break;
+ if (!is_newline_cp(c))
+ break;
p += l;
}
}
}
- if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece),
- &ids, &n_ids, &cap) != 0) {
+ if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids,
+ &cap) != 0) {
free(ids);
return -1;
}
@@ -834,14 +1000,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text,
}
/* Encode one mapped BPE unicode string to token IDs. */
-static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) {
+static int
+encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) {
*out_ids = NULL;
*out_n = 0;
- if (!mapped || !*mapped) return 0;
+ if (!mapped || !*mapped)
+ return 0;
char **syms = NULL;
int n_syms = 0;
- if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1;
+ if (split_utf8_symbols(mapped, &syms, &n_syms) != 0)
+ return -1;
if (n_syms <= 0) {
free(syms);
return 0;
@@ -857,18 +1026,21 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int
best_i = i;
}
}
- if (best_i < 0 || best_rank == INT_MAX) break;
+ if (best_i < 0 || best_rank == INT_MAX)
+ break;
char *merged = str_concat2(syms[best_i], syms[best_i + 1]);
if (!merged) {
- for (int i = 0; i < n_syms; i++) free(syms[i]);
+ for (int i = 0; i < n_syms; i++)
+ free(syms[i]);
free(syms);
return -1;
}
free(syms[best_i]);
free(syms[best_i + 1]);
syms[best_i] = merged;
- for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1];
+ for (int j = best_i + 1; j < n_syms - 1; j++)
+ syms[j] = syms[j + 1];
n_syms--;
}
@@ -878,20 +1050,23 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int
int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]);
if (id < 0) {
/* Should not happen with valid vocab + merges + byte-level mapping. */
- for (int k = 0; k < n_syms; k++) free(syms[k]);
+ for (int k = 0; k < n_syms; k++)
+ free(syms[k]);
free(syms);
free(ids);
return -1;
}
if (append_id(&ids, &n_ids, &cap, id) != 0) {
- for (int k = 0; k < n_syms; k++) free(syms[k]);
+ for (int k = 0; k < n_syms; k++)
+ free(syms[k]);
free(syms);
free(ids);
return -1;
}
}
- for (int i = 0; i < n_syms; i++) free(syms[i]);
+ for (int i = 0; i < n_syms; i++)
+ free(syms[i]);
free(syms);
*out_ids = ids;
@@ -927,72 +1102,87 @@ struct qwen_tokenizer_workspace {
};
static int reserve_bpe(qwen_tokenizer_workspace_t *ws, int need) {
- if (ws->bpe_cap >= need) return 0;
+ if (ws->bpe_cap >= need)
+ return 0;
int new_cap = ws->bpe_cap ? ws->bpe_cap : 64;
while (new_cap < need) {
- if (new_cap > INT_MAX / 2) return -1;
+ if (new_cap > INT_MAX / 2)
+ return -1;
new_cap *= 2;
}
int *p;
-#define R(field) \
+#define R(field) \
p = (int *)realloc(ws->field, (size_t)new_cap * sizeof(int)); \
- if (!p) return -1; \
+ if (!p) \
+ return -1; \
ws->field = p;
- R(bpe_ids) R(bpe_prv) R(bpe_nxt) R(bpe_rnk) R(bpe_mrg)
+ R(bpe_ids)
+ R(bpe_prv)
+ R(bpe_nxt)
+ R(bpe_rnk)
+ R(bpe_mrg)
#undef R
- ws->bpe_cap = new_cap;
+ ws->bpe_cap = new_cap;
return 0;
}
static int reserve_bytes(char **buf, size_t *cap, size_t need) {
- if (*cap >= need) return 0;
+ if (*cap >= need)
+ return 0;
size_t new_cap = *cap ? *cap : 256;
while (new_cap < need) {
- if (new_cap > SIZE_MAX / 2) return -1;
+ if (new_cap > SIZE_MAX / 2)
+ return -1;
new_cap *= 2;
}
char *p = (char *)realloc(*buf, new_cap);
- if (!p) return -1;
+ if (!p)
+ return -1;
*buf = p;
*cap = new_cap;
return 0;
}
static int reserve_ints(int **buf, int *cap, int need) {
- if (*cap >= need) return 0;
+ if (*cap >= need)
+ return 0;
int new_cap = *cap ? *cap : 64;
while (new_cap < need) {
- if (new_cap > INT_MAX / 2) return -1;
+ if (new_cap > INT_MAX / 2)
+ return -1;
new_cap *= 2;
}
int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int));
- if (!p) return -1;
+ if (!p)
+ return -1;
*buf = p;
*cap = new_cap;
return 0;
}
static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) {
- if (ws->syms_cap >= need) return 0;
+ if (ws->syms_cap >= need)
+ return 0;
int new_cap = ws->syms_cap ? ws->syms_cap : 64;
while (new_cap < need) {
- if (new_cap > INT_MAX / 2) return -1;
+ if (new_cap > INT_MAX / 2)
+ return -1;
new_cap *= 2;
}
- bpe_sym_ref_t *p =
- (bpe_sym_ref_t *)realloc(ws->syms,
- (size_t)new_cap * sizeof(*ws->syms));
- if (!p) return -1;
+ bpe_sym_ref_t *p = (bpe_sym_ref_t *)realloc(ws->syms, (size_t)new_cap * sizeof(*ws->syms));
+ if (!p)
+ return -1;
ws->syms = p;
ws->syms_cap = new_cap;
return 0;
}
-static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s,
- size_t len, size_t *off) {
- if (ws->arena_len > SIZE_MAX - len - 1) return -1;
+static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) {
+ if (ws->arena_len > SIZE_MAX - len - 1)
+ return -1;
size_t need = ws->arena_len + len + 1;
- if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1;
+ if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0)
+ return -1;
*off = ws->arena_len;
memcpy(ws->arena + ws->arena_len, s, len);
ws->arena_len += len;
@@ -1000,26 +1190,26 @@ static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s,
return 0;
}
-static const char *arena_str(const qwen_tokenizer_workspace_t *ws,
- bpe_sym_ref_t sym) {
+static const char *arena_str(const qwen_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) {
return ws->arena + sym.off;
}
-static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws,
- const char *text, const char **out) {
+static int
+normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, const char *text, const char **out) {
const unsigned char *p = (const unsigned char *)text;
const unsigned char *end = p + strlen(text);
size_t in_len = (size_t)(end - p);
- if (in_len > (SIZE_MAX - 1) / 2) return -1;
- if (reserve_bytes(&ws->normalized, &ws->normalized_cap,
- in_len * 2 + 1) != 0)
+ if (in_len > (SIZE_MAX - 1) / 2)
+ return -1;
+ if (reserve_bytes(&ws->normalized, &ws->normalized_cap, in_len * 2 + 1) != 0)
return -1;
size_t w = 0;
while (p < end) {
const unsigned char *cur = p;
int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+ if (!utf8_decode_cp_len(p, end, &cp, &len))
+ break;
p += len;
if (p < end) {
@@ -1046,10 +1236,12 @@ static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws,
}
static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws,
- const char *text, size_t len,
+ const char *text,
+ size_t len,
const char **out) {
init_gpt2_mapping();
- if (len > (SIZE_MAX - 1) / 3) return -1;
+ if (len > (SIZE_MAX - 1) / 3)
+ return -1;
if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0)
return -1;
@@ -1066,8 +1258,7 @@ static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws,
return 0;
}
-static int output_append_id(int *out_ids, int out_cap, int *n_ids,
- int *overflow, int id) {
+static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow, int id) {
if (*n_ids < out_cap && out_ids)
out_ids[*n_ids] = id;
else
@@ -1079,22 +1270,27 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids,
static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *mapped,
- int *out_ids, int out_cap,
- int *n_ids, int *overflow) {
- if (!mapped || !*mapped) return 0;
+ int *out_ids,
+ int out_cap,
+ int *n_ids,
+ int *overflow) {
+ if (!mapped || !*mapped)
+ return 0;
int n_syms = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
int l = utf8_char_len(*p);
p += l;
n_syms++;
}
- if (n_syms <= 0) return 0;
- if (reserve_syms(ws, n_syms) != 0) return -1;
+ if (n_syms <= 0)
+ return 0;
+ if (reserve_syms(ws, n_syms) != 0)
+ return -1;
ws->arena_len = 0;
int i = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
int l = utf8_char_len(*p);
size_t off = 0;
if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0)
@@ -1107,21 +1303,21 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok,
int best_rank = INT_MAX;
int best_i = -1;
for (int j = 0; j < n_syms - 1; j++) {
- int r = merge_rank(tok, arena_str(ws, ws->syms[j]),
- arena_str(ws, ws->syms[j + 1]));
+ int r = merge_rank(tok, arena_str(ws, ws->syms[j]), arena_str(ws, ws->syms[j + 1]));
if (r < best_rank) {
best_rank = r;
best_i = j;
}
}
- if (best_i < 0 || best_rank == INT_MAX) break;
+ if (best_i < 0 || best_rank == INT_MAX)
+ break;
const char *a = arena_str(ws, ws->syms[best_i]);
const char *b = arena_str(ws, ws->syms[best_i + 1]);
size_t la = strlen(a), lb = strlen(b);
- if (la > SIZE_MAX - lb) return -1;
- if (reserve_bytes(&ws->arena, &ws->arena_cap,
- ws->arena_len + la + lb + 1) != 0)
+ if (la > SIZE_MAX - lb)
+ return -1;
+ if (reserve_bytes(&ws->arena, &ws->arena_cap, ws->arena_len + la + lb + 1) != 0)
return -1;
a = arena_str(ws, ws->syms[best_i]);
@@ -1139,9 +1335,10 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok,
}
for (int j = 0; j < n_syms; j++) {
- int id = map_get((const str_int_entry_t *)tok->vocab_map,
- tok->vocab_map_cap, arena_str(ws, ws->syms[j]));
- if (id < 0) return -1;
+ int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap,
+ arena_str(ws, ws->syms[j]));
+ if (id < 0)
+ return -1;
output_append_id(out_ids, out_cap, n_ids, overflow, id);
}
return 0;
@@ -1155,35 +1352,41 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok,
static int encode_bpe_word_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *mapped,
- int *out_ids, int out_cap,
- int *n_ids, int *overflow) {
- if (!mapped || !*mapped) return 0;
+ int *out_ids,
+ int out_cap,
+ int *n_ids,
+ int *overflow) {
+ if (!mapped || !*mapped)
+ return 0;
if (!tok->int_merges || !tok->cp_to_id)
- return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap,
- n_ids, overflow);
+ return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
int n = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
p += utf8_char_len(*p);
n++;
}
- if (n <= 0) return 0;
- if (reserve_bpe(ws, n) != 0) return -1;
+ if (n <= 0)
+ return 0;
+ if (reserve_bpe(ws, n) != 0)
+ return -1;
int *ids = ws->bpe_ids, *prv = ws->bpe_prv, *nxt = ws->bpe_nxt;
int *rnk = ws->bpe_rnk, *mrg = ws->bpe_mrg;
int i = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
int len = utf8_char_len(*p);
int cp;
- if (len == 1) cp = *p;
- else if (len == 2) cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F);
- else cp = 512; /* byte-level alphabet never exceeds two bytes */
+ if (len == 1)
+ cp = *p;
+ else if (len == 2)
+ cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F);
+ else
+ cp = 512; /* byte-level alphabet never exceeds two bytes */
int id = cp < 512 ? tok->cp_to_id[cp] : -1;
if (id < 0)
- return encode_bpe_word_into_slow(tok, ws, mapped, out_ids,
- out_cap, n_ids, overflow);
+ return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
ids[i] = id;
prv[i] = i - 1;
nxt[i] = i + 1 < n ? i + 1 : -1;
@@ -1193,7 +1396,8 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok,
for (int j = 0; j < n - 1; j++)
rnk[j] = pair_merge_lookup(tok, ids[j], ids[j + 1], &mrg[j]);
- if (n > 0) rnk[n - 1] = INT_MAX;
+ if (n > 0)
+ rnk[n - 1] = INT_MAX;
for (;;) {
int best = -1, best_rank = INT_MAX;
@@ -1203,19 +1407,19 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok,
best = j;
}
}
- if (best < 0) break;
+ if (best < 0)
+ break;
ids[best] = mrg[best];
int dead = nxt[best];
nxt[best] = nxt[dead];
- if (nxt[dead] != -1) prv[nxt[dead]] = best;
+ if (nxt[dead] != -1)
+ prv[nxt[dead]] = best;
if (prv[best] != -1)
- rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best],
- &mrg[prv[best]]);
- rnk[best] = nxt[best] != -1
- ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best])
- : INT_MAX;
+ rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]);
+ rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best])
+ : INT_MAX;
}
for (int j = 0; j != -1; j = nxt[j])
@@ -1225,22 +1429,26 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok,
static int append_encoded_piece_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
- const char *piece, size_t piece_len,
- int *out_ids, int out_cap,
- int *n_ids, int *overflow) {
- if (piece_len == 0) return 0;
+ const char *piece,
+ size_t piece_len,
+ int *out_ids,
+ int out_cap,
+ int *n_ids,
+ int *overflow) {
+ if (piece_len == 0)
+ return 0;
const char *mapped = NULL;
if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0)
return -1;
- return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids,
- overflow);
+ return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
}
static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
- int *out_ids, int out_cap,
+ int *out_ids,
+ int out_cap,
int *out_n) {
const unsigned char *start = (const unsigned char *)text;
const unsigned char *p = start;
@@ -1251,7 +1459,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
while (p < end) {
const unsigned char *piece = p;
int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+ if (!utf8_decode_cp_len(p, end, &cp, &len))
+ break;
int clen = match_contraction(p, end);
if (clen > 0) {
@@ -1261,13 +1470,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
int next_cp = 0, next_len = 0;
int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len);
- if (!is_newline_cp(cp) && !is_word_cp(cp) &&
- has_next && is_letter_cp(next_cp)) {
+ if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) {
p = next + next_len;
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c)) break;
+ if (!is_letter_cp(c))
+ break;
p += l;
}
} else if (is_letter_cp(cp)) {
@@ -1275,7 +1484,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c)) break;
+ if (!is_letter_cp(c))
+ break;
p += l;
}
} else if (is_number_cp(cp)) {
@@ -1286,14 +1496,17 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
while (q < end) {
int c = 0, l = 0;
utf8_decode_cp_len(q, end, &c, &l);
- if (!is_space_cp(c)) break;
- if (is_newline_cp(c)) saw_newline = 1;
+ if (!is_space_cp(c))
+ break;
+ if (is_newline_cp(c))
+ saw_newline = 1;
q += l;
if (saw_newline) {
while (q < end) {
int c2 = 0, l2 = 0;
utf8_decode_cp_len(q, end, &c2, &l2);
- if (!is_newline_cp(c2)) break;
+ if (!is_newline_cp(c2))
+ break;
q += l2;
}
break;
@@ -1302,19 +1515,20 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
if (saw_newline) {
p = q;
- } else if (cp == ' ' && has_next && !is_space_cp(next_cp) &&
- !is_word_cp(next_cp)) {
+ } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) {
p = next + next_len;
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c)) break;
+ if (is_space_cp(c) || is_word_cp(c))
+ break;
p += l;
}
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c)) break;
+ if (!is_newline_cp(c))
+ break;
p += l;
}
} else {
@@ -1324,7 +1538,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
while (q2 < end) {
int c = 0, l = 0;
utf8_decode_cp_len(q2, end, &c, &l);
- if (!is_space_cp(c)) break;
+ if (!is_space_cp(c))
+ break;
last = q2;
q2 += l;
count++;
@@ -1339,21 +1554,22 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c)) break;
+ if (is_space_cp(c) || is_word_cp(c))
+ break;
p += l;
}
while (p < end) {
int c = 0, l = 0;
utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c)) break;
+ if (!is_newline_cp(c))
+ break;
p += l;
}
}
}
- if (append_encoded_piece_into(tok, ws, (const char *)piece,
- (size_t)(p - piece), out_ids, out_cap,
- &n_ids, &overflow) != 0)
+ if (append_encoded_piece_into(tok, ws, (const char *)piece, (size_t)(p - piece), out_ids,
+ out_cap, &n_ids, &overflow) != 0)
return -1;
}
@@ -1364,11 +1580,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) {
const char *slash = strrchr(vocab_path, '/');
if (!slash) {
- if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1;
+ if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap)
+ return -1;
return 0;
}
size_t dir_len = (size_t)(slash - vocab_path);
- if (dir_len + 12 >= out_cap) return -1;
+ if (dir_len + 12 >= out_cap)
+ return -1;
memcpy(out_path, vocab_path, dir_len);
out_path[dir_len] = '\0';
snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt");
@@ -1377,22 +1595,28 @@ static int derive_merges_path(const char *vocab_path, char *out_path, size_t out
static void trim_newline(char *line) {
size_t n = strlen(line);
- while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0';
+ while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r'))
+ line[--n] = '\0';
}
static int parse_merge_pair(char *line, char **a, char **b) {
*a = NULL;
*b = NULL;
trim_newline(line);
- if (line[0] == '\0') return -1;
- if (line[0] == '#') return -1;
+ if (line[0] == '\0')
+ return -1;
+ if (line[0] == '#')
+ return -1;
char *sp = strchr(line, ' ');
- if (!sp) return -1;
+ if (!sp)
+ return -1;
*sp = '\0';
char *p2 = sp + 1;
- while (*p2 == ' ') p2++;
- if (*line == '\0' || *p2 == '\0') return -1;
+ while (*p2 == ' ')
+ p2++;
+ if (*line == '\0' || *p2 == '\0')
+ return -1;
*a = line;
*b = p2;
return 0;
@@ -1400,13 +1624,15 @@ static int parse_merge_pair(char *line, char **a, char **b) {
static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
FILE *f = fopen(merges_path, "rb");
- if (!f) return -1;
+ if (!f)
+ return -1;
int n_pairs = 0;
char line[8192];
while (fgets(line, sizeof(line), f)) {
char *a = NULL, *b = NULL;
- if (parse_merge_pair(line, &a, &b) == 0) n_pairs++;
+ if (parse_merge_pair(line, &a, &b) == 0)
+ n_pairs++;
}
if (n_pairs <= 0) {
fclose(f);
@@ -1426,8 +1652,8 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
return -1;
}
tok->int_merges_cap = next_pow2(n_pairs * 2);
- pair_merge_entry_t *pm = (pair_merge_entry_t *)
- calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t));
+ pair_merge_entry_t *pm =
+ (pair_merge_entry_t *)calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t));
if (pm) {
for (int i = 0; i < tok->int_merges_cap; i++) {
pm[i].key = UINT64_MAX;
@@ -1440,11 +1666,13 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
int rank = 0;
while (fgets(line, sizeof(line), f)) {
char *a = NULL, *b = NULL;
- if (parse_merge_pair(line, &a, &b) != 0) continue;
+ if (parse_merge_pair(line, &a, &b) != 0)
+ continue;
size_t la = strlen(a), lb = strlen(b);
char *key = (char *)malloc(la + 1 + lb + 1);
- if (!key) continue;
+ if (!key)
+ continue;
memcpy(key, a, la);
key[la] = ' ';
memcpy(key + la + 1, b, lb);
@@ -1470,15 +1698,11 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
tok->int_merges = NULL;
tok->int_merges_cap = 0;
} else {
- pair_map_insert((pair_merge_entry_t *)tok->int_merges,
- tok->int_merges_cap,
- ((uint64_t)(uint32_t)left << 32) |
- (uint32_t)right,
- rank, merged);
+ pair_map_insert((pair_merge_entry_t *)tok->int_merges, tok->int_merges_cap,
+ ((uint64_t)(uint32_t)left << 32) | (uint32_t)right, rank, merged);
}
}
- (void)map_insert_owned((str_int_entry_t *)tok->merge_map,
- tok->merge_map_cap, &key, rank);
+ (void)map_insert_owned((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, &key, rank);
free(key);
rank++;
}
@@ -1489,17 +1713,22 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) {
init_gpt2_mapping();
tok->cp_to_id = (int *)malloc(512 * sizeof(int));
if (tok->cp_to_id) {
- for (int i = 0; i < 512; i++) tok->cp_to_id[i] = -1;
+ for (int i = 0; i < 512; i++)
+ tok->cp_to_id[i] = -1;
for (int byte = 0; byte < 256 && tok->cp_to_id; byte++) {
int cp = gpt2_byte_to_unicode[byte];
char buf[4];
int n;
- if (cp < 0x80) { buf[0] = (char)cp; n = 1; }
- else { buf[0] = (char)(0xC0 | (cp >> 6));
- buf[1] = (char)(0x80 | (cp & 0x3F)); n = 2; }
+ if (cp < 0x80) {
+ buf[0] = (char)cp;
+ n = 1;
+ } else {
+ buf[0] = (char)(0xC0 | (cp >> 6));
+ buf[1] = (char)(0x80 | (cp & 0x3F));
+ n = 2;
+ }
buf[n] = '\0';
- int id = map_get((const str_int_entry_t *)tok->vocab_map,
- tok->vocab_map_cap, buf);
+ int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, buf);
if (id < 0) {
free(tok->cp_to_id);
tok->cp_to_id = NULL;
@@ -1554,16 +1783,23 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
const char *saved = p;
while (*p && *p != '}') {
skip_ws(&p);
- if (*p == ',') { p++; continue; }
- if (*p == '}') break;
+ if (*p == ',') {
+ p++;
+ continue;
+ }
+ if (*p == '}')
+ break;
char key[4096] = {0};
- if (parse_json_string(&p, key, sizeof(key)) != 0) break;
+ if (parse_json_string(&p, key, sizeof(key)) != 0)
+ break;
skip_ws(&p);
- if (*p != ':') break;
+ if (*p != ':')
+ break;
p++;
int64_t id = parse_json_int(&p);
- if (id > max_id) max_id = (int)id;
+ if (id > max_id)
+ max_id = (int)id;
}
int vocab_size = max_id + 1;
@@ -1584,13 +1820,19 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
p = saved;
while (*p && *p != '}') {
skip_ws(&p);
- if (*p == ',') { p++; continue; }
- if (*p == '}') break;
+ if (*p == ',') {
+ p++;
+ continue;
+ }
+ if (*p == '}')
+ break;
char key[4096] = {0};
- if (parse_json_string(&p, key, sizeof(key)) != 0) break;
+ if (parse_json_string(&p, key, sizeof(key)) != 0)
+ break;
skip_ws(&p);
- if (*p != ':') break;
+ if (*p != ':')
+ break;
p++;
int64_t id = parse_json_int(&p);
@@ -1604,7 +1846,9 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
free(json);
int n_vocab_entries = 0;
- for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++;
+ for (int i = 0; i < vocab_size; i++)
+ if (tok->id_to_bpe[i])
+ n_vocab_entries++;
tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1);
tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t));
if (!tok->vocab_map) {
@@ -1620,7 +1864,8 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
char merges_path[1024];
if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) {
if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) {
- fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n",
+ fprintf(stderr,
+ "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n",
merges_path);
}
}
@@ -1629,22 +1874,26 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) {
}
const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) {
- if (!tok || token_id < 0 || token_id >= tok->vocab_size) return "";
+ if (!tok || token_id < 0 || token_id >= tok->vocab_size)
+ return "";
return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : "";
}
int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) {
- if (!tok || !token || !tok->vocab_map) return -1;
- return map_get((const str_int_entry_t *)tok->vocab_map,
- tok->vocab_map_cap, token);
+ if (!tok || !token || !tok->vocab_map)
+ return -1;
+ return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
}
int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) {
- if (out_n_tokens) *out_n_tokens = 0;
- if (!tok || !text || text[0] == '\0') return NULL;
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !text || text[0] == '\0')
+ return NULL;
char *normalized = normalize_nfc_latin(text);
- if (!normalized) return NULL;
+ if (!normalized)
+ return NULL;
int *ids = NULL;
int n_ids = 0;
@@ -1655,17 +1904,18 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o
}
free(normalized);
- if (out_n_tokens) *out_n_tokens = n_ids;
+ if (out_n_tokens)
+ *out_n_tokens = n_ids;
return ids;
}
qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) {
- return (qwen_tokenizer_workspace_t *)calloc(1,
- sizeof(qwen_tokenizer_workspace_t));
+ return (qwen_tokenizer_workspace_t *)calloc(1, sizeof(qwen_tokenizer_workspace_t));
}
void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) {
- if (!ws) return;
+ if (!ws)
+ return;
free(ws->normalized);
free(ws->mapped);
free(ws->arena);
@@ -1682,20 +1932,24 @@ void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) {
int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
- int *out_ids, int out_cap,
+ int *out_ids,
+ int out_cap,
int *out_n_tokens) {
- if (out_n_tokens) *out_n_tokens = 0;
- if (!tok || !ws || !text || out_cap < 0) return -1;
- if (text[0] == '\0') return 0;
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text || out_cap < 0)
+ return -1;
+ if (text[0] == '\0')
+ return 0;
const char *normalized = NULL;
if (normalize_nfc_latin_into(ws, text, &normalized) != 0)
return -1;
int n_ids = 0;
- int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap,
- &n_ids);
- if (out_n_tokens) *out_n_tokens = n_ids;
+ int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, &n_ids);
+ if (out_n_tokens)
+ *out_n_tokens = n_ids;
return rc;
}
@@ -1703,50 +1957,58 @@ int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
int *out_n_tokens) {
- if (out_n_tokens) *out_n_tokens = 0;
- if (!tok || !ws || !text || text[0] == '\0') return NULL;
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text || text[0] == '\0')
+ return NULL;
size_t len = strlen(text);
size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8;
- if (cap_sz > (size_t)INT_MAX) return NULL;
+ if (cap_sz > (size_t)INT_MAX)
+ return NULL;
int cap = (int)cap_sz;
if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0)
return NULL;
int n_ids = 0;
- int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap,
- &n_ids);
+ int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
if (rc == -2) {
if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0)
return NULL;
- rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap,
- &n_ids);
+ rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
}
- if (rc != 0 || n_ids <= 0) return NULL;
+ if (rc != 0 || n_ids <= 0)
+ return NULL;
int *ids = (int *)malloc((size_t)n_ids * sizeof(int));
- if (!ids) return NULL;
+ if (!ids)
+ return NULL;
memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int));
- if (out_n_tokens) *out_n_tokens = n_ids;
+ if (out_n_tokens)
+ *out_n_tokens = n_ids;
return ids;
}
void qwen_tokenizer_free(qwen_tokenizer_t *tok) {
- if (!tok) return;
+ if (!tok)
+ return;
if (tok->id_to_text) {
- for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]);
+ for (int i = 0; i < tok->vocab_size; i++)
+ free(tok->id_to_text[i]);
free(tok->id_to_text);
}
if (tok->id_to_bpe) {
- for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]);
+ for (int i = 0; i < tok->vocab_size; i++)
+ free(tok->id_to_bpe[i]);
free(tok->id_to_bpe);
}
if (tok->merge_map) {
str_int_entry_t *m = (str_int_entry_t *)tok->merge_map;
- for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key);
+ for (int i = 0; i < tok->merge_map_cap; i++)
+ free(m[i].key);
free(tok->merge_map);
}
free(tok->vocab_map);
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index d7670b7..f7a3941 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -21,8 +21,8 @@
#endif
typedef struct {
- char **id_to_text; /* [vocab_size] decoded text strings */
- char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
+ char **id_to_text; /* [vocab_size] decoded text strings */
+ char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
int vocab_size;
/* Internal hash maps (opaque to callers) */
@@ -55,7 +55,8 @@ EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *t
/* Encode UTF-8 text into token IDs using BPE.
* Returns malloc'd array of token IDs and sets *out_n_tokens.
* Returns NULL on error (and sets *out_n_tokens to 0). */
-EMBED_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+EMBED_API int *
+qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
/* Reusable scratch for allocation-light repeated encoding. */
EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
@@ -66,17 +67,18 @@ EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
* In all cases *out_n_tokens is set to the number of tokens needed/emitted
* when out_n_tokens is non-NULL. */
EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids, int out_cap,
- int *out_n_tokens);
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and
* performs only the final returned int[] allocation. */
EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
/* Free tokenizer */
EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok);
diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c
index cc9b2fb..9a5d842 100644
--- a/tests/test_bf16_model.c
+++ b/tests/test_bf16_model.c
@@ -15,12 +15,14 @@ int main(void) {
char root[1024], f32_dir[1088], bf16_dir[1088];
snprintf(root, sizeof(root), "%s/embed-bf16-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
- if (!mkdtemp(root)) { perror("mkdtemp"); return 2; }
+ if (!mkdtemp(root)) {
+ perror("mkdtemp");
+ return 2;
+ }
snprintf(f32_dir, sizeof(f32_dir), "%s/f32", root);
snprintf(bf16_dir, sizeof(bf16_dir), "%s/bf16", root);
if (mkdir(f32_dir, 0755) != 0 || mkdir(bf16_dir, 0755) != 0 ||
- tm_write_model(f32_dir, "F32") != 0 ||
- tm_write_model(bf16_dir, "BF16") != 0) {
+ tm_write_model(f32_dir, "F32") != 0 || tm_write_model(bf16_dir, "BF16") != 0) {
fprintf(stderr, "fixture creation failed\n");
return 2;
}
@@ -58,7 +60,8 @@ int main(void) {
return 1;
}
float d = fabsf(a[i] - b[i]);
- if (d > max_diff) max_diff = d;
+ if (d > max_diff)
+ max_diff = d;
norm += b[i] * b[i];
}
norm = sqrtf(norm);
@@ -73,7 +76,8 @@ int main(void) {
* (linear_nobias_weight); parity must hold there too. */
enum { LONG_N = 24 };
int long_ids[LONG_N];
- for (int i = 0; i < LONG_N; i++) long_ids[i] = (i % 14) + 1;
+ for (int i = 0; i < LONG_N; i++)
+ long_ids[i] = (i % 14) + 1;
if (embed_model_encode_into(mf32, wf32, long_ids, LONG_N, a) != 0 ||
embed_model_encode_into(mbf16, wbf16, long_ids, LONG_N, b) != 0) {
fprintf(stderr, "long-sequence embedding failed\n");
@@ -86,15 +90,15 @@ int main(void) {
return 1;
}
float d = fabsf(a[i] - b[i]);
- if (d > long_diff) long_diff = d;
+ if (d > long_diff)
+ long_diff = d;
}
if (long_diff > 2e-5f) {
fprintf(stderr, "bad widen-path parity: max_diff=%.9g\n", long_diff);
return 1;
}
- printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n",
- dim, max_diff, norm);
+ printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n", dim, max_diff, norm);
embed_workspace_free(wf32);
embed_workspace_free(wbf16);
diff --git a/tests/test_cli.c b/tests/test_cli.c
index 58cbad3..8d56f09 100644
--- a/tests/test_cli.c
+++ b/tests/test_cli.c
@@ -13,68 +13,69 @@
#include <sys/wait.h>
#include <unistd.h>
-static char g_dir[1024]; /* fixture model dir */
-static char g_out[1100]; /* captured stdout */
-static char g_err[1100]; /* captured stderr */
-static char g_in[1100]; /* stdin source */
+static char g_dir[1024]; /* fixture model dir */
+static char g_out[1100]; /* captured stdout */
+static char g_err[1100]; /* captured stderr */
+static char g_in[1100]; /* stdin source */
static const char *g_cli;
static int g_failures;
/* Run the CLI with sh-formatted args; returns its exit status (-1 on a
* harness error). stdin_text == NULL feeds /dev/null. */
-static int run_cli(const char *args, const char *stdin_text)
-{
+static int run_cli(const char *args, const char *stdin_text) {
if (stdin_text) {
FILE *f = fopen(g_in, "w");
- if (!f) return -1;
+ if (!f)
+ return -1;
fputs(stdin_text, f);
fclose(f);
}
char cmd[4096];
- int n = snprintf(cmd, sizeof(cmd), "'%s' %s > '%s' 2> '%s' < '%s'",
- g_cli, args, g_out, g_err,
+ int n = snprintf(cmd, sizeof(cmd), "'%s' %s > '%s' 2> '%s' < '%s'", g_cli, args, g_out, g_err,
stdin_text ? g_in : "/dev/null");
- if (n < 0 || (size_t)n >= sizeof(cmd)) return -1;
+ if (n < 0 || (size_t)n >= sizeof(cmd))
+ return -1;
int st = system(cmd);
- if (st == -1 || !WIFEXITED(st)) return -1;
+ if (st == -1 || !WIFEXITED(st))
+ return -1;
return WEXITSTATUS(st);
}
-static void expect(int cond, const char *what)
-{
+static void expect(int cond, const char *what) {
if (!cond) {
fprintf(stderr, "FAIL: %s\n", what);
g_failures++;
}
}
-static int file_contains(const char *path, const char *needle)
-{
+static int file_contains(const char *path, const char *needle) {
static char buf[1 << 16];
FILE *f = fopen(path, "rb");
- if (!f) return 0;
+ if (!f)
+ return 0;
size_t n = fread(buf, 1, sizeof(buf) - 1, f);
fclose(f);
buf[n] = '\0';
return strstr(buf, needle) != NULL;
}
-static int count_lines(const char *path)
-{
+static int count_lines(const char *path) {
FILE *f = fopen(path, "rb");
- if (!f) return -1;
+ if (!f)
+ return -1;
int lines = 0, c;
while ((c = fgetc(f)) != EOF)
- if (c == '\n') lines++;
+ if (c == '\n')
+ lines++;
fclose(f);
return lines;
}
/* First stdout line must hold exactly `dim` finite floats. */
-static int stdout_is_embedding(int dim)
-{
+static int stdout_is_embedding(int dim) {
FILE *f = fopen(g_out, "r");
- if (!f) return 0;
+ if (!f)
+ return 0;
int count = 0;
float v;
while (fscanf(f, "%f", &v) == 1) {
@@ -88,8 +89,7 @@ static int stdout_is_embedding(int dim)
return count == dim;
}
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
g_cli = argc > 1 ? argv[1] : "./embed";
tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE};
@@ -109,13 +109,11 @@ int main(int argc, char **argv)
/* Help and missing-argument handling. */
expect(run_cli("-h", NULL) == 0, "-h exits 0");
expect(run_cli("", NULL) == 1, "no arguments exits 1");
- expect(run_cli("-d /nonexistent-embed-dir hi", NULL) == 1,
- "missing model dir exits 1");
+ expect(run_cli("-d /nonexistent-embed-dir hi", NULL) == 1, "missing model dir exits 1");
/* -V reports the program's base name, not the path it was invoked by. */
expect(run_cli("-V", NULL) == 0, "-V exits 0");
- expect(!file_contains(g_out, "tests/"),
- "-V strips the directory from the program name");
+ expect(!file_contains(g_out, "tests/"), "-V strips the directory from the program name");
/* Mode 1: one text prints one embedding line. */
snprintf(args, sizeof(args), "-d '%s' 'hello world'", g_dir);
@@ -123,11 +121,9 @@ int main(int argc, char **argv)
expect(stdout_is_embedding(dims.hidden), "single text prints dim floats");
/* Thread count. */
- snprintf(args, sizeof(args),
- "-d '%s' -t 2 -v 'hello world'", g_dir);
+ snprintf(args, sizeof(args), "-d '%s' -t 2 -v 'hello world'", g_dir);
expect(run_cli(args, NULL) == 0, "-t 2 -v exits 0");
- expect(file_contains(g_err, "Using 2 CPU thread(s)"),
- "-v reports the thread count");
+ expect(file_contains(g_err, "Using 2 CPU thread(s)"), "-v reports the thread count");
/* Mode 2+: several texts print a bare numeric matrix, one row per line,
* with no titles or labels. Two texts give a 2x2 with a 1.0 diagonal. */
@@ -152,8 +148,7 @@ int main(int argc, char **argv)
/* -e appends the raw embeddings (label-free) after the matrix: three
* matrix rows, a blank separator, then three embedding rows. */
- snprintf(args, sizeof(args),
- "-d '%s' -e -b 1 'hello' 'world' 'held'", g_dir);
+ snprintf(args, sizeof(args), "-d '%s' -e -b 1 'hello' 'world' 'held'", g_dir);
expect(run_cli(args, NULL) == 0, "-e -b 1 with three texts exits 0");
expect(count_lines(g_out) == 7, "-e prints matrix then raw embeddings");
@@ -167,16 +162,17 @@ int main(int argc, char **argv)
snprintf(args, sizeof(args), "-d '%s' --stream", g_dir);
expect(run_cli(args, "hello\nworld\n") == 0, "--stream exits 0");
expect(count_lines(g_out) == 2, "--stream writes one line per input");
- expect(file_contains(g_out, "{\"embedding\":["),
- "--stream writes embedding JSON");
+ expect(file_contains(g_out, "{\"embedding\":["), "--stream writes embedding JSON");
snprintf(args, sizeof(args), "-d '%s' --stream -b 2 -v", g_dir);
- expect(run_cli(args, "hello\nworld\nheld\n") == 0,
- "--stream -b 2 exits 0");
+ expect(run_cli(args, "hello\nworld\nheld\n") == 0, "--stream -b 2 exits 0");
expect(count_lines(g_out) == 3, "--stream -b 2 covers the tail batch");
/* Flag validation: missing model dir must exit 1. */
- static const struct { const char *args; const char *what; } bad[] = {
+ static const struct {
+ const char *args;
+ const char *what;
+ } bad[] = {
{"-d '' t", "empty model dir"},
};
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++)
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 4a2a57d..fb3d2df 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -18,8 +18,7 @@ static void expect_close(const char *name, float got, float want, float tol) {
float diff = fabsf(got - want);
float scale = fmaxf(1.0f, fmaxf(fabsf(got), fabsf(want)));
if (diff > tol * scale) {
- fprintf(stderr, "%s: got %.9g want %.9g diff %.9g\n",
- name, got, want, diff);
+ fprintf(stderr, "%s: got %.9g want %.9g diff %.9g\n", name, got, want, diff);
failures++;
}
}
@@ -39,10 +38,7 @@ static float bf16_to_f32(uint16_t x) {
static void test_rms_norm(void) {
const int seq = 2, hidden = 4;
- const float x[8] = {
- 1.0f, 2.0f, -3.0f, 4.0f,
- 0.5f, -1.0f, 2.0f, -0.5f
- };
+ const float x[8] = {1.0f, 2.0f, -3.0f, 4.0f, 0.5f, -1.0f, 2.0f, -0.5f};
const float w[4] = {1.0f, 0.5f, -1.0f, 2.0f};
float out[8];
@@ -66,10 +62,7 @@ static void test_rope_neox(void) {
const int seq = 2, heads = 1, head_dim = 4;
const int positions[2] = {0, 3};
float cosv[8], sinv[8];
- float x[8] = {
- 1.0f, 2.0f, 3.0f, 4.0f,
- -1.0f, 0.5f, 2.0f, -0.25f
- };
+ float x[8] = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, 0.5f, 2.0f, -0.25f};
float orig[8];
memcpy(orig, x, sizeof(x));
@@ -92,10 +85,17 @@ static void test_rope_neox(void) {
}
}
-static void reference_packed_gqa(float *out, const float *Q, const float *K,
- const float *V, const int *offsets, int batch,
- int n_heads, int n_kv_heads, int head_dim,
- float scale, int causal) {
+static void reference_packed_gqa(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int causal) {
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
int heads_per_kv = n_heads / n_kv_heads;
@@ -112,22 +112,28 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K,
for (int j = start; j < key_end; j++) {
const float *k = K + j * kv_hidden + kv_h * head_dim;
float score = 0.0f;
- for (int d = 0; d < head_dim; d++) score += q[d] * k[d];
+ for (int d = 0; d < head_dim; d++)
+ score += q[d] * k[d];
score *= scale;
- if (score > max_score) max_score = score;
+ if (score > max_score)
+ max_score = score;
}
float denom = 0.0f;
- for (int d = 0; d < head_dim; d++) o[d] = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o[d] = 0.0f;
for (int j = start; j < key_end; j++) {
const float *k = K + j * kv_hidden + kv_h * head_dim;
const float *v = V + j * kv_hidden + kv_h * head_dim;
float score = 0.0f;
- for (int d = 0; d < head_dim; d++) score += q[d] * k[d];
+ for (int d = 0; d < head_dim; d++)
+ score += q[d] * k[d];
float wt = expf(score * scale - max_score);
denom += wt;
- for (int d = 0; d < head_dim; d++) o[d] += wt * v[d];
+ for (int d = 0; d < head_dim; d++)
+ o[d] += wt * v[d];
}
- for (int d = 0; d < head_dim; d++) o[d] /= denom;
+ for (int d = 0; d < head_dim; d++)
+ o[d] /= denom;
}
}
}
@@ -151,11 +157,9 @@ static void test_packed_gqa_attention(void) {
V[i] = sinf((float)i * 0.07f) - cosf((float)i * 0.13f);
}
- qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
- reference_packed_gqa(want, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 0);
for (int i = 0; i < total * q_hidden; i++)
@@ -172,7 +176,8 @@ static void test_packed_gqa_attention_long(void) {
float *V = malloc((size_t)total * kv_hidden * sizeof(*V));
float *got = malloc((size_t)total * q_hidden * sizeof(*got));
float *want = malloc((size_t)total * q_hidden * sizeof(*want));
- if (!Q || !K || !V || !got || !want) exit(2);
+ if (!Q || !K || !V || !got || !want)
+ exit(2);
for (int i = 0; i < total * q_hidden; i++)
Q[i] = sinf((float)i * 0.017f) - 0.25f;
@@ -182,11 +187,9 @@ static void test_packed_gqa_attention_long(void) {
}
qwen_set_threads(4);
- qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
- reference_packed_gqa(want, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 0);
qwen_set_threads(1);
@@ -209,7 +212,8 @@ static void test_packed_causal_gqa_attention(void) {
float *V = malloc((size_t)total * kv_hidden * sizeof(*V));
float *got = malloc((size_t)total * q_hidden * sizeof(*got));
float *want = malloc((size_t)total * q_hidden * sizeof(*want));
- if (!Q || !K || !V || !got || !want) exit(2);
+ if (!Q || !K || !V || !got || !want)
+ exit(2);
for (int i = 0; i < total * q_hidden; i++)
Q[i] = sinf((float)i * 0.019f) - 0.2f;
@@ -219,11 +223,9 @@ static void test_packed_causal_gqa_attention(void) {
}
qwen_set_threads(4);
- qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
- reference_packed_gqa(want, Q, K, V, offsets, 2,
- n_heads, n_kv_heads, head_dim,
+ reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 1);
qwen_set_threads(1);
@@ -238,18 +240,13 @@ static void test_packed_causal_gqa_attention(void) {
static void test_bf16_linear(void) {
enum { seq = 2, in_dim = 3, out_dim = 2 };
- const float x[6] = {
- 1.0f, -2.0f, 0.5f,
- -1.0f, 0.25f, 2.0f
- };
- const float wf32[6] = {
- 0.5f, -1.0f, 2.0f,
- -0.25f, 0.75f, 1.5f
- };
+ const float x[6] = {1.0f, -2.0f, 0.5f, -1.0f, 0.25f, 2.0f};
+ const float wf32[6] = {0.5f, -1.0f, 2.0f, -0.25f, 0.75f, 1.5f};
uint16_t wbf16[6];
float got[seq * out_dim];
- for (int i = 0; i < 6; i++) wbf16[i] = f32_to_bf16(wf32[i]);
+ for (int i = 0; i < 6; i++)
+ wbf16[i] = f32_to_bf16(wf32[i]);
qwen_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim);
for (int s = 0; s < seq; s++) {
@@ -287,8 +284,7 @@ static void check_bf16_qkv(int n_threads) {
}
qwen_set_threads(n_threads);
- qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv,
- seq, in_dim, q_dim, kv_dim);
+ qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim);
for (int s = 0; s < seq; s++) {
for (int o = 0; o < q_dim; o++) {
@@ -349,7 +345,7 @@ static void test_bf16_matvec(void) {
* rms_norm_worker; row-wise math is identical, so results must match the
* single-thread run exactly. */
static void test_rms_norm_threaded(void) {
- enum { seq = 96, hidden = 3072 }; /* 294912 elements */
+ enum { seq = 96, hidden = 3072 }; /* 294912 elements */
size_t elems = (size_t)seq * hidden;
float *x = (float *)malloc(elems * sizeof(float));
float *w = (float *)malloc((size_t)hidden * sizeof(float));
@@ -358,7 +354,10 @@ static void test_rms_norm_threaded(void) {
if (!x || !w || !one || !many) {
fprintf(stderr, "rms_norm_threaded: allocation failure\n");
failures++;
- free(many); free(one); free(w); free(x);
+ free(many);
+ free(one);
+ free(w);
+ free(x);
return;
}
unsigned s = 7u;
@@ -377,13 +376,15 @@ static void test_rms_norm_threaded(void) {
for (size_t i = 0; i < elems; i++) {
if (one[i] != many[i]) {
- fprintf(stderr, "rms_norm_threaded: mismatch at %zu: %.9g %.9g\n",
- i, one[i], many[i]);
+ fprintf(stderr, "rms_norm_threaded: mismatch at %zu: %.9g %.9g\n", i, one[i], many[i]);
failures++;
break;
}
}
- free(many); free(one); free(w); free(x);
+ free(many);
+ free(one);
+ free(w);
+ free(x);
}
/* bf16->f32 widening is an exact bit shift; check the SIMD body and the
@@ -407,8 +408,7 @@ static void test_bf16_widen_buf(void) {
memcpy(&got_bits, &got[i], sizeof(got_bits));
memcpy(&want_bits, &want, sizeof(want_bits));
if (got_bits != want_bits) {
- fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n",
- n, i, got[i], want);
+ fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n", n, i, got[i], want);
failures++;
}
}
@@ -436,8 +436,7 @@ static void check_bf16_pair(int n_threads) {
}
qwen_set_threads(n_threads);
- qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb,
- seq, in_dim, a_dim, b_dim);
+ qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim);
for (int s = 0; s < seq; s++) {
for (int o = 0; o < a_dim; o++) {
@@ -477,8 +476,10 @@ static void test_generic_vs_impl(void) {
s = s * 1664525u + 1013904223u;
b[i] = (float)(int32_t)s * 1e-9f;
}
- for (int i = 0; i < IN; i++) x[i] = a[i];
- for (int o = 0; o < OUT; o++) bias[o] = b[o];
+ for (int i = 0; i < IN; i++)
+ x[i] = a[i];
+ for (int o = 0; o < OUT; o++)
+ bias[o] = b[o];
for (size_t i = 0; i < sizeof(W) / sizeof(W[0]); i++) {
s = s * 1664525u + 1013904223u;
W[i] = f32_to_bf16((float)(int32_t)s * 1e-9f);
@@ -497,8 +498,7 @@ static void test_generic_vs_impl(void) {
const int sizes[] = {1, 7, 64, NMAX};
for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) {
int n = sizes[t];
- expect_close("dot generic-vs-impl",
- qwen_dot_f32_generic(a, b, n),
+ expect_close("dot generic-vs-impl", qwen_dot_f32_generic(a, b, n),
qwen_dot_f32_impl(a, b, n), 1e-5f);
memcpy(d0, a, (size_t)n * sizeof(float));
@@ -538,7 +538,8 @@ int main(void) {
test_bf16_widen_buf();
test_bf16_pair();
test_generic_vs_impl();
- if (failures != 0) return 1;
+ if (failures != 0)
+ return 1;
puts("ok: kernel golden tests passed");
return 0;
}
diff --git a/tests/test_late.c b/tests/test_late.c
index 94bdfe2..e491596 100644
--- a/tests/test_late.c
+++ b/tests/test_late.c
@@ -15,50 +15,50 @@
enum { TOKEN_DIM = 6 };
-static float max_abs_diff(const float *a, const float *b, size_t n)
-{
+static float max_abs_diff(const float *a, const float *b, size_t n) {
float m = 0.0f;
for (size_t i = 0; i < n; i++) {
float d = fabsf(a[i] - b[i]);
- if (d > m) m = d;
+ if (d > m)
+ m = d;
}
return m;
}
-static float row_norm(const float *v, int n)
-{
+static float row_norm(const float *v, int n) {
float s = 0.0f;
- for (int i = 0; i < n; i++) s += v[i] * v[i];
+ for (int i = 0; i < n; i++)
+ s += v[i] * v[i];
return sqrtf(s);
}
/* Parity between the grouped batch scorer and per-document MaxSim. */
-static int check_maxsim_batch_parity(const float *query, int query_tokens,
- const float *docs, const int *offsets,
- int n_docs, int dim, const char *what)
-{
+static int check_maxsim_batch_parity(const float *query,
+ int query_tokens,
+ const float *docs,
+ const int *offsets,
+ int n_docs,
+ int dim,
+ const char *what) {
float scores[8];
- if (n_docs > (int)(sizeof(scores) / sizeof(scores[0]))) return 1;
- if (embed_late_maxsim_batch(query, query_tokens, docs, offsets, n_docs,
- dim, scores) != 0) {
+ if (n_docs > (int)(sizeof(scores) / sizeof(scores[0])))
+ return 1;
+ if (embed_late_maxsim_batch(query, query_tokens, docs, offsets, n_docs, dim, scores) != 0) {
fprintf(stderr, "maxsim_batch (%s) failed\n", what);
return 1;
}
for (int i = 0; i < n_docs; i++) {
- float one = embed_late_maxsim(query, query_tokens,
- docs + (size_t)offsets[i] * dim,
- offsets[i + 1] - offsets[i], dim);
+ float one = embed_late_maxsim(query, query_tokens, docs + (size_t)offsets[i] * dim,
+ offsets[i + 1] - offsets[i], dim);
if (fabsf(scores[i] - one) > 0.0001f) {
- fprintf(stderr, "maxsim_batch (%s) doc %d: %g vs %g\n",
- what, i, scores[i], one);
+ fprintf(stderr, "maxsim_batch (%s) doc %d: %g vs %g\n", what, i, scores[i], one);
return 1;
}
}
return 0;
}
-int main(void)
-{
+int main(void) {
int rc = 1;
tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE};
int hidden = dims.hidden;
@@ -77,8 +77,7 @@ int main(void)
char dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-late-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
- if (!mkdtemp(dir) || tf_write_vocab(dir) != 0 ||
- tm_write_model_dims(dir, "F32", &dims) != 0) {
+ if (!mkdtemp(dir) || tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0) {
fprintf(stderr, "fixture creation failed\n");
return 2;
}
@@ -96,8 +95,7 @@ int main(void)
fprintf(stderr, "late load succeeded without 1_Dense\n");
goto fail;
}
- if (embed_late_model_config(NULL) != NULL ||
- embed_late_model_token_dim(NULL) != 0 ||
+ if (embed_late_model_config(NULL) != NULL || embed_late_model_token_dim(NULL) != 0 ||
embed_late_workspace_new(NULL) != NULL) {
fprintf(stderr, "NULL-argument handling failed\n");
goto fail;
@@ -106,8 +104,7 @@ int main(void)
embed_late_model_free(NULL);
/* A head whose tensor is not named linear.weight must be rejected. */
- if (tm_write_late_projection_named(dir, "F32", TOKEN_DIM, hidden,
- "dense.weight") != 0) {
+ if (tm_write_late_projection_named(dir, "F32", TOKEN_DIM, hidden, "dense.weight") != 0) {
fprintf(stderr, "failed to write malformed projection\n");
goto fail;
}
@@ -133,8 +130,7 @@ int main(void)
}
const embed_config_t *cfg = embed_late_model_config(late);
- if (!cfg || cfg->hidden_size != hidden ||
- embed_late_model_token_dim(late) != TOKEN_DIM) {
+ if (!cfg || cfg->hidden_size != hidden || embed_late_model_token_dim(late) != TOKEN_DIM) {
fprintf(stderr, "late config/token_dim mismatch\n");
goto fail;
}
@@ -253,33 +249,30 @@ int main(void)
}
offsets[i + 1] = offsets[i] + lens[i];
}
- doc_vecs = (float *)calloc((size_t)offsets[3] * TOKEN_DIM,
- sizeof(float));
+ doc_vecs = (float *)calloc((size_t)offsets[3] * TOKEN_DIM, sizeof(float));
if (!doc_vecs) {
fprintf(stderr, "allocation failure\n");
goto fail;
}
for (int i = 0; i < 3; i++) {
- if (embed_late_model_encode_tokens(
- late, ws, doc_ids[i], lens[i], 1,
- doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) {
+ if (embed_late_model_encode_tokens(late, ws, doc_ids[i], lens[i], 1,
+ doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) {
fprintf(stderr, "doc encode failed\n");
goto fail;
}
}
- if (check_maxsim_batch_parity(vecs, n_tokens, doc_vecs, offsets, 3,
- TOKEN_DIM, "encoded docs") != 0)
+ if (check_maxsim_batch_parity(vecs, n_tokens, doc_vecs, offsets, 3, TOKEN_DIM,
+ "encoded docs") != 0)
goto fail;
float scores[3];
int bad_first[4] = {1, 2, 3, 4};
int empty_doc[3] = {0, 2, 2};
- if (embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3,
- TOKEN_DIM, scores) == 0 ||
- embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2,
- TOKEN_DIM, scores) == 0 ||
- embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, offsets, 3,
- TOKEN_DIM, NULL) == 0) {
+ if (embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3, TOKEN_DIM, scores) ==
+ 0 ||
+ embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2, TOKEN_DIM, scores) ==
+ 0 ||
+ embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, offsets, 3, TOKEN_DIM, NULL) == 0) {
fprintf(stderr, "maxsim_batch accepted invalid arguments\n");
goto fail;
}
@@ -305,20 +298,20 @@ int main(void)
s = s * 1664525u + 1013904223u;
big_d[i] = (float)((s >> 8) & 0xFFFF) / 32768.0f - 1.0f;
}
- if (check_maxsim_batch_parity(big_q, BIG_Q, big_d, offsets, 2,
- BIG_DIM, "oversized doc") != 0)
+ if (check_maxsim_batch_parity(big_q, BIG_Q, big_d, offsets, 2, BIG_DIM, "oversized doc") !=
+ 0)
goto fail;
}
- printf("ok: late API token_dim=%d tokens=%d hidden=%d\n",
- TOKEN_DIM, n_tokens, hidden);
+ printf("ok: late API token_dim=%d tokens=%d hidden=%d\n", TOKEN_DIM, n_tokens, hidden);
rc = 0;
fail:
free(big_d);
free(big_q);
free(doc_vecs);
- for (int i = 0; i < 3; i++) free(doc_ids[i]);
+ for (int i = 0; i < 3; i++)
+ free(doc_ids[i]);
free(expected);
free(vecs);
free(raw);
diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c
index 4d63a35..d66e0f7 100644
--- a/tests/test_qwen3.c
+++ b/tests/test_qwen3.c
@@ -9,24 +9,22 @@
#include <string.h>
#include <unistd.h>
-static float max_abs_diff(const float *a, const float *b, int n)
-{
+static float max_abs_diff(const float *a, const float *b, int n) {
float out = 0.0f;
for (int i = 0; i < n; i++) {
float d = fabsf(a[i] - b[i]);
- if (d > out) out = d;
+ if (d > out)
+ out = d;
}
return out;
}
-int main(void)
-{
+int main(void) {
char dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-qwen3-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
tm_dims_t dims = {4, 2, 1, 2, 8, 16};
- if (!mkdtemp(dir) ||
- tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) {
+ if (!mkdtemp(dir) || tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) {
fprintf(stderr, "fixture creation failed\n");
return 2;
}
@@ -39,8 +37,8 @@ int main(void)
return 1;
}
if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL ||
- cfg->pooling_mode != EMBED_POOL_LAST_TOKEN ||
- !cfg->normalize_embeddings || !cfg->append_terminal_token) {
+ cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || !cfg->normalize_embeddings ||
+ !cfg->append_terminal_token) {
fprintf(stderr, "Qwen3 config semantics were not detected\n");
return 1;
}
@@ -58,15 +56,15 @@ int main(void)
float expected[4];
memcpy(expected, states + 3 * 4, sizeof(expected));
- if (embed_l2_normalize(expected, 4) != 0 ||
- max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f ||
max_abs_diff(pooled, expected, 4) > 1e-5f) {
fprintf(stderr, "last-token pooling mismatch\n");
return 1;
}
float norm_sq = 0.0f;
- for (int d = 0; d < 4; d++) norm_sq += embedding[d] * embedding[d];
+ for (int d = 0; d < 4; d++)
+ norm_sq += embedding[d] * embedding[d];
if (fabsf(norm_sq - 1.0f) > 1e-5f) {
fprintf(stderr, "Qwen3 embedding is not normalized: %.9g\n", norm_sq);
return 1;
@@ -78,8 +76,7 @@ int main(void)
float single[4];
if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
- max_abs_diff(batch, embedding, 4) > 1e-5f ||
- max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) {
fprintf(stderr, "Qwen3 packed batch parity failed\n");
return 1;
}
diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c
index d527951..36ec8c5 100644
--- a/tests/test_safetensors.c
+++ b/tests/test_safetensors.c
@@ -15,29 +15,31 @@ static int failures = 0;
static char tmproot[1024];
/* Write u64-LE header length + JSON header + payload (safetensors layout). */
-static void write_st(const char *path, const char *header,
- const void *payload, size_t payload_len)
-{
+static void
+write_st(const char *path, const char *header, const void *payload, size_t payload_len) {
FILE *f = fopen(path, "wb");
- if (!f) { perror(path); exit(2); }
+ if (!f) {
+ perror(path);
+ exit(2);
+ }
uint64_t n = strlen(header);
unsigned char len8[8];
- for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(n >> (8 * i));
+ for (int i = 0; i < 8; i++)
+ len8[i] = (unsigned char)(n >> (8 * i));
fwrite(len8, 1, 8, f);
fwrite(header, 1, n, f);
- if (payload_len) fwrite(payload, 1, payload_len, f);
+ if (payload_len)
+ fwrite(payload, 1, payload_len, f);
fclose(f);
}
-static const char *path_in(const char *rel)
-{
+static const char *path_in(const char *rel) {
static char buf[2048];
snprintf(buf, sizeof(buf), "%s/%s", tmproot, rel);
return buf;
}
-static void check(const char *mode, const char *path, int want_ok)
-{
+static void check(const char *mode, const char *path, int want_ok) {
int got_ok;
if (strcmp(mode, "file") == 0) {
safetensors_file_t *sf = safetensors_open(path);
@@ -49,8 +51,8 @@ static void check(const char *mode, const char *path, int want_ok)
multi_safetensors_close(ms);
}
if (got_ok != want_ok) {
- fprintf(stderr, "%s %s: got %s want %s\n", mode, path,
- got_ok ? "ok" : "fail", want_ok ? "ok" : "fail");
+ fprintf(stderr, "%s %s: got %s want %s\n", mode, path, got_ok ? "ok" : "fail",
+ want_ok ? "ok" : "fail");
failures++;
}
}
@@ -62,11 +64,13 @@ static const char ONE_TENSOR_B[] =
static const char ONE_TENSOR_X[] =
"{\"x\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}";
-int main(void)
-{
+int main(void) {
snprintf(tmproot, sizeof(tmproot), "%s/embed-st-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
- if (!mkdtemp(tmproot)) { perror("mkdtemp"); return 2; }
+ if (!mkdtemp(tmproot)) {
+ perror("mkdtemp");
+ return 2;
+ }
static const char zeros[4 * 1025] = {0};
@@ -77,14 +81,15 @@ int main(void)
{
size_t cap = 1025 * 64 + 16;
char *hdr = malloc(cap);
- if (!hdr) return 2;
+ if (!hdr)
+ return 2;
size_t off = 0;
off += (size_t)snprintf(hdr + off, cap - off, "{");
for (int i = 0; i < 1025; i++) {
off += (size_t)snprintf(hdr + off, cap - off,
- "%s\"t%d\":{\"dtype\":\"F32\",\"shape\":[1],"
- "\"data_offsets\":[%d,%d]}",
- i ? "," : "", i, i * 4, i * 4 + 4);
+ "%s\"t%d\":{\"dtype\":\"F32\",\"shape\":[1],"
+ "\"data_offsets\":[%d,%d]}",
+ i ? "," : "", i, i * 4, i * 4 + 4);
}
snprintf(hdr + off, cap - off, "}");
write_st(path_in("too_many.safetensors"), hdr, zeros, sizeof(zeros));
@@ -94,27 +99,26 @@ int main(void)
write_st(path_in("too_many_dims.safetensors"),
"{\"x\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],"
- "\"data_offsets\":[0,4]}}", zeros, 4);
+ "\"data_offsets\":[0,4]}}",
+ zeros, 4);
check("file", path_in("too_many_dims.safetensors"), 0);
write_st(path_in("bad_offsets.safetensors"),
"{\"x\":{\"dtype\":\"F32\",\"shape\":[1],"
- "\"data_offsets\":[4,0]}}", zeros, 4);
+ "\"data_offsets\":[4,0]}}",
+ zeros, 4);
check("file", path_in("bad_offsets.safetensors"), 0);
mkdir(path_in("valid_multi"), 0755);
- write_st(path_in("valid_multi/model-00001-of-00002.safetensors"),
- ONE_TENSOR_A, zeros, 4);
- write_st(path_in("valid_multi/model-00002-of-00002.safetensors"),
- ONE_TENSOR_B, zeros, 4);
+ write_st(path_in("valid_multi/model-00001-of-00002.safetensors"), ONE_TENSOR_A, zeros, 4);
+ write_st(path_in("valid_multi/model-00002-of-00002.safetensors"), ONE_TENSOR_B, zeros, 4);
check("multi", path_in("valid_multi"), 1);
/* Payload accounting across shards: two 4-byte F32 tensors. */
{
multi_safetensors_t *ms = multi_safetensors_open(path_in("valid_multi"));
size_t nbytes = 0;
- if (!ms || multi_safetensors_data_nbytes(ms, &nbytes) != 0 ||
- nbytes != 8) {
+ if (!ms || multi_safetensors_data_nbytes(ms, &nbytes) != 0 || nbytes != 8) {
fprintf(stderr, "multi data_nbytes: got %zu want 8\n", nbytes);
failures++;
}
@@ -122,18 +126,15 @@ int main(void)
}
mkdir(path_in("missing"), 0755);
- write_st(path_in("missing/model-00001-of-00002.safetensors"),
- ONE_TENSOR_X, zeros, 4);
+ write_st(path_in("missing/model-00001-of-00002.safetensors"), ONE_TENSOR_X, zeros, 4);
check("multi", path_in("missing"), 0);
mkdir(path_in("too_many_shards"), 0755);
- write_st(path_in("too_many_shards/model-00001-of-00009.safetensors"),
- ONE_TENSOR_X, zeros, 4);
+ write_st(path_in("too_many_shards/model-00001-of-00009.safetensors"), ONE_TENSOR_X, zeros, 4);
check("multi", path_in("too_many_shards"), 0);
mkdir(path_in("bad_name"), 0755);
- write_st(path_in("bad_name/model-foo.safetensors"),
- ONE_TENSOR_X, zeros, 4);
+ write_st(path_in("bad_name/model-foo.safetensors"), ONE_TENSOR_X, zeros, 4);
check("multi", path_in("bad_name"), 0);
if (failures) {
diff --git a/tests/test_server.c b/tests/test_server.c
index a55462c..d799107 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -16,30 +16,35 @@
static int test_failures = 0;
-#define TEST_ASSERT(cond) do { \
- if (!(cond)) { \
- fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
- test_failures++; \
- } \
-} while (0)
+#define TEST_ASSERT(cond) \
+ do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ test_failures++; \
+ } \
+ } while (0)
-static void test_base64_rfc4648(void)
-{
+static void test_base64_rfc4648(void) {
/* RFC 4648 test vectors. */
- static const struct { const char *in, *out; } cases[] = {
- {"", ""}, {"f", "Zg=="}, {"fo", "Zm8="}, {"foo", "Zm9v"},
- {"foob", "Zm9vYg=="}, {"fooba", "Zm9vYmE="}, {"foobar", "Zm9vYmFy"},
+ static const struct {
+ const char *in, *out;
+ } cases[] = {
+ {"", ""},
+ {"f", "Zg=="},
+ {"fo", "Zm8="},
+ {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="},
+ {"fooba", "Zm9vYmE="},
+ {"foobar", "Zm9vYmFy"},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
- char *got = base64_encode((const unsigned char *)cases[i].in,
- strlen(cases[i].in));
+ char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in));
TEST_ASSERT(strcmp(got, cases[i].out) == 0);
free(got);
}
}
-static void test_quantize_int8_tanh(void)
-{
+static void test_quantize_int8_tanh(void) {
TEST_ASSERT(quantize_int8_tanh(0.0f) == 0);
/* tanh saturates to +-1, scaled by 127. */
TEST_ASSERT(quantize_int8_tanh(100.0f) == 127);
@@ -50,8 +55,7 @@ static void test_quantize_int8_tanh(void)
TEST_ASSERT(quantize_int8_tanh(0.5f) == 59);
}
-static void test_encode_embedding_int8(void)
-{
+static void test_encode_embedding_int8(void) {
float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
signed char expect[4] = {0, 59, -59, 127};
char *got = encode_embedding(emb, 4, "base64_int8");
@@ -61,8 +65,7 @@ static void test_encode_embedding_int8(void)
free(want);
}
-static void test_encode_embedding_binary(void)
-{
+static void test_encode_embedding_binary(void) {
/* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */
float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f};
unsigned char expect = 0x85; /* bits 0, 2, 7 set */
@@ -82,8 +85,7 @@ static void test_encode_embedding_binary(void)
free(want);
}
-static void test_encode_embedding_base64_float32(void)
-{
+static void test_encode_embedding_base64_float32(void) {
/* OpenAI/DashScope "base64" is base64 of the raw little-endian float32
* vector - lossless, unlike the int8 formats. */
float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
@@ -98,8 +100,7 @@ static void test_encode_embedding_base64_float32(void)
free(want);
}
-static void test_append_embedding_value_float(void)
-{
+static void test_append_embedding_value_float(void) {
float emb[1] = {0.5f};
/* OpenAI/DashScope (Qwen3): the true float32 value. */
@@ -115,16 +116,14 @@ static void test_append_embedding_value_float(void)
sbuf_free(&b2);
}
-static void test_encoding_from_root_family(void)
-{
+static void test_encoding_from_root_family(void) {
cJSON *detail = cJSON_CreateArray();
/* Default encoding differs by family when the field is absent. */
cJSON *empty = cJSON_CreateObject();
- TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_OPENAI),
- "float") == 0);
- TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_PERPLEXITY),
- "base64_int8") == 0);
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_OPENAI), "float") == 0);
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_PERPLEXITY), "base64_int8") ==
+ 0);
cJSON_Delete(empty);
TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
@@ -144,8 +143,7 @@ static void test_encoding_from_root_family(void)
cJSON_Delete(detail);
}
-static void test_collect_job_batch_deadline(void)
-{
+static void test_collect_job_batch_deadline(void) {
http_server s;
memset(&s, 0, sizeof(s));
s.batch_size = 32;
@@ -158,10 +156,8 @@ static void test_collect_job_batch_deadline(void)
job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u};
job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u};
snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
- snprintf(before_deadline.path, sizeof(before_deadline.path), "%s",
- "/v1/embeddings");
- snprintf(after_deadline.path, sizeof(after_deadline.path), "%s",
- "/v1/embeddings");
+ snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings");
+ snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings");
enqueue_job(&first);
enqueue_job(&before_deadline);
enqueue_job(&after_deadline);
@@ -179,8 +175,7 @@ static void test_collect_job_batch_deadline(void)
pthread_mutex_destroy(&s.mu);
}
-static void test_collect_job_batch_zero_wait(void)
-{
+static void test_collect_job_batch_zero_wait(void) {
http_server s;
memset(&s, 0, sizeof(s));
s.batch_size = 32;
@@ -211,10 +206,10 @@ static void test_collect_job_batch_zero_wait(void)
#define TEST_API_KEY "tt-test-key"
-static int find_free_port(void)
-{
+static int find_free_port(void) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
- if (fd < 0) return -1;
+ if (fd < 0)
+ return -1;
struct sockaddr_in a;
memset(&a, 0, sizeof(a));
a.sin_family = AF_INET;
@@ -230,10 +225,10 @@ static int find_free_port(void)
return port;
}
-static int tcp_connect(int port)
-{
+static int tcp_connect(int port) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
- if (fd < 0) return -1;
+ if (fd < 0)
+ return -1;
struct sockaddr_in a;
memset(&a, 0, sizeof(a));
a.sin_family = AF_INET;
@@ -246,13 +241,13 @@ static int tcp_connect(int port)
return fd;
}
-static int send_all(int fd, const char *buf, size_t n)
-{
+static int send_all(int fd, const char *buf, size_t n) {
size_t off = 0;
while (off < n) {
ssize_t w = write(fd, buf + off, n - off);
if (w < 0) {
- if (errno == EINTR) continue;
+ if (errno == EINTR)
+ continue;
return -1;
}
off += (size_t)w;
@@ -262,10 +257,14 @@ static int send_all(int fd, const char *buf, size_t n)
/* Send one request; raw == NULL builds a normal request, otherwise `raw`
* bytes go on the wire verbatim (for malformed-HTTP checks). */
-static int http_send(int fd, const char *method, const char *path,
- const char *auth, const char *body, const char *raw)
-{
- if (raw) return send_all(fd, raw, strlen(raw));
+static int http_send(int fd,
+ const char *method,
+ const char *path,
+ const char *auth,
+ const char *body,
+ const char *raw) {
+ if (raw)
+ return send_all(fd, raw, strlen(raw));
char head[512];
size_t blen = body ? strlen(body) : 0;
int n = snprintf(head, sizeof(head),
@@ -275,55 +274,71 @@ static int http_send(int fd, const char *method, const char *path,
"Content-Type: application/json\r\n"
"Content-Length: %zu\r\n"
"\r\n",
- method, path,
- auth ? "Authorization: Bearer " : "",
- auth ? auth : "", auth ? "\r\n" : "",
- blen);
- if (n < 0 || (size_t)n >= sizeof(head)) return -1;
- if (send_all(fd, head, (size_t)n) != 0) return -1;
+ method, path, auth ? "Authorization: Bearer " : "", auth ? auth : "",
+ auth ? "\r\n" : "", blen);
+ if (n < 0 || (size_t)n >= sizeof(head))
+ return -1;
+ if (send_all(fd, head, (size_t)n) != 0)
+ return -1;
return blen ? send_all(fd, body, blen) : 0;
}
/* Read one response: returns the status code (or -1) and the body, which
* the caller frees. Reads exactly Content-Length body bytes so it works
* with keep-alive connections. */
-static int http_recv(int fd, char **out_body)
-{
+static int http_recv(int fd, char **out_body) {
*out_body = NULL;
size_t cap = 8192, len = 0;
char *buf = malloc(cap);
- if (!buf) return -1;
+ if (!buf)
+ return -1;
char *hdr_end = NULL;
while (!hdr_end) {
if (len + 1 >= cap) {
cap *= 2;
char *nb = realloc(buf, cap);
- if (!nb) { free(buf); return -1; }
+ if (!nb) {
+ free(buf);
+ return -1;
+ }
buf = nb;
}
ssize_t r = read(fd, buf + len, cap - len - 1);
- if (r <= 0) { free(buf); return -1; }
+ if (r <= 0) {
+ free(buf);
+ return -1;
+ }
len += (size_t)r;
buf[len] = '\0';
hdr_end = strstr(buf, "\r\n\r\n");
}
int status = -1;
- if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) { free(buf); return -1; }
+ if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) {
+ free(buf);
+ return -1;
+ }
size_t content_length = 0;
const char *cl = strstr(buf, "Content-Length:");
- if (cl && cl < hdr_end) content_length = strtoul(cl + 15, NULL, 10);
+ if (cl && cl < hdr_end)
+ content_length = strtoul(cl + 15, NULL, 10);
size_t body_off = (size_t)(hdr_end - buf) + 4;
size_t need = body_off + content_length;
if (need + 1 > cap) {
char *nb = realloc(buf, need + 1);
- if (!nb) { free(buf); return -1; }
+ if (!nb) {
+ free(buf);
+ return -1;
+ }
buf = nb;
}
while (len < need) {
ssize_t r = read(fd, buf + len, need - len);
- if (r <= 0) { free(buf); return -1; }
+ if (r <= 0) {
+ free(buf);
+ return -1;
+ }
len += (size_t)r;
}
buf[need] = '\0';
@@ -334,19 +349,25 @@ static int http_recv(int fd, char **out_body)
/* One-shot request helper; returns status, body via out_body (caller frees,
* may be NULL when only the status matters). */
-static int http_req(int port, const char *method, const char *path,
- const char *auth, const char *body, const char *raw,
- char **out_body)
-{
+static int http_req(int port,
+ const char *method,
+ const char *path,
+ const char *auth,
+ const char *body,
+ const char *raw,
+ char **out_body) {
int fd = tcp_connect(port);
- if (fd < 0) return -1;
+ if (fd < 0)
+ return -1;
char *resp = NULL;
int status = -1;
if (http_send(fd, method, path, auth, body, raw) == 0)
status = http_recv(fd, &resp);
close(fd);
- if (out_body) *out_body = resp;
- else free(resp);
+ if (out_body)
+ *out_body = resp;
+ else
+ free(resp);
return status;
}
@@ -356,18 +377,17 @@ typedef struct {
int rc;
} srv_ctx;
-static void *srv_thread_main(void *arg)
-{
+static void *srv_thread_main(void *arg) {
srv_ctx *s = (srv_ctx *)arg;
s->rc = embed_run_server(&s->cfg);
return NULL;
}
-static cJSON *parse_embeddings_body(const char *body, int want_items)
-{
+static cJSON *parse_embeddings_body(const char *body, int want_items) {
cJSON *root = cJSON_Parse(body);
TEST_ASSERT(root != NULL);
- if (!root) return NULL;
+ if (!root)
+ return NULL;
cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
TEST_ASSERT(cJSON_IsArray(data));
TEST_ASSERT(cJSON_GetArraySize(data) == want_items);
@@ -378,71 +398,63 @@ static cJSON *parse_embeddings_body(const char *body, int want_items)
return root;
}
-static void test_http_embeddings(int port)
-{
+static void test_http_embeddings(int port) {
char *body = NULL;
/* Routing. */
- TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL,
- NULL) == 204);
- TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL,
- NULL, NULL) == 404);
- TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL,
- NULL) == 404);
+ TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL, NULL) == 204);
+ TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL, NULL, NULL) == 404);
+ TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL, NULL) == 404);
/* Auth: required, and the exact key. */
- const char *ok_req =
- "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\","
- "\"encoding_format\":\"float\"}";
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL,
- NULL) == 401);
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req,
- NULL, NULL) == 401);
+ const char *ok_req = "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\","
+ "\"encoding_format\":\"float\"}";
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL, NULL) == 401);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req, NULL, NULL) == 401);
/* Malformed HTTP (400) and malformed JSON (422 detail). */
- TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL,
- "GARBAGE\r\n\r\n", NULL) == 400);
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
- "{nope", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL, "GARBAGE\r\n\r\n", NULL) == 400);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{nope", NULL, NULL) == 422);
/* Validation: missing/unknown model, wrong endpoint family, valid but
* unloaded id, bad encoding enum, dimensions out of range; the
* API answers validation errors with 422 detail arrays. */
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
- "{\"input\":\"hi\"}", NULL, NULL) == 422);
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
- "{\"model\":\"foo\",\"input\":\"hi\"}", NULL,
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"input\":\"hi\"}", NULL,
NULL) == 422);
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"foo\",\"input\":\"hi\"}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-0.6b\","
- "\"input\":\"hi\"}", NULL, NULL) == 422);
+ "\"input\":\"hi\"}",
+ NULL, NULL) == 422);
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
- "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}",
- NULL, NULL) == 503);
+ "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}", NULL, NULL) == 503);
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\","
- "\"encoding_format\":\"yaml\"}", NULL, NULL) == 422);
+ "\"encoding_format\":\"yaml\"}",
+ NULL, NULL) == 422);
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\","
- "\"dimensions\":64}", NULL, NULL) == 422);
+ "\"dimensions\":64}",
+ NULL, NULL) == 422);
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\","
- "\"input\":[]}", NULL, NULL) == 422);
+ "\"input\":[]}",
+ NULL, NULL) == 422);
/* Contextual endpoint: standard id is the wrong enum, the 4B contextual
* id is valid but not loaded. */
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\","
- "\"input\":[[\"a\"]]}", NULL, NULL) == 422);
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ "\"input\":[[\"a\"]]}",
+ NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-4b\","
- "\"input\":[[\"a\"]]}", NULL, NULL) == 503);
+ "\"input\":[[\"a\"]]}",
+ NULL, NULL) == 503);
/* Happy path, float encoding, single string input. */
- TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req,
- NULL, &body) == 200);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req, NULL, &body) == 200);
cJSON *root = body ? parse_embeddings_body(body, 1) : NULL;
if (root) {
cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
@@ -456,12 +468,12 @@ static void test_http_embeddings(int port)
cJSON *v;
cJSON_ArrayForEach(v, emb) {
TEST_ASSERT(cJSON_IsNumber(v));
- if (v->valuedouble != 0.0) nonzero = 1;
+ if (v->valuedouble != 0.0)
+ nonzero = 1;
}
TEST_ASSERT(nonzero);
cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage");
- cJSON *total = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
TEST_ASSERT(total && cJSON_IsNumber(total) && total->valueint >= 1);
cJSON_Delete(root);
}
@@ -478,19 +490,15 @@ static void test_http_embeddings(int port)
root = body ? parse_embeddings_body(body, 3) : NULL;
if (root) {
cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
- cJSON *e0 = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(data, 0), "embedding");
- cJSON *e1 = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(data, 1), "embedding");
+ cJSON *e0 = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, 0), "embedding");
+ cJSON *e1 = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, 1), "embedding");
for (int i = 0; i < 3; i++) {
- cJSON *idx = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(data, i), "index");
+ cJSON *idx = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, i), "index");
TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == i);
}
int differ = 0;
for (int i = 0; i < 1024 && !differ; i++) {
- if (cJSON_GetArrayItem(e0, i)->valuedouble !=
- cJSON_GetArrayItem(e1, i)->valuedouble)
+ if (cJSON_GetArrayItem(e0, i)->valuedouble != cJSON_GetArrayItem(e1, i)->valuedouble)
differ = 1;
}
TEST_ASSERT(differ);
@@ -502,12 +510,12 @@ static void test_http_embeddings(int port)
/* Default encoding is base64_int8: a string of 1024 quantized bytes. */
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\","
- "\"input\":\"hello\"}", NULL, &body) == 200);
+ "\"input\":\"hello\"}",
+ NULL, &body) == 200);
root = body ? parse_embeddings_body(body, 1) : NULL;
if (root) {
cJSON *emb = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"),
- 0), "embedding");
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), 0), "embedding");
TEST_ASSERT(cJSON_IsString(emb));
TEST_ASSERT(strlen(emb->valuestring) == ((1024 + 2) / 3) * 4);
cJSON_Delete(root);
@@ -524,8 +532,7 @@ static void test_http_embeddings(int port)
root = body ? parse_embeddings_body(body, 1) : NULL;
if (root) {
cJSON *emb = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"),
- 0), "embedding");
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), 0), "embedding");
TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 128);
cJSON_Delete(root);
}
@@ -548,28 +555,24 @@ static void test_http_embeddings(int port)
NULL, &body) == 200);
cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL;
if (qwen_full && qwen_mrl) {
- cJSON *full_data =
- cJSON_GetObjectItemCaseSensitive(qwen_full, "data");
- cJSON *mrl_data =
- cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data");
- cJSON *full_emb = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(full_data, 0), "embedding");
- cJSON *mrl_emb = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(mrl_data, 0), "embedding");
- TEST_ASSERT(cJSON_IsArray(full_emb) &&
- cJSON_GetArraySize(full_emb) == 1024);
- TEST_ASSERT(cJSON_IsArray(mrl_emb) &&
- cJSON_GetArraySize(mrl_emb) == 32);
+ cJSON *full_data = cJSON_GetObjectItemCaseSensitive(qwen_full, "data");
+ cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data");
+ cJSON *full_emb =
+ cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(full_data, 0), "embedding");
+ cJSON *mrl_emb =
+ cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(mrl_data, 0), "embedding");
+ TEST_ASSERT(cJSON_IsArray(full_emb) && cJSON_GetArraySize(full_emb) == 1024);
+ TEST_ASSERT(cJSON_IsArray(mrl_emb) && cJSON_GetArraySize(mrl_emb) == 32);
double prefix_diff = 0.0;
for (int i = 0; i < 32; i++) {
double d = fabs(cJSON_GetArrayItem(full_emb, i)->valuedouble -
cJSON_GetArrayItem(mrl_emb, i)->valuedouble);
- if (d > prefix_diff) prefix_diff = d;
+ if (d > prefix_diff)
+ prefix_diff = d;
}
TEST_ASSERT(prefix_diff > 0.0);
cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage");
- cJSON *total = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
TEST_ASSERT(total && total->valueint == 2);
}
cJSON_Delete(qwen_full);
@@ -585,11 +588,12 @@ static void test_http_embeddings(int port)
fds[i] = tcp_connect(port);
TEST_ASSERT(fds[i] >= 0);
if (fds[i] >= 0)
- TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings",
- TEST_API_KEY, ok_req, NULL) == 0);
+ TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings", TEST_API_KEY, ok_req, NULL) ==
+ 0);
}
for (int i = 0; i < N_CONC; i++) {
- if (fds[i] < 0) continue;
+ if (fds[i] < 0)
+ continue;
char *resp = NULL;
TEST_ASSERT(http_recv(fds[i], &resp) == 200);
free(resp);
@@ -598,8 +602,7 @@ static void test_http_embeddings(int port)
}
/* data[di].data[ci].embedding from a contextual response. */
-static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci)
-{
+static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci) {
cJSON *docs = cJSON_GetObjectItemCaseSensitive(root, "data");
cJSON *doc = cJSON_GetArrayItem(docs, di);
cJSON *chunks = cJSON_GetObjectItemCaseSensitive(doc, "data");
@@ -607,40 +610,40 @@ static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci)
return cJSON_GetObjectItemCaseSensitive(item, "embedding");
}
-static double emb_max_absdiff(cJSON *a, cJSON *b)
-{
- if (!cJSON_IsArray(a) || !cJSON_IsArray(b)) return 1e9;
+static double emb_max_absdiff(cJSON *a, cJSON *b) {
+ if (!cJSON_IsArray(a) || !cJSON_IsArray(b))
+ return 1e9;
int n = cJSON_GetArraySize(a);
- if (n != cJSON_GetArraySize(b)) return 1e9;
+ if (n != cJSON_GetArraySize(b))
+ return 1e9;
double m = 0.0;
for (int i = 0; i < n; i++) {
- double d = fabs(cJSON_GetArrayItem(a, i)->valuedouble -
- cJSON_GetArrayItem(b, i)->valuedouble);
- if (d > m) m = d;
+ double d =
+ fabs(cJSON_GetArrayItem(a, i)->valuedouble - cJSON_GetArrayItem(b, i)->valuedouble);
+ if (d > m)
+ m = d;
}
return m;
}
-static void test_http_contextual(int port)
-{
+static void test_http_contextual(int port) {
char *body = NULL;
/* Input must be an array of chunk arrays; chunks must be non-empty. */
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-0.6b\","
- "\"input\":\"hi\"}", NULL, NULL) == 422);
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ "\"input\":\"hi\"}",
+ NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-0.6b\","
- "\"input\":[[\"\"]]}", NULL, NULL) == 422);
+ "\"input\":[[\"\"]]}",
+ NULL, NULL) == 422);
/* Happy path: chunks concatenate per document with one separator token
* between them, one embedding per chunk in document order. Fixture
* token counts: hello=1, world=3, held=2, one separator in doc 0, so
* usage must report 7. */
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-0.6b\","
"\"input\":[[\"hello\",\"world\"],[\"held\"]],"
"\"encoding_format\":\"float\"}",
@@ -664,21 +667,18 @@ static void test_http_contextual(int port)
cJSON *cidx = cJSON_GetObjectItemCaseSensitive(item, "index");
cJSON *emb = ctx_chunk_emb(multi, di, ci);
TEST_ASSERT(cidx && cidx->valueint == ci);
- TEST_ASSERT(cJSON_IsArray(emb) &&
- cJSON_GetArraySize(emb) == 1024);
+ TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 1024);
}
}
cJSON *usage = cJSON_GetObjectItemCaseSensitive(multi, "usage");
- cJSON *total = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
TEST_ASSERT(total && total->valueint == 7);
}
/* A single-chunk document has no separators and its pooling span covers
* the whole sequence, so it must match the standard embedding of the
* same text from the same weights. */
- TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
- TEST_API_KEY,
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-context-v1-0.6b\","
"\"input\":[[\"hello\"]],"
"\"encoding_format\":\"float\"}",
@@ -699,8 +699,7 @@ static void test_http_contextual(int port)
if (multi && single && std) {
cJSON *std_emb = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(std, "data"),
- 0), "embedding");
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(std, "data"), 0), "embedding");
cJSON *alone = ctx_chunk_emb(single, 0, 0);
cJSON *in_doc = ctx_chunk_emb(multi, 0, 0);
TEST_ASSERT(emb_max_absdiff(std_emb, alone) < 1e-4);
@@ -713,8 +712,7 @@ static void test_http_contextual(int port)
cJSON_Delete(std);
}
-static void test_http_rerank(int port)
-{
+static void test_http_rerank(int port) {
char *body = NULL;
TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY,
@@ -735,55 +733,45 @@ static void test_http_rerank(int port)
"\"top_n\":1,\"top_k\":1}",
NULL, NULL) == 422);
- const char *request =
- "{\"model\":\"pplx-embed-v1-late-0.6b\","
- "\"query\":\"hello\",\"documents\":["
- "\"hello!\",\"world\",\"quoted \\\"document\\\"\"],"
- "\"top_n\":2,\"return_documents\":true}";
- TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY, request,
- NULL, &body) == 200);
+ const char *request = "{\"model\":\"pplx-embed-v1-late-0.6b\","
+ "\"query\":\"hello\",\"documents\":["
+ "\"hello!\",\"world\",\"quoted \\\"document\\\"\"],"
+ "\"top_n\":2,\"return_documents\":true}";
+ TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY, request, NULL, &body) == 200);
cJSON *root = body ? cJSON_Parse(body) : NULL;
TEST_ASSERT(root != NULL);
if (root) {
cJSON *object = cJSON_GetObjectItemCaseSensitive(root, "object");
cJSON *model = cJSON_GetObjectItemCaseSensitive(root, "model");
cJSON *results = cJSON_GetObjectItemCaseSensitive(root, "results");
- TEST_ASSERT(cJSON_IsString(object) &&
- strcmp(object->valuestring, "list") == 0);
+ TEST_ASSERT(cJSON_IsString(object) && strcmp(object->valuestring, "list") == 0);
TEST_ASSERT(cJSON_IsString(model) &&
- strcmp(model->valuestring,
- "pplx-embed-v1-late-0.6b") == 0);
- TEST_ASSERT(cJSON_IsArray(results) &&
- cJSON_GetArraySize(results) == 2);
+ strcmp(model->valuestring, "pplx-embed-v1-late-0.6b") == 0);
+ TEST_ASSERT(cJSON_IsArray(results) && cJSON_GetArraySize(results) == 2);
double previous = INFINITY;
int seen[3] = {0};
cJSON *result;
cJSON_ArrayForEach(result, results) {
- cJSON *index =
- cJSON_GetObjectItemCaseSensitive(result, "index");
- cJSON *score =
- cJSON_GetObjectItemCaseSensitive(result, "relevance_score");
- cJSON *document =
- cJSON_GetObjectItemCaseSensitive(result, "document");
- TEST_ASSERT(cJSON_IsNumber(index) &&
- index->valueint >= 0 && index->valueint < 3);
- TEST_ASSERT(cJSON_IsNumber(score) &&
- score->valuedouble <= previous);
+ cJSON *index = cJSON_GetObjectItemCaseSensitive(result, "index");
+ cJSON *score = cJSON_GetObjectItemCaseSensitive(result, "relevance_score");
+ cJSON *document = cJSON_GetObjectItemCaseSensitive(result, "document");
+ TEST_ASSERT(cJSON_IsNumber(index) && index->valueint >= 0 && index->valueint < 3);
+ TEST_ASSERT(cJSON_IsNumber(score) && score->valuedouble <= previous);
TEST_ASSERT(cJSON_IsString(document));
- if (cJSON_IsNumber(index) && index->valueint >= 0 &&
- index->valueint < 3)
+ if (cJSON_IsNumber(index) && index->valueint >= 0 && index->valueint < 3)
seen[index->valueint]++;
- if (cJSON_IsNumber(score)) previous = score->valuedouble;
+ if (cJSON_IsNumber(score))
+ previous = score->valuedouble;
}
TEST_ASSERT(seen[0] + seen[1] + seen[2] == 2);
cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage");
- cJSON *query_tokens = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "query_tokens") : NULL;
- cJSON *document_tokens = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL;
- cJSON *total_tokens = usage
- ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ cJSON *query_tokens =
+ usage ? cJSON_GetObjectItemCaseSensitive(usage, "query_tokens") : NULL;
+ cJSON *document_tokens =
+ usage ? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL;
+ cJSON *total_tokens =
+ usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
TEST_ASSERT(query_tokens && document_tokens && total_tokens);
if (query_tokens && document_tokens && total_tokens) {
TEST_ASSERT(cJSON_IsNumber(query_tokens));
@@ -808,18 +796,16 @@ static void test_http_rerank(int port)
TEST_ASSERT(root != NULL);
if (root) {
cJSON *results = cJSON_GetObjectItemCaseSensitive(root, "results");
- TEST_ASSERT(cJSON_IsArray(results) &&
- cJSON_GetArraySize(results) == 1);
- cJSON *document = cJSON_GetObjectItemCaseSensitive(
- cJSON_GetArrayItem(results, 0), "document");
+ TEST_ASSERT(cJSON_IsArray(results) && cJSON_GetArraySize(results) == 1);
+ cJSON *document =
+ cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(results, 0), "document");
TEST_ASSERT(document == NULL);
cJSON_Delete(root);
}
free(body);
}
-static void test_http_server(void)
-{
+static void test_http_server(void) {
char dir[1024], late_dir[1024], qwen_dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
@@ -845,14 +831,11 @@ static void test_http_server(void)
/* The 0.6b server slot requires hidden_size 1024; everything else stays
* tiny. The model vocab covers every tokenizer fixture id. */
tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE};
- if (tf_write_vocab(dir) != 0 ||
- tm_write_model_dims(dir, "F32", &dims) != 0 ||
- tf_write_vocab(late_dir) != 0 ||
- tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
+ if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 ||
+ tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 ||
tf_write_vocab(qwen_dir) != 0 ||
- tm_write_qwen3_model_dims(qwen_dir, "F32", &dims,
- TF_EOT_ID) != 0) {
+ tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0) {
fprintf(stderr, "FAIL: fixture write\n");
test_failures++;
return;
@@ -888,9 +871,13 @@ static void test_http_server(void)
}
int up = 0;
- for (int i = 0; i < 200; i++) { /* up to ~10 s */
+ for (int i = 0; i < 200; i++) { /* up to ~10 s */
int fd = tcp_connect(port);
- if (fd >= 0) { close(fd); up = 1; break; }
+ if (fd >= 0) {
+ close(fd);
+ up = 1;
+ break;
+ }
usleep(50 * 1000);
}
TEST_ASSERT(up);
@@ -905,8 +892,7 @@ static void test_http_server(void)
TEST_ASSERT(ctx.rc == 0);
}
-int main(void)
-{
+int main(void) {
test_base64_rfc4648();
test_quantize_int8_tanh();
test_encode_embedding_int8();
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index bed5b7d..5920977 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -16,31 +16,33 @@
static int failures = 0;
int qwen_verbose = 0;
-#define TEST_ASSERT(cond) do { \
- if (!(cond)) { \
- fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
- failures++; \
- } \
-} while (0)
+#define TEST_ASSERT(cond) \
+ do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ failures++; \
+ } \
+ } while (0)
static void check_ids(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *text,
- const int *want, int n_want, const char *label)
-{
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ const int *want,
+ int n_want,
+ const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL);
- if (!ids) return;
+ if (!ids)
+ return;
if (n != n_want) {
- fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n",
- label, text, n, n_want);
+ fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", label, text, n, n_want);
failures++;
} else {
for (int i = 0; i < n; i++) {
if (ids[i] != want[i]) {
- fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n",
- label, text, i, ids[i], want[i]);
+ fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", label, text, i, ids[i],
+ want[i]);
failures++;
}
}
@@ -51,25 +53,30 @@ static void check_ids(const qwen_tokenizer_t *tok,
/* Concatenated decode of an encoding must reproduce the (NFC-normalized)
* input - byte-level BPE is lossless. */
static void check_roundtrip(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *text,
- const char *want, const char *label)
-{
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ const char *want,
+ const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL && n > 0);
- if (!ids) return;
+ if (!ids)
+ return;
size_t cap = 1;
for (int i = 0; i < n; i++)
cap += strlen(qwen_tokenizer_decode(tok, ids[i]));
char *got = (char *)malloc(cap);
- if (!got) { free(ids); TEST_ASSERT(0); return; }
+ if (!got) {
+ free(ids);
+ TEST_ASSERT(0);
+ return;
+ }
got[0] = '\0';
for (int i = 0; i < n; i++)
strcat(got, qwen_tokenizer_decode(tok, ids[i]));
if (strcmp(got, want) != 0) {
- fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n",
- label, text, got, want);
+ fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got,
+ want);
failures++;
}
free(got);
@@ -79,15 +86,21 @@ static void check_roundtrip(const qwen_tokenizer_t *tok,
/* Two spellings of the same text must encode identically. */
static void check_same_ids(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
- const char *a, const char *b, const char *label)
-{
+ const char *a,
+ const char *b,
+ const char *label) {
int na = 0, nb = 0;
- int *ia = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, a, &na);
- if (!ia) { TEST_ASSERT(0); return; }
- int *ib = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, b, &nb);
- if (!ib) { free(ia); TEST_ASSERT(0); return; }
+ int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na);
+ if (!ia) {
+ TEST_ASSERT(0);
+ return;
+ }
+ int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb);
+ if (!ib) {
+ free(ia);
+ TEST_ASSERT(0);
+ return;
+ }
if (na != nb || memcmp(ia, ib, (size_t)na * sizeof(int)) != 0) {
fprintf(stderr, "FAIL %s same-ids \"%s\" vs \"%s\"\n", label, a, b);
failures++;
@@ -96,9 +109,8 @@ static void check_same_ids(const qwen_tokenizer_t *tok,
free(ia);
}
-static void run_cases(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *label)
-{
+static void
+run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) {
/* h e l l o -> he -> he,ll -> hell -> hello */
const int hello[] = {259};
check_ids(tok, ws, "hello", hello, 1, label);
@@ -147,14 +159,14 @@ static void run_cases(const qwen_tokenizer_t *tok,
/* NFC: decomposed base+combining-mark spellings must match the
* precomposed codepoint (one pair per mark in the composition table). */
- check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */
- check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */
- check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */
- check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */
- check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */
- check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */
- check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */
- check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */
+ check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */
+ check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */
+ check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */
+ check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */
+ check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */
+ check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */
+ check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */
+ check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */
check_roundtrip(tok, ws, "cafe\xCC\x81", "caf\xC3\xA9", label);
/* 'x' + acute has no composition: bytes pass through untouched. */
check_roundtrip(tok, ws, "x\xCC\x81", "x\xCC\x81", label);
@@ -177,30 +189,37 @@ static void run_cases(const qwen_tokenizer_t *tok,
check_roundtrip(tok, ws, "\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", label);
check_roundtrip(tok, ws, "tab\tand\nnl 42!", "tab\tand\nnl 42!", label);
const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'};
- check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label);
+ check_ids(tok, ws,
+ "a\xE2\x80\xA6"
+ "b",
+ ellipsis, 5, label);
}
-static void write_file(const char *dir, const char *name, const char *body)
-{
+static void write_file(const char *dir, const char *name, const char *body) {
char path[2048];
snprintf(path, sizeof(path), "%s/%s", dir, name);
FILE *f = fopen(path, "w");
- if (!f) { perror(path); exit(2); }
+ if (!f) {
+ perror(path);
+ exit(2);
+ }
fputs(body, f);
fclose(f);
}
/* Loader rejection paths and vocab.json escape handling, in throwaway
* subdirectories of the main fixture dir. */
-static void test_loader_edges(const char *root)
-{
+static void test_loader_edges(const char *root) {
char dir[1152], path[2048];
TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL);
/* Not a JSON object. */
snprintf(dir, sizeof(dir), "%s/badjson", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "[1, 2]\n");
write_file(dir, "merges.txt", "a b\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
@@ -209,7 +228,10 @@ static void test_loader_edges(const char *root)
/* Missing or pair-less merges.txt is tolerated: the tokenizer loads
* and encoding falls back to byte/vocab-level (no merges applied). */
snprintf(dir, sizeof(dir), "%s/nomerges", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
qwen_tokenizer_t *nom = qwen_tokenizer_load(path);
@@ -225,7 +247,10 @@ static void test_loader_edges(const char *root)
}
snprintf(dir, sizeof(dir), "%s/emptymerges", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
write_file(dir, "merges.txt", "#version: 0.2\n\n \n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
@@ -235,7 +260,10 @@ static void test_loader_edges(const char *root)
/* Escape forms in vocab keys; a malformed merge line is skipped. */
snprintf(dir, sizeof(dir), "%s/escapes", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json",
"{\"a\":0, \"b\":1, \"ab\":2,\n"
" \"\\u20AC\":3, \"x\\ny\":4, \"q\\\"r\":5,\n"
@@ -264,12 +292,14 @@ static void test_loader_edges(const char *root)
qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json");
TEST_ASSERT(rel != NULL);
qwen_tokenizer_free(rel);
- if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); }
+ if (chdir(saved_cwd) != 0) {
+ perror("chdir back");
+ exit(2);
+ }
}
}
-int main(void)
-{
+int main(void) {
char dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-tok-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
@@ -282,7 +312,8 @@ int main(void)
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir);
qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
TEST_ASSERT(tok != NULL);
- if (!tok) return 1;
+ if (!tok)
+ return 1;
/* The complete byte alphabet means the integer-pair tables must exist. */
TEST_ASSERT(tok->int_merges != NULL);
@@ -312,21 +343,17 @@ int main(void)
{
int out[8];
int n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8,
- &n) == 0 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262);
/* On overflow the prefix is written and n reports the total
* required count, so a caller can re-size and retry. */
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2,
- &n) == -2 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0,
- &n) == -2 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 &&
- n == 0);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0);
TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1);
TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1);
}
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
index 4f83936..a906689 100644
--- a/tests/test_workspace.c
+++ b/tests/test_workspace.c
@@ -14,12 +14,12 @@
#include <string.h>
#include <unistd.h>
-static float max_abs_diff(const float *a, const float *b, int n)
-{
+static float max_abs_diff(const float *a, const float *b, int n) {
float m = 0.0f;
for (int i = 0; i < n; i++) {
float d = fabsf(a[i] - b[i]);
- if (d > m) m = d;
+ if (d > m)
+ m = d;
}
return m;
}
@@ -28,11 +28,11 @@ static float max_abs_diff(const float *a, const float *b, int n)
* results validated in main. reference is the batched row for ids0. */
static int check_alloc_and_pooling_variants(const embed_model_t *model,
embed_workspace_t *ws,
- const int *ids0, int n0,
+ const int *ids0,
+ int n0,
const float *states,
const float *reference,
- const embed_config_t *cfg)
-{
+ const embed_config_t *cfg) {
int rc = 1;
int dim = cfg->hidden_size;
float *emb = NULL, *fwd = NULL, *pooled = NULL, *span_out = NULL;
@@ -73,22 +73,19 @@ static int check_alloc_and_pooling_variants(const embed_model_t *model,
/* One span covering the whole sequence pools to the embedding, and
* two halves recombine into it by token-count weighting. */
embed_span_t whole = {0, n0};
- if (embed_model_encode_spans(model, ws, ids0, n0, &whole, 1,
- span_out) != 0 ||
+ if (embed_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 ||
max_abs_diff(span_out, reference, dim) > 0.00005f) {
fprintf(stderr, "whole-sequence span disagrees with embedding\n");
goto done;
}
int h = n0 / 2;
embed_span_t halves[2] = {{0, h}, {h, n0 - h}};
- if (embed_model_encode_spans(model, ws, ids0, n0, halves, 2,
- span_out) != 0) {
+ if (embed_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) {
fprintf(stderr, "embed_model_encode_spans failed for two spans\n");
goto done;
}
for (int d = 0; d < dim; d++) {
- float combined = ((float)h * span_out[d] +
- (float)(n0 - h) * span_out[dim + d]) / (float)n0;
+ float combined = ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0;
if (fabsf(combined - reference[d]) > 0.00005f) {
fprintf(stderr, "half spans do not recombine into embedding\n");
goto done;
@@ -108,10 +105,10 @@ done:
* doc0 is chunks 0 and 1 joined by a separator token; doc1 is chunk 2. */
static int check_spans_batch_parity(const embed_model_t *model,
embed_workspace_t *ws,
- int *const ids[], const int *ntok,
+ int *const ids[],
+ const int *ntok,
int separator_id,
- const embed_config_t *cfg)
-{
+ const embed_config_t *cfg) {
int rc = 1;
int dim = cfg->hidden_size;
int n0 = ntok[0] + 1 + ntok[1];
@@ -128,10 +125,9 @@ static int check_spans_batch_parity(const embed_model_t *model,
embed_span_t spans0[2] = {{0, ntok[0]}, {ntok[0] + 1, ntok[1]}};
embed_span_t span1 = {0, ntok[2]};
- if (embed_model_encode_spans(model, ws, doc0, n0, spans0, 2,
- expected) != 0 ||
+ if (embed_model_encode_spans(model, ws, doc0, n0, spans0, 2, expected) != 0 ||
embed_model_encode_spans(model, ws, ids[2], ntok[2], &span1, 1,
- expected + (size_t)2 * dim) != 0) {
+ expected + (size_t)2 * dim) != 0) {
fprintf(stderr, "singleton contextual embedding failed\n");
goto done;
}
@@ -164,8 +160,7 @@ done:
return rc;
}
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
char fixture_dir[1024];
const char *model_dir;
if (argc == 2) {
@@ -214,12 +209,10 @@ int main(int argc, char **argv)
return 1;
}
- const char *texts[] = {
- "query: what is the capital of France?",
- "document: Paris is the capital of France.",
- "document: Berlin is the capital of Germany.",
- "document: Istanbul is a major city in Turkey."
- };
+ const char *texts[] = {"query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: Istanbul is a major city in Turkey."};
const int batch = (int)(sizeof(texts) / sizeof(texts[0]));
/* Initialized before the first `goto fail` so the cleanup loop never
* sees indeterminate pointers. */
@@ -291,8 +284,10 @@ int main(int argc, char **argv)
const float *row = batched + (size_t)i * dim;
float diff = max_abs_diff(row, single, dim);
float cos = embed_cosine_similarity(row, single, dim);
- if (diff > worst_diff) worst_diff = diff;
- if (cos < worst_cos) worst_cos = cos;
+ if (diff > worst_diff)
+ worst_diff = diff;
+ if (cos < worst_cos)
+ worst_cos = cos;
}
memcpy(normalized, batched, (size_t)dim * sizeof(float));
@@ -308,8 +303,7 @@ int main(int argc, char **argv)
for (int i = 0; i < dim; i++)
normalized_norm_sq += normalized[i] * normalized[i];
if (fabsf(normalized_norm_sq - 1.0f) > 0.00005f) {
- fprintf(stderr, "embed_l2_normalize produced norm_sq=%g\n",
- normalized_norm_sq);
+ fprintf(stderr, "embed_l2_normalize produced norm_sq=%g\n", normalized_norm_sq);
free(states);
free(normalized);
free(single);
@@ -326,8 +320,8 @@ int main(int argc, char **argv)
goto fail;
}
- if (check_alloc_and_pooling_variants(model, batch_ws, ids[0], ntok[0],
- states, batched, cfg) != 0) {
+ if (check_alloc_and_pooling_variants(model, batch_ws, ids[0], ntok[0], states, batched, cfg) !=
+ 0) {
free(states);
free(normalized);
free(single);
@@ -338,9 +332,9 @@ int main(int argc, char **argv)
/* Same separator resolution as the server: the fixture vocab defines
* <|endoftext|>, real snapshots use the reserved id. */
int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>");
- if (sep_id < 0) sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id,
- cfg) != 0) {
+ if (sep_id < 0)
+ sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) {
free(states);
free(normalized);
free(single);
@@ -353,24 +347,26 @@ int main(int argc, char **argv)
free(single);
free(batched);
- for (int i = 0; i < batch; i++) free(ids[i]);
+ for (int i = 0; i < batch; i++)
+ free(ids[i]);
qwen_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);
if (worst_diff > 0.00005f || worst_cos < 0.99999f) {
- fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n",
- worst_diff, worst_cos);
+ fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n", worst_diff,
+ worst_cos);
return 1;
}
- printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n",
- dim, batch, batch_ws_bytes, worst_diff, worst_cos);
+ printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n", dim,
+ batch, batch_ws_bytes, worst_diff, worst_cos);
return 0;
fail:
- for (int i = 0; i < batch; i++) free(ids[i]);
+ for (int i = 0; i < batch; i++)
+ free(ids[i]);
qwen_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index d4ae431..42542c1 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -14,15 +14,13 @@
#include <sys/stat.h>
#include <sys/types.h>
-static uint16_t tm_f32_to_bf16(float x)
-{
+static uint16_t tm_f32_to_bf16(float x) {
uint32_t u;
memcpy(&u, &x, sizeof(u));
return (uint16_t)(u >> 16);
}
-static float tm_bf16_to_f32(uint16_t b)
-{
+static float tm_bf16_to_f32(uint16_t b) {
uint32_t u = ((uint32_t)b) << 16;
float f;
memcpy(&f, &u, sizeof(f));
@@ -30,8 +28,7 @@ static float tm_bf16_to_f32(uint16_t b)
}
/* Deterministic, BF16-exact value for element i of tensor `name`. */
-static float tm_value(const char *name, size_t i)
-{
+static float tm_value(const char *name, size_t i) {
float v;
size_t len = strlen(name);
if ((len >= 11 && strcmp(name + len - 11, "norm.weight") == 0) ||
@@ -46,7 +43,7 @@ static float tm_value(const char *name, size_t i)
typedef struct {
const char *name;
- int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */
+ int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */
} tm_spec_t;
/* Always one layer; everything else is free so a test can match a real
@@ -57,22 +54,21 @@ typedef struct {
enum { TM_N_SPECS = 13 };
-static int tm_write_config(const char *dir, const tm_dims_t *d,
- const char *model_type, int eos_token_id)
-{
+static int
+tm_write_config(const char *dir, const tm_dims_t *d, const char *model_type, int eos_token_id) {
char path[2048];
snprintf(path, sizeof(path), "%s/config.json", dir);
FILE *f = fopen(path, "w");
- if (!f) return -1;
- fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ if (!f)
+ return -1;
+ fprintf(f,
+ "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
"\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
"\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
"\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0",
- d->hidden, d->heads, d->kv_heads, d->head_dim,
- d->intermediate, d->vocab);
+ d->hidden, d->heads, d->kv_heads, d->head_dim, d->intermediate, d->vocab);
if (model_type)
- fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d",
- model_type, eos_token_id);
+ fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d", model_type, eos_token_id);
fputs("}", f);
fclose(f);
return 0;
@@ -80,9 +76,8 @@ static int tm_write_config(const char *dir, const tm_dims_t *d,
/* Write a safetensors file holding `specs` with tm_value() contents.
* dtype: "F32" or "BF16". Returns 0 on success. */
-static int tm_write_safetensors(const char *path, const char *dtype,
- const tm_spec_t *specs, int n_specs)
-{
+static int
+tm_write_safetensors(const char *path, const char *dtype, const tm_spec_t *specs, int n_specs) {
int bf16 = strcmp(dtype, "BF16") == 0;
size_t esize = bf16 ? 2 : 4;
@@ -95,23 +90,26 @@ static int tm_write_safetensors(const char *path, const char *dtype,
size_t end = doff + n * esize;
if (s->cols)
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff,
- "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d,%d],"
- "\"data_offsets\":[%zu,%zu]}",
- t ? "," : "", s->name, dtype, s->rows, s->cols, doff, end);
+ "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d,%d],"
+ "\"data_offsets\":[%zu,%zu]}",
+ t ? "," : "", s->name, dtype, s->rows, s->cols, doff, end);
else
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff,
- "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d],"
- "\"data_offsets\":[%zu,%zu]}",
- t ? "," : "", s->name, dtype, s->rows, doff, end);
+ "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d],"
+ "\"data_offsets\":[%zu,%zu]}",
+ t ? "," : "", s->name, dtype, s->rows, doff, end);
doff = end;
}
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "}");
- if (hoff >= sizeof(header) - 1) return -1;
+ if (hoff >= sizeof(header) - 1)
+ return -1;
FILE *f = fopen(path, "wb");
- if (!f) return -1;
+ if (!f)
+ return -1;
unsigned char len8[8];
- for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(hoff >> (8 * i));
+ for (int i = 0; i < 8; i++)
+ len8[i] = (unsigned char)(hoff >> (8 * i));
fwrite(len8, 1, 8, f);
fwrite(header, 1, hoff, f);
for (int t = 0; t < n_specs; t++) {
@@ -133,9 +131,7 @@ static int tm_write_safetensors(const char *path, const char *dtype,
/* Write config.json + model.safetensors with the given dimensions into dir.
* dtype: "F32" or "BF16". Returns 0 on success. */
-static int tm_write_model_dims(const char *dir, const char *dtype,
- const tm_dims_t *d)
-{
+static int tm_write_model_dims(const char *dir, const char *dtype, const tm_dims_t *d) {
int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
const tm_spec_t specs[TM_N_SPECS] = {
{"embed_tokens.weight", d->vocab, d->hidden},
@@ -154,37 +150,37 @@ static int tm_write_model_dims(const char *dir, const char *dtype,
};
char path[2048];
- if (tm_write_config(dir, d, NULL, -1) != 0) return -1;
+ if (tm_write_config(dir, d, NULL, -1) != 0)
+ return -1;
snprintf(path, sizeof(path), "%s/model.safetensors", dir);
return tm_write_safetensors(path, dtype, specs, TM_N_SPECS);
}
-static inline int tm_write_qwen3_model_dims(const char *dir, const char *dtype,
+static inline int tm_write_qwen3_model_dims(const char *dir,
+ const char *dtype,
const tm_dims_t *d,
- int eos_token_id)
-{
- if (tm_write_model_dims(dir, dtype, d) != 0) return -1;
+ int eos_token_id) {
+ if (tm_write_model_dims(dir, dtype, d) != 0)
+ return -1;
return tm_write_config(dir, d, "qwen3", eos_token_id);
}
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */
-static inline int tm_write_late_projection_named(const char *dir,
- const char *dtype,
- int token_dim, int hidden,
- const char *weight_name)
-{
+static inline int tm_write_late_projection_named(
+ const char *dir, const char *dtype, int token_dim, int hidden, const char *weight_name) {
char path[2048];
snprintf(path, sizeof(path), "%s/1_Dense", dir);
- if (mkdir(path, 0755) != 0 && errno != EEXIST) return -1;
+ if (mkdir(path, 0755) != 0 && errno != EEXIST)
+ return -1;
snprintf(path, sizeof(path), "%s/1_Dense/config.json", dir);
FILE *f = fopen(path, "w");
- if (!f) return -1;
- fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}",
- hidden, token_dim);
+ if (!f)
+ return -1;
+ fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}", hidden, token_dim);
fclose(f);
const tm_spec_t spec = {weight_name, token_dim, hidden};
@@ -192,16 +188,13 @@ static inline int tm_write_late_projection_named(const char *dir,
return tm_write_safetensors(path, dtype, &spec, 1);
}
-static inline int tm_write_late_projection(const char *dir, const char *dtype,
- int token_dim, int hidden)
-{
- return tm_write_late_projection_named(dir, dtype, token_dim, hidden,
- "linear.weight");
+static inline int
+tm_write_late_projection(const char *dir, const char *dtype, int token_dim, int hidden) {
+ return tm_write_late_projection_named(dir, dtype, token_dim, hidden, "linear.weight");
}
/* The historical default fixture: 4-dim hidden, 16-token vocab. */
-static inline int tm_write_model(const char *dir, const char *dtype)
-{
+static inline int tm_write_model(const char *dir, const char *dtype) {
static const tm_dims_t d = {4, 2, 1, 2, 8, 16};
return tm_write_model_dims(dir, dtype, &d);
}
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
index 7890927..24e1563 100644
--- a/tests/tok_fixture.h
+++ b/tests/tok_fixture.h
@@ -10,15 +10,13 @@
#include <string.h>
/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */
-static int tf_byte_to_cp(int b)
-{
+static int tf_byte_to_cp(int b) {
static int table[256];
static int init = 0;
if (!init) {
int n = 0;
for (int i = 0; i < 256; i++) {
- int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) ||
- (i >= 174 && i <= 255);
+ int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || (i >= 174 && i <= 255);
table[i] = normal ? i : 256 + n++;
}
init = 1;
@@ -26,9 +24,12 @@ static int tf_byte_to_cp(int b)
return table[b];
}
-static int tf_cp_to_utf8(int cp, char *out)
-{
- if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; }
+static int tf_cp_to_utf8(int cp, char *out) {
+ if (cp < 0x80) {
+ out[0] = (char)cp;
+ out[1] = '\0';
+ return 1;
+ }
out[0] = (char)(0xC0 | (cp >> 6));
out[1] = (char)(0x80 | (cp & 0x3F));
out[2] = '\0';
@@ -36,11 +37,11 @@ static int tf_cp_to_utf8(int cp, char *out)
}
/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */
-static const char *tf_json_escape(const char *s, char *buf, size_t cap)
-{
+static const char *tf_json_escape(const char *s, char *buf, size_t cap) {
size_t o = 0;
for (; *s && o + 2 < cap; s++) {
- if (*s == '"' || *s == '\\') buf[o++] = '\\';
+ if (*s == '"' || *s == '\\')
+ buf[o++] = '\\';
buf[o++] = *s;
}
buf[o] = '\0';
@@ -52,8 +53,7 @@ static const char *TF_MERGED[] = {
"he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor",
};
static const char *TF_MERGES[] = {
- "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d",
- "\xC4\xA0w or",
+ "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", "\xC4\xA0w or",
};
enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
/* Special tokens go last so contextual and late-interaction ids resolve
@@ -65,17 +65,16 @@ enum { TF_DOCUMENT_ID = TF_QUERY_ID + 1 };
enum { TF_VOCAB_SIZE = TF_DOCUMENT_ID + 1 };
/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
-static int tf_write_vocab(const char *dir)
-{
+static int tf_write_vocab(const char *dir) {
char path[2048], tok[4], esc[16];
snprintf(path, sizeof(path), "%s/vocab.json", dir);
FILE *f = fopen(path, "w");
- if (!f) return -1;
+ if (!f)
+ return -1;
fputs("{", f);
for (int b = 0; b < 256; b++) {
tf_cp_to_utf8(tf_byte_to_cp(b), tok);
- fprintf(f, "%s\"%s\": %d", b ? ", " : "",
- tf_json_escape(tok, esc, sizeof(esc)), b);
+ fprintf(f, "%s\"%s\": %d", b ? ", " : "", tf_json_escape(tok, esc, sizeof(esc)), b);
}
for (int i = 0; i < TF_N_MERGED; i++)
fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
@@ -88,7 +87,8 @@ static int tf_write_vocab(const char *dir)
snprintf(path, sizeof(path), "%s/merges.txt", dir);
f = fopen(path, "w");
- if (!f) return -1;
+ if (!f)
+ return -1;
for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++)
fprintf(f, "%s\n", TF_MERGES[i]);
fclose(f);