From c4a1c97a09da6d8d88c2ac834bf6234506fc41bc Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Mon, 15 Jun 2026 08:40:11 +0200 Subject: chore: run clang-format --- bench/bench.h | 74 ++- bench/bench_kernels.c | 115 ++-- bench/bench_model.c | 24 +- bench/bench_tokenizer.c | 63 +-- embed.c | 1056 ++++++++++++++++++------------------- embed.h | 108 ++-- embed_cli.c | 388 +++++++------- embed_cuda.cu | 874 +++++++++++++++---------------- embed_cuda.h | 20 +- embed_internal.h | 54 +- embed_mlx.c | 1159 ++++++++++++++++++++++------------------- embed_mlx.h | 69 ++- embed_server.c | 1300 +++++++++++++++++++++++----------------------- embed_server.h | 2 +- qwen_kernels.c | 534 +++++++++++-------- qwen_kernels.h | 111 ++-- qwen_kernels_avx.c | 259 +++++---- qwen_kernels_generic.c | 16 +- qwen_kernels_impl.h | 30 +- qwen_kernels_neon.c | 16 +- qwen_safetensors.c | 285 ++++++---- qwen_safetensors.h | 9 +- qwen_tokenizer.c | 998 ++++++++++++++++++++++------------- qwen_tokenizer.h | 22 +- tests/test_bf16_model.c | 20 +- tests/test_cli.c | 76 ++- tests/test_kernels.c | 119 ++--- tests/test_late.c | 85 ++- tests/test_qwen3.c | 25 +- tests/test_safetensors.c | 67 +-- tests/test_server.c | 438 ++++++++-------- tests/test_tokenizer.c | 159 +++--- tests/test_workspace.c | 78 ++- tests/tiny_model.h | 97 ++-- tests/tok_fixture.h | 36 +- 35 files changed, 4752 insertions(+), 4034 deletions(-) diff --git a/bench/bench.h b/bench/bench.h index 9a46c71..6089174 100644 --- a/bench/bench.h +++ b/bench/bench.h @@ -30,8 +30,8 @@ #include typedef struct { - long n; /* iterations: the measured loop must run exactly n times */ - double t0_ns; /* set by bench_begin(); end of un-measured setup */ + long n; /* iterations: the measured loop must run exactly n times */ + double t0_ns; /* set by bench_begin(); end of un-measured setup */ } bench_state_t; typedef void (*bench_fn_t)(bench_state_t *); @@ -45,36 +45,29 @@ typedef struct { * compiler cannot delete the measured loop. */ static volatile float bench_sink; -static double bench_now_ns(void) -{ +static double bench_now_ns(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec * 1e9 + (double)ts.tv_nsec; } -static void bench_begin(bench_state_t *b) -{ - b->t0_ns = bench_now_ns(); -} +static void bench_begin(bench_state_t *b) { b->t0_ns = bench_now_ns(); } /* One timed run of fn at n iterations; returns ns/op. */ -static double bench_run_once(bench_fn_t fn, long n) -{ - bench_state_t b = { n, 0.0 }; +static double bench_run_once(bench_fn_t fn, long n) { + bench_state_t b = {n, 0.0}; b.t0_ns = bench_now_ns(); fn(&b); double elapsed = bench_now_ns() - b.t0_ns; return elapsed / (double)n; } -static int bench_cmp_double(const void *a, const void *c) -{ +static int bench_cmp_double(const void *a, const void *c) { double x = *(const double *)a, y = *(const double *)c; return (x > y) - (x < y); } -static double bench_median(double *v, int n) -{ +static double bench_median(double *v, int n) { qsort(v, (size_t)n, sizeof(*v), bench_cmp_double); return n % 2 ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); } @@ -88,13 +81,13 @@ typedef struct { const char *meta; } bench_opts_t; -static void bench_parse_args(bench_opts_t *o, int argc, char **argv) -{ +static void bench_parse_args(bench_opts_t *o, int argc, char **argv) { o->benchtime_ms = 300; o->count = 10; o->filter = NULL; o->json_path = NULL; - if (!o->meta) o->meta = ""; + if (!o->meta) + o->meta = ""; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--benchtime-ms") && i + 1 < argc) o->benchtime_ms = atol(argv[++i]); @@ -105,15 +98,16 @@ static void bench_parse_args(bench_opts_t *o, int argc, char **argv) else if (!strcmp(argv[i], "--json") && i + 1 < argc) o->json_path = argv[++i]; } - if (o->benchtime_ms < 1) o->benchtime_ms = 1; - if (o->count < 1) o->count = 1; - if (o->count > 1000) o->count = 1000; + if (o->benchtime_ms < 1) + o->benchtime_ms = 1; + if (o->count < 1) + o->count = 1; + if (o->count > 1000) + o->count = 1000; } /* Run all cases (subject to the filter) and report. Returns 0 on success. */ -static int bench_main(const bench_opts_t *o, - const bench_case_t *cases, int n_cases) -{ +static int bench_main(const bench_opts_t *o, const bench_case_t *cases, int n_cases) { FILE *jf = NULL; if (o->json_path) { jf = fopen(o->json_path, "w"); @@ -123,14 +117,16 @@ static int bench_main(const bench_opts_t *o, } char host[256] = "unknown"; gethostname(host, sizeof(host) - 1); - fprintf(jf, "{\"host\":\"%s\",\"benchtime_ms\":%ld,\"count\":%d," + fprintf(jf, + "{\"host\":\"%s\",\"benchtime_ms\":%ld,\"count\":%d," "\"meta\":\"%s\",\"benchmarks\":[", host, o->benchtime_ms, o->count, o->meta); } double *samples = (double *)malloc((size_t)o->count * sizeof(double)); if (!samples) { - if (jf) fclose(jf); + if (jf) + fclose(jf); return 1; } @@ -140,7 +136,8 @@ static int bench_main(const bench_opts_t *o, free(all); free(iters); free(samples); - if (jf) fclose(jf); + if (jf) + fclose(jf); return 1; } @@ -156,8 +153,10 @@ static int bench_main(const bench_opts_t *o, while (ns_per_op * (double)n < target_ns) { double want = target_ns / (ns_per_op > 0.0 ? ns_per_op : 1.0); long next = (long)(want * 1.2) + 1; - if (next > n * 100) next = n * 100; /* distrust tiny timings */ - if (next <= n) next = n + 1; + if (next > n * 100) + next = n * 100; /* distrust tiny timings */ + if (next <= n) + next = n + 1; n = next; ns_per_op = bench_run_once(cases[c].fn, n); } @@ -174,8 +173,7 @@ static int bench_main(const bench_opts_t *o, for (int s = 0; s < o->count; s++) { for (int c = 0; c < n_cases; c++) if (iters[c]) - all[(size_t)c * o->count + s] = - bench_run_once(cases[c].fn, iters[c]); + all[(size_t)c * o->count + s] = bench_run_once(cases[c].fn, iters[c]); fprintf(stderr, "pass %d/%d (%.0fs elapsed)\n", s + 1, o->count, (bench_now_ns() - sample_t0) / 1e9); } @@ -184,22 +182,20 @@ static int bench_main(const bench_opts_t *o, for (int c = 0; c < n_cases; c++) { if (!iters[c]) continue; - memcpy(samples, all + (size_t)c * o->count, - (size_t)o->count * sizeof(double)); + memcpy(samples, all + (size_t)c * o->count, (size_t)o->count * sizeof(double)); if (jf) { - fprintf(jf, "%s{\"name\":\"%s\",\"iters\":%ld,\"ns_per_op\":[", - emitted ? "," : "", cases[c].name, iters[c]); + fprintf(jf, "%s{\"name\":\"%s\",\"iters\":%ld,\"ns_per_op\":[", emitted ? "," : "", + cases[c].name, iters[c]); for (int s = 0; s < o->count; s++) fprintf(jf, "%s%.3f", s ? "," : "", samples[s]); fprintf(jf, "]}"); } double med = bench_median(samples, o->count); - double spread = o->count > 1 - ? 100.0 * (samples[o->count - 1] - samples[0]) / med : 0.0; - printf("%-44s %10ld iters %14.1f ns/op (spread %.1f%%)\n", - cases[c].name, iters[c], med, spread); + double spread = o->count > 1 ? 100.0 * (samples[o->count - 1] - samples[0]) / med : 0.0; + printf("%-44s %10ld iters %14.1f ns/op (spread %.1f%%)\n", cases[c].name, iters[c], med, + spread); fflush(stdout); emitted++; } diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c index cc5dc54..4639d18 100644 --- a/bench/bench_kernels.c +++ b/bench/bench_kernels.c @@ -12,12 +12,12 @@ #include -int qwen_verbose = 0; /* defined by embed.c in full builds */ +int qwen_verbose = 0; /* defined by embed.c in full builds */ enum { HIDDEN = 1024, - Q_DIM = 2048, /* 16 heads * 128 */ - KV_DIM = 1024, /* 8 heads * 128 */ + Q_DIM = 2048, /* 16 heads * 128 */ + KV_DIM = 1024, /* 8 heads * 128 */ HEAD_DIM = 128, INTER = 3072, SEQ = 8, @@ -25,8 +25,7 @@ enum { }; /* Deterministic fill in [-1, 1]; small values keep softmax/silu sane. */ -static void fill_f32(float *x, size_t n, unsigned seed) -{ +static void fill_f32(float *x, size_t n, unsigned seed) { unsigned s = seed * 2654435761u + 1u; for (size_t i = 0; i < n; i++) { s = s * 1664525u + 1013904223u; @@ -34,15 +33,13 @@ static void fill_f32(float *x, size_t n, unsigned seed) } } -static uint16_t f32_to_bf16(float x) -{ +static uint16_t f32_to_bf16(float x) { uint32_t u; memcpy(&u, &x, sizeof(u)); return (uint16_t)(u >> 16); } -static void fill_bf16(uint16_t *x, size_t n, unsigned seed) -{ +static void fill_bf16(uint16_t *x, size_t n, unsigned seed) { unsigned s = seed * 2654435761u + 1u; for (size_t i = 0; i < n; i++) { s = s * 1664525u + 1013904223u; @@ -50,26 +47,25 @@ static void fill_bf16(uint16_t *x, size_t n, unsigned seed) } } -static float *alloc_f32(size_t n, unsigned seed) -{ +static float *alloc_f32(size_t n, unsigned seed) { float *x = (float *)malloc(n * sizeof(float)); - if (!x) exit(2); + if (!x) + exit(2); fill_f32(x, n, seed); return x; } -static uint16_t *alloc_bf16(size_t n, unsigned seed) -{ +static uint16_t *alloc_bf16(size_t n, unsigned seed) { uint16_t *x = (uint16_t *)malloc(n * sizeof(uint16_t)); - if (!x) exit(2); + if (!x) + exit(2); fill_bf16(x, n, seed); return x; } /* ---- F32 projections (matvec at seq 1, GEMM at seq 8) ---- */ -static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) -{ +static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) { float *x = alloc_f32((size_t)seq * in, 1); float *w = alloc_f32((size_t)out * in, 2); float *y = alloc_f32((size_t)seq * out, 3); @@ -77,18 +73,19 @@ static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) for (long i = 0; i < b->n; i++) qwen_linear_nobias(y, x, w, seq, in, out); bench_sink += y[0]; - free(y); free(w); free(x); + free(y); + free(w); + free(x); } -static void bm_linear_f32_s1_qproj(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, Q_DIM); } -static void bm_linear_f32_s1_gate(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, INTER); } -static void bm_linear_f32_s1_down(bench_state_t *b) { bm_linear_f32(b, 1, INTER, HIDDEN); } -static void bm_linear_f32_s8_gate(bench_state_t *b) { bm_linear_f32(b, SEQ, HIDDEN, INTER); } +static void bm_linear_f32_s1_qproj(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, Q_DIM); } +static void bm_linear_f32_s1_gate(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, INTER); } +static void bm_linear_f32_s1_down(bench_state_t *b) { bm_linear_f32(b, 1, INTER, HIDDEN); } +static void bm_linear_f32_s8_gate(bench_state_t *b) { bm_linear_f32(b, SEQ, HIDDEN, INTER); } /* ---- BF16 weights ---- */ -static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) -{ +static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) { float *x = alloc_f32((size_t)seq * in, 4); uint16_t *w = alloc_bf16((size_t)out * in, 5); float *y = alloc_f32((size_t)seq * out, 6); @@ -96,14 +93,15 @@ static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) for (long i = 0; i < b->n; i++) qwen_linear_nobias_bf16(y, x, w, seq, in, out); bench_sink += y[0]; - free(y); free(w); free(x); + free(y); + free(w); + free(x); } static void bm_linear_bf16_s1_gate(bench_state_t *b) { bm_linear_bf16(b, 1, HIDDEN, INTER); } static void bm_linear_bf16_s8_gate(bench_state_t *b) { bm_linear_bf16(b, SEQ, HIDDEN, INTER); } -static void bm_qkv_bf16(bench_state_t *b) -{ +static void bm_qkv_bf16(bench_state_t *b) { float *x = alloc_f32((size_t)SEQ * HIDDEN, 7); uint16_t *wq = alloc_bf16((size_t)Q_DIM * HIDDEN, 8); uint16_t *wk = alloc_bf16((size_t)KV_DIM * HIDDEN, 9); @@ -113,14 +111,18 @@ static void bm_qkv_bf16(bench_state_t *b) float *v = alloc_f32((size_t)SEQ * KV_DIM, 13); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, - SEQ, HIDDEN, Q_DIM, KV_DIM); + qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM); bench_sink += q[0] + k[0] + v[0]; - free(v); free(k); free(q); free(wv); free(wk); free(wq); free(x); + free(v); + free(k); + free(q); + free(wv); + free(wk); + free(wq); + free(x); } -static void bm_bf16_widen(bench_state_t *b) -{ +static void bm_bf16_widen(bench_state_t *b) { size_t n = (size_t)INTER * HIDDEN; uint16_t *src = alloc_bf16(n, 14); float *dst = alloc_f32(n, 15); @@ -128,13 +130,13 @@ static void bm_bf16_widen(bench_state_t *b) for (long i = 0; i < b->n; i++) qwen_bf16_to_f32_buf(dst, src, n); bench_sink += dst[0]; - free(dst); free(src); + free(dst); + free(src); } /* ---- attention pieces ---- */ -static void bm_attn_scores(bench_state_t *b) -{ +static void bm_attn_scores(bench_state_t *b) { /* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */ float *qm = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 16); float *km = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 17); @@ -143,11 +145,12 @@ static void bm_attn_scores(bench_state_t *b) for (long i = 0; i < b->n; i++) qwen_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); bench_sink += s[0]; - free(s); free(km); free(qm); + free(s); + free(km); + free(qm); } -static void bm_softmax(bench_state_t *b) -{ +static void bm_softmax(bench_state_t *b) { float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19); bench_begin(b); for (long i = 0; i < b->n; i++) @@ -156,23 +159,22 @@ static void bm_softmax(bench_state_t *b) free(x); } -static void bm_rope(bench_state_t *b) -{ +static void bm_rope(bench_state_t *b) { float *x = alloc_f32((size_t)ATTN_SEQ * Q_DIM, 20); float *cosv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 21); float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, - HEAD_DIM); + qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM); bench_sink += x[0]; - free(sinv); free(cosv); free(x); + free(sinv); + free(cosv); + free(x); } /* ---- norms, activations, reductions ---- */ -static void bm_rms_norm(bench_state_t *b) -{ +static void bm_rms_norm(bench_state_t *b) { float *x = alloc_f32((size_t)SEQ * HIDDEN, 23); float *w = alloc_f32(HIDDEN, 24); float *y = alloc_f32((size_t)SEQ * HIDDEN, 25); @@ -180,13 +182,14 @@ static void bm_rms_norm(bench_state_t *b) for (long i = 0; i < b->n; i++) qwen_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); bench_sink += y[0]; - free(y); free(w); free(x); + free(y); + free(w); + free(x); } /* The engine's SwiGLU op mutates gate in place, so each iteration restores * it first; the memcpy is a constant part of the measured loop. */ -static void bm_silu_mul(bench_state_t *b) -{ +static void bm_silu_mul(bench_state_t *b) { size_t n = (size_t)SEQ * INTER; float *gate0 = alloc_f32(n, 26); float *gate = alloc_f32(n, 26); @@ -197,17 +200,19 @@ static void bm_silu_mul(bench_state_t *b) qwen_silu_mul_inplace(gate, up, (int)n); } bench_sink += gate[0]; - free(up); free(gate); free(gate0); + free(up); + free(gate); + free(gate0); } -static void bm_dot(bench_state_t *b) -{ +static void bm_dot(bench_state_t *b) { float *x = alloc_f32(HIDDEN, 28); float *y = alloc_f32(HIDDEN, 29); bench_begin(b); for (long i = 0; i < b->n; i++) bench_sink += qwen_dot_f32(x, y, HIDDEN); - free(y); free(x); + free(y); + free(x); } static const bench_case_t CASES[] = { @@ -227,14 +232,14 @@ static const bench_case_t CASES[] = { {"reduce/dot_1024", bm_dot}, }; -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { int threads = 1; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--threads") && i + 1 < argc) threads = atoi(argv[i + 1]); } - if (threads < 1) threads = 1; + if (threads < 1) + threads = 1; qwen_set_threads(threads); char meta[128]; diff --git a/bench/bench_model.c b/bench/bench_model.c index 9b897e9..5cff015 100644 --- a/bench/bench_model.c +++ b/bench/bench_model.c @@ -19,15 +19,15 @@ static int g_hidden; enum { MAX_BATCH = 8 }; -static void bm_embed(bench_state_t *b, int batch, int len) -{ +static void bm_embed(bench_state_t *b, int batch, int len) { int *ids = (int *)malloc((size_t)batch * len * sizeof(int)); float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float)); - if (!ids || !out) exit(2); + if (!ids || !out) + exit(2); unsigned s = 99u; for (int i = 0; i < batch * len; i++) { s = s * 1664525u + 1013904223u; - ids[i] = (int)(s % 1000u); /* ordinary vocab rows */ + ids[i] = (int)(s % 1000u); /* ordinary vocab rows */ } embed_input_t inputs[MAX_BATCH]; for (int i = 0; i < batch; i++) { @@ -44,8 +44,8 @@ static void bm_embed(bench_state_t *b, int batch, int len) free(ids); } -static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); } -static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); } +static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); } +static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); } static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); } static const bench_case_t CASES[] = { @@ -54,11 +54,9 @@ static const bench_case_t CASES[] = { {"embed/b1_len256", bm_embed_b1_len256}, }; -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { if (argc < 2 || argv[1][0] == '-') { - fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n", - argv[0]); + fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n", argv[0]); return 2; } @@ -67,7 +65,8 @@ int main(int argc, char **argv) if (!strcmp(argv[i], "--threads") && i + 1 < argc) threads = atoi(argv[i + 1]); } - if (threads < 1) threads = 1; + if (threads < 1) + threads = 1; qwen_set_threads(threads); g_model = embed_model_load(argv[1]); @@ -83,8 +82,7 @@ int main(int argc, char **argv) g_hidden = embed_model_config(g_model)->hidden_size; char meta[160]; - snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", - threads, g_hidden); + snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", threads, g_hidden); bench_opts_t opts = {0}; opts.meta = meta; diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index 1033f2e..fb6d37a 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -10,24 +10,23 @@ int qwen_verbose = 0; -static double now_ms(void) -{ +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; } -static int same_ids(const int *a, int na, const int *b, int nb) -{ - if (na != nb) return 0; +static int same_ids(const int *a, int na, const int *b, int nb) { + if (na != nb) + return 0; for (int i = 0; i < na; i++) { - if (a[i] != b[i]) return 0; + if (a[i] != b[i]) + return 0; } return 1; } -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { int status = 1; int **gold = NULL; int *gold_n = NULL; @@ -41,7 +40,8 @@ int main(int argc, char **argv) const char *vocab = argv[1]; int runs = atoi(argv[2]); - if (runs <= 0) return 2; + if (runs <= 0) + return 2; int n_texts = argc - 3; char **texts = argv + 3; @@ -56,17 +56,18 @@ int main(int argc, char **argv) gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); - if (!gold || !gold_n || !bufs || !caps) goto cleanup; + if (!gold || !gold_n || !bufs || !caps) + goto cleanup; int tokens_per_pass = 0; for (int i = 0; i < n_texts; i++) { gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); - if (!gold[i] || gold_n[i] <= 0) goto cleanup; + if (!gold[i] || gold_n[i] <= 0) + goto cleanup; tokens_per_pass += gold_n[i]; int n_ws = 0; - int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], - &n_ws); + int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { free(ids_ws); goto cleanup; @@ -74,12 +75,13 @@ int main(int argc, char **argv) free(ids_ws); caps[i] = (int)(strlen(texts[i]) * 2 + 8); - if (caps[i] < gold_n[i]) caps[i] = gold_n[i]; + if (caps[i] < gold_n[i]) + caps[i] = gold_n[i]; bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int)); - if (!bufs[i]) goto cleanup; + if (!bufs[i]) + goto cleanup; int n_into = 0; - if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], - &n_into) != 0 || + if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || !same_ids(gold[i], gold_n[i], bufs[i], n_into)) goto cleanup; } @@ -105,8 +107,7 @@ int main(int argc, char **argv) for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], - &n); + int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); if (!ids || n != gold_n[i]) { free(ids); goto cleanup; @@ -122,30 +123,30 @@ int main(int argc, char **argv) for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], - caps[i], &n); - if (rc != 0 || n != gold_n[i]) goto cleanup; + int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); + if (rc != 0 || n != gold_n[i]) + goto cleanup; into_tokens += n; } } double into_ms = now_ms() - t0; - printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, - tokens_per_pass); + printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, tokens_per_pass); printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms, (double)old_tokens / (old_ms / 1000.0)); - printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", - ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); - printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", - into_ms, (double)into_tokens / (into_ms / 1000.0), - old_ms / into_ms); + printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", ws_ms, + (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); + printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", into_ms, + (double)into_tokens / (into_ms / 1000.0), old_ms / into_ms); status = 0; cleanup: for (int i = 0; i < n_texts; i++) { - if (gold) free(gold[i]); - if (bufs) free(bufs[i]); + if (gold) + free(gold[i]); + if (bufs) + free(bufs[i]); } free(gold); free(gold_n); diff --git a/embed.c b/embed.c index bd72846..d9e7936 100644 --- a/embed.c +++ b/embed.c @@ -31,32 +31,32 @@ struct embed_late_workspace { int embed_verbose = 0; int qwen_verbose = 0; -#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 -#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 -#define EMBED_MIN_WORKSPACE_SEQ_CAP 16 +#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 +#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 +#define EMBED_MIN_WORKSPACE_SEQ_CAP 16 #define EMBED_WORKSPACE_SEQ_GRANULARITY 16 /* ======================================================================== * Size helpers * ======================================================================== */ -static int mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; +static int mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; *out = a * b; return 0; } -static int add_size(size_t a, size_t b, size_t *out) -{ - if (b > SIZE_MAX - a) return -1; +static int add_size(size_t a, size_t b, size_t *out) { + if (b > SIZE_MAX - a) + return -1; *out = a + b; return 0; } -static int grow_cap(int current, int needed, int *out) -{ - if (needed <= 0) return -1; +static int grow_cap(int current, int needed, int *out) { + if (needed <= 0) + return -1; int cap = needed; if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP) @@ -65,7 +65,8 @@ static int grow_cap(int current, int needed, int *out) int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY; if (rem != 0) { int add = EMBED_WORKSPACE_SEQ_GRANULARITY - rem; - if (cap > INT_MAX - add) return -1; + if (cap > INT_MAX - add) + return -1; cap += add; } @@ -76,54 +77,57 @@ static int grow_cap(int current, int needed, int *out) return 0; } -static int realloc_floats(float **ptr, size_t count) -{ +static int realloc_floats(float **ptr, size_t count) { size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) return -1; + if (mul_size(count, sizeof(float), &bytes) != 0) + return -1; void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) return -1; + if (!p && bytes != 0) + return -1; *ptr = (float *)p; return 0; } -static float *malloc_floats(size_t count) -{ +static float *malloc_floats(size_t count) { size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) return NULL; + if (mul_size(count, sizeof(float), &bytes) != 0) + return NULL; return (float *)malloc(bytes); } -static int model_dir_has_file(const char *model_dir, const char *rel) -{ +static int model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; FILE *f = fopen(path, "rb"); - if (!f) return 0; + if (!f) + return 0; fclose(f); return 1; } -static int model_dir_has_late_projection(const char *model_dir) -{ +static int model_dir_has_late_projection(const char *model_dir) { return model_dir_has_file(model_dir, "1_Dense/config.json") && model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -static int realloc_floats_2d(float **ptr, int rows, int cols) -{ - if (rows < 0 || cols < 0) return -1; +static int realloc_floats_2d(float **ptr, int rows, int cols) { + if (rows < 0 || cols < 0) + return -1; size_t count; - if (mul_size((size_t)rows, (size_t)cols, &count) != 0) return -1; + if (mul_size((size_t)rows, (size_t)cols, &count) != 0) + return -1; return realloc_floats(ptr, count); } -static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) -{ - if (!ws || !ws->model || n_tokens <= 0) return -1; - if (n_tokens <= ws->states_seq_cap) return 0; +static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) { + if (!ws || !ws->model || n_tokens <= 0) + return -1; + if (n_tokens <= ws->states_seq_cap) + return 0; int hidden = ws->model->base->config.hidden_size; if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) return -1; @@ -131,13 +135,14 @@ static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) return 0; } -static int realloc_ints(int **ptr, size_t count) -{ +static int realloc_ints(int **ptr, size_t count) { size_t bytes; - if (mul_size(count, sizeof(int), &bytes) != 0) return -1; + if (mul_size(count, sizeof(int), &bytes) != 0) + return -1; void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) return -1; + if (!p && bytes != 0) + return -1; *ptr = (int *)p; return 0; } @@ -146,15 +151,14 @@ static int realloc_ints(int **ptr, size_t count) * Minimal JSON helpers for config.json parsing * ======================================================================== */ -static const char *skip_ws(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; +static const char *skip_ws(const char *p) { + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + p++; return p; } /* Find "key": in a flat JSON object. Returns pointer to value start. */ -static const char *json_find_key(const char *json, const char *key) -{ +static const char *json_find_key(const char *json, const char *key) { size_t klen = strlen(key); const char *p = json; while ((p = strstr(p, key)) != NULL) { @@ -163,7 +167,8 @@ static const char *json_find_key(const char *json, const char *key) const char *after = p + klen; if (*after == '"') { after = skip_ws(after + 1); - if (*after == ':') return skip_ws(after + 1); + if (*after == ':') + return skip_ws(after + 1); } } p += klen; @@ -171,32 +176,30 @@ static const char *json_find_key(const char *json, const char *key) return NULL; } -static int json_get_int(const char *json, const char *key, int fallback) -{ +static int json_get_int(const char *json, const char *key, int fallback) { const char *v = json_find_key(json, key); - if (!v) return fallback; + if (!v) + return fallback; return atoi(v); } -static double json_get_double(const char *json, const char *key, double fallback) -{ +static double json_get_double(const char *json, const char *key, double fallback) { const char *v = json_find_key(json, key); - if (!v) return fallback; + if (!v) + return fallback; return atof(v); } -static int json_string_equals(const char *json, const char *key, - const char *expected) -{ +static int json_string_equals(const char *json, const char *key, const char *expected) { const char *v = json_find_key(json, key); - if (!v || *v != '"') return 0; + if (!v || *v != '"') + return 0; v++; size_t len = strlen(expected); return strncmp(v, expected, len) == 0 && v[len] == '"'; } -static int parse_config(embed_config_t *cfg, const char *model_dir) -{ +static int parse_config(embed_config_t *cfg, const char *model_dir) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); if (n < 0 || (size_t)n >= sizeof(path)) { @@ -221,22 +224,24 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) } char *buf = (char *)malloc((size_t)sz + 1); if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; + fclose(f); + free(buf); + return -1; } fclose(f); buf[sz] = '\0'; - cfg->hidden_size = json_get_int(buf, "hidden_size", 0); - cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); - cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); - cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); - cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM); + cfg->hidden_size = json_get_int(buf, "hidden_size", 0); + cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); + cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); + cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM); cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); - cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); - cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); - cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); - cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; - cfg->pooling_mode = EMBED_POOL_MEAN; + cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); + cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); + cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->pooling_mode = EMBED_POOL_MEAN; cfg->normalize_embeddings = 0; cfg->append_terminal_token = 0; @@ -249,45 +254,40 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) cfg->q_dim = 0; cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim) cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim) cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; free(buf); /* Sanity checks */ - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "embed_model_load: invalid config in %s " + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || + cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || + cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || + cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || + cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + fprintf(stderr, + "embed_model_load: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim, + cfg->intermediate_size); return -1; } if (cfg->n_layers > EMBED_MAX_LAYERS) { - fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n", - cfg->n_layers, EMBED_MAX_LAYERS); + fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n", cfg->n_layers, + EMBED_MAX_LAYERS); return -1; } if (embed_verbose >= 1) - fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " + fprintf(stderr, + "config: hidden=%d, layers=%d, heads=%d/%d, " "inter=%d, head_dim=%d, attention=%s, pooling=%s\n", - cfg->hidden_size, cfg->n_layers, - cfg->n_heads, cfg->n_kv_heads, + cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->intermediate_size, cfg->head_dim, - cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? "causal" : "bidirectional", - cfg->pooling_mode == EMBED_POOL_LAST_TOKEN - ? "last-token" : "mean"); + cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional", + cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean"); return 0; } @@ -296,48 +296,57 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) * Weight loading (direct mmap pointers into safetensors) * ======================================================================== */ -static size_t dtype_size(safetensor_dtype_t dtype) -{ +static size_t dtype_size(safetensor_dtype_t dtype) { switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; + case DTYPE_F32: + return sizeof(float); + case DTYPE_BF16: + return sizeof(uint16_t); + default: + return 0; } } -static const char *dtype_name(safetensor_dtype_t dtype) -{ +static const char *dtype_name(safetensor_dtype_t dtype) { switch (dtype) { - case DTYPE_F32: return "F32"; - case DTYPE_F16: return "F16"; - case DTYPE_BF16: return "BF16"; - case DTYPE_I32: return "I32"; - case DTYPE_I64: return "I64"; - case DTYPE_BOOL: return "BOOL"; - default: return "UNKNOWN"; + case DTYPE_F32: + return "F32"; + case DTYPE_F16: + return "F16"; + case DTYPE_BF16: + return "BF16"; + case DTYPE_I32: + return "I32"; + case DTYPE_I64: + return "I64"; + case DTYPE_BOOL: + return "BOOL"; + default: + return "UNKNOWN"; } } static int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, const char *name, - const int64_t *shape, int ndim, - int bf16_ok) -{ + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim, + int bf16_ok) { if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", - name, dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); + fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, + dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); return 0; } if (t->ndim != ndim) { - fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); + fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } size_t numel = 1; for (int i = 0; i < ndim; i++) { if (t->shape[i] != shape[i]) { - fprintf(stderr, "embed: bad shape for %s at dim %d: " + fprintf(stderr, + "embed: bad shape for %s at dim %d: " "got %lld, expected %lld\n", name, i, (long long)t->shape[i], (long long)shape[i]); return 0; @@ -351,8 +360,7 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf, size_t elem_size = dtype_size(t->dtype); if (elem_size == 0) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", - name, dtype_name(t->dtype)); + fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); return 0; } @@ -362,14 +370,13 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf, return 0; } if (t->data_size != bytes) { - fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); + fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + bytes); return 0; } size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { fprintf(stderr, "embed: tensor data out of bounds for %s\n", name); return 0; @@ -377,13 +384,11 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms, - const char *name, - const int64_t *shape, int ndim) -{ - embed_weight_ref_t ref = { DTYPE_UNKNOWN, NULL }; +static embed_weight_ref_t +load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { + embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL}; safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { fprintf(stderr, "embed: tensor not found: %s\n", name); return ref; @@ -396,11 +401,10 @@ static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms, return ref; } -static float *load_norm_f32(multi_safetensors_t *ms, const char *name, - const int64_t *shape, int ndim) -{ +static float * +load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { fprintf(stderr, "embed: tensor not found: %s\n", name); return NULL; @@ -410,17 +414,14 @@ static float *load_norm_f32(multi_safetensors_t *ms, const char *name, return safetensors_get_f32(sf, t); } -static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) -{ +static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { for (int i = 0; i < n; i++) { uint32_t u = ((uint32_t)src[i]) << 16; memcpy(dst + i, &u, sizeof(float)); } } -static void copy_weight_row(float *dst, const embed_weight_ref_t *w, - size_t row, int row_width) -{ +static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) { if (w->dtype == DTYPE_F32) { const float *src = (const float *)w->data + row * (size_t)row_width; memcpy(dst, src, (size_t)row_width * sizeof(float)); @@ -430,23 +431,26 @@ static void copy_weight_row(float *dst, const embed_weight_ref_t *w, } } -static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) -{ - if (ws->bf16_widen_count >= count) return 0; +static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) { + if (ws->bf16_widen_count >= count) + return 0; float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float)); - if (!p) return -1; + if (!p) + return -1; ws->bf16_widen = p; ws->bf16_widen_count = count; return 0; } -static void linear_nobias_weight(embed_workspace_t *ws, float *y, - const float *x, const embed_weight_ref_t *w, - int seq_len, int in_dim, int out_dim) -{ +static void linear_nobias_weight(embed_workspace_t *ws, + float *y, + const float *x, + const embed_weight_ref_t *w, + int seq_len, + int in_dim, + int out_dim) { if (w->dtype == DTYPE_F32) { - qwen_linear_nobias(y, x, (const float *)w->data, - seq_len, in_dim, out_dim); + qwen_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); return; } if (seq_len > 16) { @@ -454,31 +458,31 @@ static void linear_nobias_weight(embed_workspace_t *ws, float *y, * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ size_t count = (size_t)out_dim * (size_t)in_dim; if (ensure_bf16_widen(ws, count) == 0) { - qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, - count); + qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); return; } } - qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, - seq_len, in_dim, out_dim); + qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); } -static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k, - float *v, const float *x, +static void linear_qkv_weight(embed_workspace_t *ws, + float *q, + float *k, + float *v, + const float *x, const embed_weight_ref_t *wq, const embed_weight_ref_t *wk, const embed_weight_ref_t *wv, - int seq_len, int in_dim, int q_dim, int kv_dim) -{ - if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && - wv->dtype == DTYPE_BF16 && + int seq_len, + int in_dim, + int q_dim, + int kv_dim) { + if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_qkv(q, k, v, x, - (const uint16_t *)wq->data, - (const uint16_t *)wk->data, - (const uint16_t *)wv->data, - seq_len, in_dim, q_dim, kv_dim); + qwen_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, + (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len, + in_dim, q_dim, kv_dim); return; } @@ -487,18 +491,20 @@ static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k, linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim); } -static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b, +static void linear_pair_weight(embed_workspace_t *ws, + float *a, + float *b, const float *x, const embed_weight_ref_t *wa, const embed_weight_ref_t *wb, - int seq_len, int in_dim, int a_dim, int b_dim) -{ + int seq_len, + int in_dim, + int a_dim, + int b_dim) { if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_pair(a, b, x, - (const uint16_t *)wa->data, - (const uint16_t *)wb->data, - seq_len, in_dim, a_dim, b_dim); + qwen_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, + (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); return; } @@ -510,26 +516,29 @@ static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b, * Working buffer management * ======================================================================== */ -static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) -{ - if (seq <= ws->buf_seq_cap) return 0; +static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) { + if (seq <= ws->buf_seq_cap) + return 0; int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) return -1; - -#define R(ptr, n) do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) return -1; \ -} while (0) + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) + return -1; - R(ws->x, c->hidden_size); - R(ws->x_norm, c->hidden_size); - R(ws->q, c->q_dim); - R(ws->k, c->kv_dim); - R(ws->v, c->kv_dim); +#define R(ptr, n) \ + do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ + return -1; \ + } while (0) + + R(ws->x, c->hidden_size); + R(ws->x_norm, c->hidden_size); + R(ws->q, c->q_dim); + R(ws->k, c->kv_dim); + R(ws->v, c->kv_dim); R(ws->attn_out, c->q_dim); R(ws->proj_out, c->hidden_size); R(ws->ffn_gate, c->intermediate_size); - R(ws->ffn_up, c->intermediate_size); + R(ws->ffn_up, c->intermediate_size); #undef R @@ -537,15 +546,14 @@ static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int se return 0; } -static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, - int batch) -{ - size_t bytes = - qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) return; +static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { + size_t bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) + return; void *p = realloc(ws->attn_scores, bytes); - if (!p) return; + if (!p) + return; ws->attn_scores = (float *)p; ws->attn_scores_bytes = bytes; } @@ -554,20 +562,20 @@ static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, * RoPE cache * ======================================================================== */ -static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, - int n_pos) -{ - if (n_pos <= ws->rope_cache_cap) return 0; +static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) { + if (n_pos <= ws->rope_cache_cap) + return 0; int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; while (cap < n_pos) { - if (cap > INT_MAX / 2) return -1; + if (cap > INT_MAX / 2) + return -1; cap *= 2; } - int head_dim = cfg->head_dim; - int half = head_dim / 2; - float theta = cfg->rope_theta; + int head_dim = cfg->head_dim; + int half = head_dim / 2; + float theta = cfg->rope_theta; size_t count; if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) @@ -585,8 +593,10 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); float angle = fp * inv; float cv = cosf(angle), sv = sinf(angle); - c[d] = cv; c[half + d] = cv; - s[d] = sv; s[half + d] = sv; + c[d] = cv; + c[half + d] = cv; + s[d] = sv; + s[half + d] = sv; } } @@ -594,12 +604,13 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, return 0; } -static int ensure_offsets(embed_workspace_t *ws, int batch) -{ - if (!ws || batch <= 0 || batch == INT_MAX) return -1; +static int ensure_offsets(embed_workspace_t *ws, int batch) { + if (!ws || batch <= 0 || batch == INT_MAX) + return -1; int needed = batch + 1; - if (needed <= ws->offsets_cap) return 0; + if (needed <= ws->offsets_cap) + return 0; if (realloc_ints(&ws->offsets, (size_t)needed) != 0) return -1; @@ -611,19 +622,18 @@ static int ensure_offsets(embed_workspace_t *ws, int batch) * Model / workspace load and free * ======================================================================== */ -static embed_model_t *model_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - int allow_late) -{ +static embed_model_t * +model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { if (!allow_late && model_dir_has_late_projection(model_dir)) { fprintf(stderr, "embed_model_load: late-interaction models require " - "token-level MaxSim support and are not valid pooled " - "embedding models yet\n"); + "token-level MaxSim support and are not valid pooled " + "embedding models yet\n"); return NULL; } embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t)); - if (!model) return NULL; + if (!model) + return NULL; /* Parse config.json */ if (parse_config(&model->config, model_dir) != 0) { @@ -634,19 +644,19 @@ static embed_model_t *model_load_range_ex(const char *model_dir, const embed_config_t *cfg = &model->config; if (layer_end == -1) layer_end = cfg->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > cfg->n_layers) { - fprintf(stderr, "embed_model_load: invalid layer range [%d, %d) " - "for %d-layer model\n", layer_start, layer_end, cfg->n_layers); + if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) { + fprintf(stderr, + "embed_model_load: invalid layer range [%d, %d) " + "for %d-layer model\n", + layer_start, layer_end, cfg->n_layers); free(model); return NULL; } model->layer_start = layer_start; model->layer_end = layer_end; - model->attention = - cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? qwen_causal_gqa_attention_packed_with_scratch - : qwen_bidirectional_gqa_attention_packed_with_scratch; + model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL + ? qwen_causal_gqa_attention_packed_with_scratch + : qwen_bidirectional_gqa_attention_packed_with_scratch; /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); @@ -663,57 +673,53 @@ static embed_model_t *model_load_range_ex(const char *model_dir, /* Only the first stage needs token embeddings. */ embed_weights_t *w = &model->weights; if (layer_start == 0) { - const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size }; - w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", - embed_shape, 2); - if (!w->embed_tokens.data) goto fail; + const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size}; + w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", embed_shape, 2); + if (!w->embed_tokens.data) + goto fail; } /* Allocate layer array */ w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); - if (!w->layers) goto fail; + if (!w->layers) + goto fail; /* Per-layer weights */ char name[256]; for (int i = layer_start; i < layer_end; i++) { embed_layer_t *l = &w->layers[i]; -#define LOAD_NORM(field, fmt, d0) do { \ - const int64_t expect[1] = { (d0) }; \ - snprintf(name, sizeof(name), fmt, i); \ - l->field = load_norm_f32(ms, name, expect, 1); \ - if (!l->field) goto fail; \ -} while (0) - -#define LOAD2(field, fmt, d0, d1) do { \ - const int64_t expect[2] = { (d0), (d1) }; \ - snprintf(name, sizeof(name), fmt, i); \ - l->field = load_weight_direct(ms, name, expect, 2); \ - if (!l->field.data) goto fail; \ -} while (0) - - LOAD2(wq, "layers.%d.self_attn.q_proj.weight", - cfg->q_dim, cfg->hidden_size); - LOAD2(wk, "layers.%d.self_attn.k_proj.weight", - cfg->kv_dim, cfg->hidden_size); - LOAD2(wv, "layers.%d.self_attn.v_proj.weight", - cfg->kv_dim, cfg->hidden_size); - LOAD2(wo, "layers.%d.self_attn.o_proj.weight", - cfg->hidden_size, cfg->q_dim); - LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", - cfg->head_dim); - LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", - cfg->head_dim); - LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", - cfg->hidden_size); - LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", - cfg->hidden_size); - LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", - cfg->intermediate_size, cfg->hidden_size); - LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", - cfg->intermediate_size, cfg->hidden_size); - LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", - cfg->hidden_size, cfg->intermediate_size); +#define LOAD_NORM(field, fmt, d0) \ + do { \ + const int64_t expect[1] = {(d0)}; \ + snprintf(name, sizeof(name), fmt, i); \ + l->field = load_norm_f32(ms, name, expect, 1); \ + if (!l->field) \ + goto fail; \ + } while (0) + +#define LOAD2(field, fmt, d0, d1) \ + do { \ + const int64_t expect[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), fmt, i); \ + l->field = load_weight_direct(ms, name, expect, 2); \ + if (!l->field.data) \ + goto fail; \ + } while (0) + + LOAD2(wq, "layers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size); + LOAD2(wk, "layers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size); + LOAD2(wv, "layers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size); + LOAD2(wo, "layers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim); + LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", cfg->head_dim); + LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", cfg->head_dim); + LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", cfg->hidden_size); + LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", cfg->hidden_size); + LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", cfg->intermediate_size, + cfg->hidden_size); + LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size); + LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", cfg->hidden_size, + cfg->intermediate_size); #undef LOAD2 #undef LOAD_NORM @@ -724,13 +730,15 @@ static embed_model_t *model_load_range_ex(const char *model_dir, /* Only the final stage applies the output RMSNorm. */ if (layer_end == cfg->n_layers) { - const int64_t norm_shape[1] = { cfg->hidden_size }; + const int64_t norm_shape[1] = {cfg->hidden_size}; w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1); - if (!w->norm) goto fail; + if (!w->norm) + goto fail; } if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: layers [%d, %d) loaded " + fprintf(stderr, + "embed_model_load: layers [%d, %d) loaded " "(%d-dim embeddings)\n", layer_start, layer_end, cfg->hidden_size); @@ -741,14 +749,13 @@ fail: return NULL; } -embed_model_t *embed_model_load(const char *model_dir) -{ +embed_model_t *embed_model_load(const char *model_dir) { return model_load_range_ex(model_dir, 0, -1, 0); } -void embed_model_free(embed_model_t *model) -{ - if (!model) return; +void embed_model_free(embed_model_t *model) { + if (!model) + return; if (model->weights.layers) { for (int i = 0; i < model->config.n_layers; i++) { embed_layer_t *l = &model->weights.layers[i]; @@ -765,38 +772,43 @@ void embed_model_free(embed_model_t *model) free(model); } -embed_workspace_t *embed_workspace_new(const embed_model_t *model) -{ - if (!model) return NULL; +embed_workspace_t *embed_workspace_new(const embed_model_t *model) { + if (!model) + return NULL; embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t)); - if (!ws) return NULL; + if (!ws) + return NULL; ws->model = model; return ws; } -void embed_workspace_free(embed_workspace_t *ws) -{ - if (!ws) return; - free(ws->x); free(ws->x_norm); - free(ws->q); free(ws->k); free(ws->v); - free(ws->attn_out); free(ws->proj_out); - free(ws->ffn_gate); free(ws->ffn_up); +void embed_workspace_free(embed_workspace_t *ws) { + if (!ws) + return; + free(ws->x); + free(ws->x_norm); + free(ws->q); + free(ws->k); + free(ws->v); + free(ws->attn_out); + free(ws->proj_out); + free(ws->ffn_gate); + free(ws->ffn_up); free(ws->attn_scores); free(ws->offsets); - free(ws->rope_cos); free(ws->rope_sin); + free(ws->rope_cos); + free(ws->rope_sin); free(ws->bf16_widen); free(ws); } -const embed_config_t *embed_model_config(const embed_model_t *model) -{ +const embed_config_t *embed_model_config(const embed_model_t *model) { return model ? &model->config : NULL; } -static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, - const char *name) -{ - if (!sf || !name) return NULL; +static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) { + if (!sf || !name) + return NULL; for (int i = 0; i < sf->num_tensors; i++) { if (strcmp(sf->tensors[i].name, name) == 0) return &sf->tensors[i]; @@ -804,22 +816,22 @@ static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, return NULL; } -embed_late_model_t *embed_late_model_load(const char *model_dir) -{ +embed_late_model_t *embed_late_model_load(const char *model_dir) { if (!model_dir_has_late_projection(model_dir)) { fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n"); return NULL; } embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late)); - if (!late) return NULL; + if (!late) + return NULL; late->base = model_load_range_ex(model_dir, 0, -1, 1); - if (!late->base) goto fail; + if (!late->base) + goto fail; char path[1024]; - int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", - model_dir); + int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir); if (n < 0 || (size_t)n >= sizeof(path)) { fprintf(stderr, "embed_late_model_load: model path too long\n"); goto fail; @@ -831,8 +843,7 @@ embed_late_model_t *embed_late_model_load(const char *model_dir) goto fail; } - const safetensor_t *t = find_single_safetensor(late->dense_sf, - "linear.weight"); + const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight"); if (!t) { fprintf(stderr, "embed_late_model_load: missing linear.weight\n"); goto fail; @@ -842,9 +853,8 @@ embed_late_model_t *embed_late_model_load(const char *model_dir) goto fail; } - int64_t shape[2] = { t->shape[0], late->base->config.hidden_size }; - if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", - shape, 2, 1)) + int64_t shape[2] = {t->shape[0], late->base->config.hidden_size}; + if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) goto fail; late->projection.dtype = t->dtype; @@ -857,19 +867,20 @@ fail: return NULL; } -void embed_late_model_free(embed_late_model_t *model) -{ - if (!model) return; +void embed_late_model_free(embed_late_model_t *model) { + if (!model) + return; embed_model_free(model->base); safetensors_close(model->dense_sf); free(model); } -embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) -{ - if (!model || !model->base) return NULL; +embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) { + if (!model || !model->base) + return NULL; embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws)); - if (!ws) return NULL; + if (!ws) + return NULL; ws->model = model; ws->base_ws = embed_workspace_new(model->base); if (!ws->base_ws) { @@ -879,43 +890,41 @@ embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model return ws; } -void embed_late_workspace_free(embed_late_workspace_t *ws) -{ - if (!ws) return; +void embed_late_workspace_free(embed_late_workspace_t *ws) { + if (!ws) + return; embed_workspace_free(ws->base_ws); free(ws->states); free(ws); } -const embed_config_t *embed_late_model_config(const embed_late_model_t *model) -{ +const embed_config_t *embed_late_model_config(const embed_late_model_t *model) { return model && model->base ? &model->base->config : NULL; } -int embed_late_model_token_dim(const embed_late_model_t *model) -{ +int embed_late_model_token_dim(const embed_late_model_t *model) { return model ? model->token_dim : 0; } int embed_late_model_encode_tokens(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ - if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || - !out_vectors || model->token_dim <= 0) + embed_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors || + model->token_dim <= 0) return -1; int hidden = model->base->config.hidden_size; int dim = model->token_dim; if (ensure_late_state_buffer(ws, n_tokens) != 0) return -1; - if (embed_model_forward_into(model->base, ws->base_ws, token_ids, - n_tokens, ws->states) != 0) + if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0) return -1; - linear_nobias_weight(ws->base_ws, out_vectors, ws->states, - &model->projection, n_tokens, hidden, dim); + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden, + dim); if (normalize) { for (int i = 0; i < n_tokens; i++) { if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) @@ -925,12 +934,12 @@ int embed_late_model_encode_tokens(const embed_late_model_t *model, return 0; } -float embed_late_maxsim(const float *query_vectors, int query_tokens, - const float *doc_vectors, int doc_tokens, - int dim) -{ - if (!query_vectors || !doc_vectors || query_tokens <= 0 || - doc_tokens <= 0 || dim <= 0) +float embed_late_maxsim(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + int doc_tokens, + int dim) { + if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0) return 0.0f; float score = 0.0f; @@ -940,20 +949,23 @@ float embed_late_maxsim(const float *query_vectors, int query_tokens, for (int di = 0; di < doc_tokens; di++) { const float *d = doc_vectors + (size_t)di * dim; float dot = qwen_dot_f32(q, d, dim); - if (dot > best) best = dot; + if (dot > best) + best = dot; } score += best; } return score; } -int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, - const float *doc_vectors, - const int *doc_offsets, int docs, - int dim, float *scores) -{ - if (!query_vectors || !doc_vectors || !doc_offsets || !scores || - query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0) +int embed_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores) { + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || + docs <= 0 || dim <= 0 || doc_offsets[0] != 0) return -1; for (int i = 0; i < docs; i++) { @@ -969,12 +981,12 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, enum { LATE_GROUP_TOKEN_BUDGET = 4096 }; int total = doc_offsets[docs]; int budget = LATE_GROUP_TOKEN_BUDGET; - size_t cap = (size_t)query_tokens * - (size_t)(total < budget ? total : budget); + size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget); int largest = 0; for (int i = 0; i < docs; i++) { int len = doc_offsets[i + 1] - doc_offsets[i]; - if (len > largest) largest = len; + if (len > largest) + largest = len; } if ((size_t)query_tokens * (size_t)largest > cap) cap = (size_t)query_tokens * (size_t)largest; @@ -985,16 +997,13 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, int first = doc; int start = doc_offsets[first]; doc++; - while (doc < docs && - doc_offsets[doc + 1] - start <= budget && - doc_offsets[doc + 1] - start <= - (int)(cap / (size_t)query_tokens)) + while (doc < docs && doc_offsets[doc + 1] - start <= budget && + doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens)) doc++; int group_tokens = doc_offsets[doc] - start; - qwen_matmul_t(sim, query_vectors, - doc_vectors + (size_t)start * dim, - query_tokens, dim, group_tokens); + qwen_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, + group_tokens); for (int i = first; i < doc; i++) { int s0 = doc_offsets[i] - start; @@ -1004,7 +1013,8 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, const float *row = sim + (size_t)qi * group_tokens; float best = row[s0]; for (int t = s0 + 1; t < s1; t++) - if (row[t] > best) best = row[t]; + if (row[t] > best) + best = row[t]; score += best; } scores[i] = score; @@ -1020,63 +1030,58 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, int start = doc_offsets[i]; int end = doc_offsets[i + 1]; scores[i] = embed_late_maxsim(query_vectors, query_tokens, - doc_vectors + (size_t)start * dim, - end - start, dim); + doc_vectors + (size_t)start * dim, end - start, dim); } return 0; } -size_t embed_workspace_nbytes(const embed_workspace_t *ws) -{ - if (!ws) return 0; +size_t embed_workspace_nbytes(const embed_workspace_t *ws) { + if (!ws) + return 0; size_t total = sizeof(*ws); const embed_model_t *model = ws->model; - if (!model) return total; + if (!model) + return total; const embed_config_t *c = &model->config; size_t row_floats = 0; const int dims[] = { - c->hidden_size, /* x */ - c->hidden_size, /* x_norm */ - c->q_dim, /* q */ - c->kv_dim, /* k */ - c->kv_dim, /* v */ - c->q_dim, /* attn_out */ - c->hidden_size, /* proj_out */ - c->intermediate_size, /* ffn_gate */ - c->intermediate_size, /* ffn_up */ + c->hidden_size, /* x */ + c->hidden_size, /* x_norm */ + c->q_dim, /* q */ + c->kv_dim, /* k */ + c->kv_dim, /* v */ + c->q_dim, /* attn_out */ + c->hidden_size, /* proj_out */ + c->intermediate_size, /* ffn_gate */ + c->intermediate_size, /* ffn_up */ }; for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { - if (dims[i] < 0 || - add_size(row_floats, (size_t)dims[i], &row_floats) != 0) + if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) return SIZE_MAX; } size_t buf_floats = 0; - if (ws->buf_seq_cap > 0 && - mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) + if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) return SIZE_MAX; size_t rope_floats = 0; if (ws->rope_cache_cap > 0) { size_t rope_rows = 0; - if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, - &rope_rows) != 0 || + if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || mul_size(rope_rows, 2, &rope_floats) != 0) return SIZE_MAX; } size_t all_floats = 0, bytes = 0; if (add_size(buf_floats, rope_floats, &all_floats) != 0 || - mul_size(all_floats, sizeof(float), &bytes) != 0 || - add_size(total, bytes, &total) != 0) + mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) return SIZE_MAX; size_t offset_bytes = 0; - if (ws->offsets_cap > 0 && - mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) + if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) return SIZE_MAX; if (add_size(total, offset_bytes, &total) != 0) return SIZE_MAX; @@ -1097,9 +1102,12 @@ size_t embed_workspace_nbytes(const embed_workspace_t *ws) * Packed forward pass * ======================================================================== */ -static int build_offsets(const embed_input_t *inputs, int batch, int require_ids, - int *offsets, int *total_out, int *max_out) -{ +static int build_offsets(const embed_input_t *inputs, + int batch, + int require_ids, + int *offsets, + int *total_out, + int *max_out) { if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) return -1; @@ -1124,52 +1132,54 @@ static int build_offsets(const embed_input_t *inputs, int batch, int require_ids static int forward_packed_slice_inplace(const embed_model_t *model, embed_workspace_t *ws, const embed_input_t *inputs, - int batch, const int *offsets, - int total_seq, int max_seq, + int batch, + const int *offsets, + int total_seq, + int max_seq, const float *input_states, - int layer_start, int layer_end, - int apply_final_norm) -{ - if (!model || !ws || ws->model != model || !inputs || - batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0) + int layer_start, + int layer_end, + int apply_final_norm) { + if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || + total_seq <= 0 || max_seq <= 0) return -1; - const embed_config_t *cfg = &model->config; - const embed_weights_t *w = &model->weights; - if (layer_start < 0 || layer_start > layer_end || - layer_end > cfg->n_layers || + const embed_config_t *cfg = &model->config; + const embed_weights_t *w = &model->weights; + if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers || layer_start < model->layer_start || layer_end > model->layer_end || - (layer_start == 0 && input_states) || - (layer_start != 0 && !input_states) || + (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || (layer_start == 0 && !w->embed_tokens.data) || (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) return -1; - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; int n_kv_heads = cfg->n_kv_heads; - int head_dim = cfg->head_dim; - int q_dim = cfg->q_dim; - int kv_dim = cfg->kv_dim; - int inter = cfg->intermediate_size; - float eps = cfg->rms_norm_eps; - - if (ensure_buffers(ws, cfg, total_seq) != 0) return -1; - if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1; + int head_dim = cfg->head_dim; + int q_dim = cfg->q_dim; + int kv_dim = cfg->kv_dim; + int inter = cfg->intermediate_size; + float eps = cfg->rms_norm_eps; + + if (ensure_buffers(ws, cfg, total_seq) != 0) + return -1; + if (ensure_rope_cache(ws, cfg, max_seq) != 0) + return -1; ensure_attention_scores(ws, offsets, batch); const float *rope_cos = ws->rope_cos; const float *rope_sin = ws->rope_sin; - float *x = ws->x; - float *x_norm = ws->x_norm; - float *q_buf = ws->q; - float *k_buf = ws->k; - float *v_buf = ws->v; + float *x = ws->x; + float *x_norm = ws->x_norm; + float *q_buf = ws->q; + float *k_buf = ws->k; + float *v_buf = ws->v; float *attn_out = ws->attn_out; float *proj_out = ws->proj_out; float *ffn_gate = ws->ffn_gate; - float *ffn_up = ws->ffn_up; + float *ffn_up = ws->ffn_up; /* A first stage performs embedding lookup. Later stages resume from a * packed hidden-state tensor received from the preceding stage. */ @@ -1179,12 +1189,12 @@ static int forward_packed_slice_inplace(const embed_model_t *model, for (int i = 0; i < inputs[b].n_tokens; i++) { int id = inputs[b].ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", - id, b, i); + fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, + i); return -1; } - copy_weight_row(x + (size_t)(start + i) * hidden, - &w->embed_tokens, (size_t)id, hidden); + copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id, + hidden); } } } else { @@ -1200,35 +1210,31 @@ static int forward_packed_slice_inplace(const embed_model_t *model, qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, - &l->wq, &l->wk, &l->wv, - total_seq, hidden, q_dim, kv_dim); + linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, + hidden, q_dim, kv_dim); - qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); for (int b = 0; b < batch; b++) { int start = offsets[b]; int len = offsets[b + 1] - start; - qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, - rope_cos, rope_sin, len, n_heads, head_dim); - qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, - rope_cos, rope_sin, len, n_kv_heads, head_dim); + qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, + head_dim); + qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, + n_kv_heads, head_dim); } - model->attention( - attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, - n_kv_heads, head_dim, scale, ws->attn_scores, - ws->attn_scores_bytes); + model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, - &l->gate_proj, &l->up_proj, - total_seq, hidden, inter, inter); + linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, + hidden, inter, inter); qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); @@ -1242,21 +1248,20 @@ static int forward_packed_slice_inplace(const embed_model_t *model, static int forward_packed_inplace(const embed_model_t *model, embed_workspace_t *ws, const embed_input_t *inputs, - int batch, const int *offsets, - int total_seq, int max_seq, - int apply_final_norm) -{ - return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, NULL, 0, - model->config.n_layers, - apply_final_norm); + int batch, + const int *offsets, + int total_seq, + int max_seq, + int apply_final_norm) { + return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL, + 0, model->config.n_layers, apply_final_norm); } static int pool_embeddings(const embed_model_t *model, const embed_workspace_t *ws, - const int *offsets, int batch, - float *out_embeddings) -{ + const int *offsets, + int batch, + float *out_embeddings) { const embed_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; @@ -1297,8 +1302,7 @@ static int pool_embeddings(const embed_model_t *model, emb[d] *= inv_len; } - if (cfg->normalize_embeddings && - embed_l2_normalize(emb, hidden) != 0) + if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0) return -1; } return 0; @@ -1308,38 +1312,40 @@ static int pool_embeddings(const embed_model_t *model, * Public forward/embed APIs * ======================================================================== */ -int embed_model_forward_into(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_states) -{ - if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1; +int embed_model_forward_into(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states) { + if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) + return -1; size_t count, bytes; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, - &count) != 0 || + if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 || mul_size(count, sizeof(float), &bytes) != 0) return -1; - embed_input_t input = { token_ids, n_tokens }; - int offsets[2] = { 0, n_tokens }; - if (forward_packed_inplace(model, ws, &input, 1, offsets, - n_tokens, n_tokens, 1) != 0) + embed_input_t input = {token_ids, n_tokens}; + int offsets[2] = {0, n_tokens}; + if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0) return -1; memcpy(out_states, ws->x, bytes); return 0; } -int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_tokens, - const embed_span_t *spans, int n_spans, - float *out_embeddings) -{ - if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || - !out_embeddings) +int embed_pool_spans(const embed_config_t *cfg, + const float *states, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings) { + if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings) return -1; int hidden = cfg->hidden_size; - if (hidden <= 0) return -1; + if (hidden <= 0) + return -1; for (int s = 0; s < n_spans; s++) { int start = spans[s].start; @@ -1365,17 +1371,20 @@ int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_token } int embed_model_encode_spans_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_context_input_t *inputs, int batch, - float *out_embeddings) -{ + embed_workspace_t *ws, + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; - if (ensure_offsets(ws, batch) != 0) return -1; - if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) return -1; + if (ensure_offsets(ws, batch) != 0) + return -1; + if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) + return -1; embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); - if (!packed_inputs) return -1; + if (!packed_inputs) + return -1; int *offsets = ws->offsets; int total_seq = 0; int max_seq = 0; @@ -1383,9 +1392,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model, for (int b = 0; b < batch; b++) { const embed_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_seq > INT_MAX - n_tokens || - total_spans > INT_MAX - input->n_spans) { + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) { free(packed_inputs); return -1; } @@ -1393,12 +1401,12 @@ int embed_model_encode_spans_batch(const embed_model_t *model, offsets[b] = total_seq; total_seq += n_tokens; total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; + if (n_tokens > max_seq) + max_seq = n_tokens; for (int s = 0; s < input->n_spans; s++) { int start = input->spans[s].start; int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || - len > n_tokens - start) { + if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) { free(packed_inputs); return -1; } @@ -1406,8 +1414,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model, } offsets[batch] = total_seq; - if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, - total_seq, max_seq, 1) != 0) { + if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) != + 0) { free(packed_inputs); return -1; } @@ -1417,39 +1425,43 @@ int embed_model_encode_spans_batch(const embed_model_t *model, int hidden = model->config.hidden_size; for (int b = 0; b < batch; b++) { const embed_context_input_t *input = &inputs[b]; - if (embed_pool_spans(&model->config, - ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) + if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, + input->input.n_tokens, input->spans, input->n_spans, + out_embeddings + (size_t)span_offset * hidden) != 0) return -1; span_offset += input->n_spans; } return 0; } -int embed_model_encode_spans(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - const embed_span_t *spans, int n_spans, - float *out_embeddings) -{ +int embed_model_encode_spans(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings) { embed_context_input_t input = { - .input = { token_ids, n_tokens }, + .input = {token_ids, n_tokens}, .spans = spans, .n_spans = n_spans, }; return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings); } -float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens) -{ - if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; +float *embed_model_forward(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens) { + if (!model || !ws || !token_ids || n_tokens <= 0) + return NULL; size_t n; if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0) return NULL; float *out = malloc_floats(n); - if (!out) return NULL; + if (!out) + return NULL; if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { free(out); @@ -1458,13 +1470,16 @@ float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws, return out; } -int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws, - const embed_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; +int embed_model_encode_batch(const embed_model_t *model, + embed_workspace_t *ws, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) + return -1; - if (ensure_offsets(ws, batch) != 0) return -1; + if (ensure_offsets(ws, batch) != 0) + return -1; int *offsets = ws->offsets; int total_seq = 0; @@ -1472,20 +1487,19 @@ int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws, if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) return -1; - int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, 0); + int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); if (rc == 0) rc = pool_embeddings(model, ws, offsets, batch, out_embeddings); return rc; } -int embed_pool_batch(const embed_config_t *cfg, const float *states, - const int *seq_lengths, int batch, - float *out_embeddings) -{ - if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || - cfg->hidden_size <= 0) +int embed_pool_batch(const embed_config_t *cfg, + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings) { + if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0) return -1; int offset = 0; @@ -1496,14 +1510,12 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states, float *emb = out_embeddings + (size_t)b * cfg->hidden_size; if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) { - const float *row = - states + (size_t)(offset + len - 1) * cfg->hidden_size; + const float *row = states + (size_t)(offset + len - 1) * cfg->hidden_size; memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); } else { memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); for (int t = 0; t < len; t++) { - const float *row = - states + (size_t)(offset + t) * cfg->hidden_size; + const float *row = states + (size_t)(offset + t) * cfg->hidden_size; for (int d = 0; d < cfg->hidden_size; d++) emb[d] += row[d]; } @@ -1511,31 +1523,35 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states, for (int d = 0; d < cfg->hidden_size; d++) emb[d] *= inv_len; } - if (cfg->normalize_embeddings && - embed_l2_normalize(emb, cfg->hidden_size) != 0) + if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0) return -1; offset += len; } return 0; } -int embed_model_encode_into(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_embedding) -{ - embed_input_t input = { token_ids, n_tokens }; +int embed_model_encode_into(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; return embed_model_encode_batch(model, ws, &input, 1, out_embedding); } -float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens) -{ - if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; +float *embed_model_encode(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens) { + if (!model || !ws || !token_ids || n_tokens <= 0) + return NULL; int hidden = model->config.hidden_size; - if (hidden <= 0) return NULL; + if (hidden <= 0) + return NULL; float *emb = malloc_floats((size_t)hidden); - if (!emb) return NULL; + if (!emb) + return NULL; if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) { free(emb); @@ -1548,15 +1564,16 @@ float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws, * Vector helpers * ======================================================================== */ -int embed_l2_normalize(float *vec, int dim) -{ - if (!vec || dim <= 0) return -1; +int embed_l2_normalize(float *vec, int dim) { + if (!vec || dim <= 0) + return -1; float norm_sq = 0.0f; for (int i = 0; i < dim; i++) norm_sq += vec[i] * vec[i]; - if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) return -1; + if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) + return -1; float inv_norm = 1.0f / sqrtf(norm_sq); for (int i = 0; i < dim; i++) @@ -1564,9 +1581,9 @@ int embed_l2_normalize(float *vec, int dim) return 0; } -float embed_cosine_similarity(const float *a, const float *b, int dim) -{ - if (!a || !b || dim <= 0) return 0.0f; +float embed_cosine_similarity(const float *a, const float *b, int dim) { + if (!a || !b || dim <= 0) + return 0.0f; float dot = 0.0f; float norm_a_sq = 0.0f; @@ -1577,8 +1594,7 @@ float embed_cosine_similarity(const float *a, const float *b, int dim) norm_b_sq += b[i] * b[i]; } - if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || - !isfinite(norm_a_sq) || !isfinite(norm_b_sq)) + if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || !isfinite(norm_b_sq)) return 0.0f; return dot / sqrtf(norm_a_sq * norm_b_sq); } diff --git a/embed.h b/embed.h index f7048b4..24e53e1 100644 --- a/embed.h +++ b/embed.h @@ -23,9 +23,9 @@ * Shared constants and family defaults * ======================================================================== */ -#define EMBED_VOCAB_SIZE 151936 -#define EMBED_HEAD_DIM 128 -#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */ +#define EMBED_VOCAB_SIZE 151936 +#define EMBED_HEAD_DIM 128 +#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */ #define EMBED_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ /* ======================================================================== @@ -48,8 +48,8 @@ typedef struct { int n_heads; int n_kv_heads; int head_dim; - int q_dim; /* n_heads * head_dim */ - int kv_dim; /* n_kv_heads * head_dim */ + int q_dim; /* n_heads * head_dim */ + int kv_dim; /* n_kv_heads * head_dim */ int intermediate_size; int vocab_size; float rms_norm_eps; @@ -141,16 +141,20 @@ EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws); * * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ -EMBED_API float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens); +EMBED_API float *embed_model_encode(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens); /* * Compute one embedding into caller-provided out[hidden_size]. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_model_encode_into(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_embedding); +EMBED_API int embed_model_encode_into(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); /* * Compute embeddings for a true packed/ragged batch. @@ -161,9 +165,11 @@ EMBED_API int embed_model_encode_into(const embed_model_t *model, embed_workspac * * Returns 0 on success, -1 on error. */ -EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws, - const embed_input_t *inputs, int batch, - float *out_embeddings); +EMBED_API int embed_model_encode_batch(const embed_model_t *model, + embed_workspace_t *ws, + const embed_input_t *inputs, + int batch, + float *out_embeddings); /* * Apply the config's pooling mode to packed final hidden states. @@ -171,9 +177,11 @@ EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspa * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive * token count per sequence. The states must already include final RMSNorm. */ -EMBED_API int embed_pool_batch(const embed_config_t *cfg, const float *states, - const int *seq_lengths, int batch, - float *out_embeddings); +EMBED_API int embed_pool_batch(const embed_config_t *cfg, + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings); /* * Run the transformer forward pass WITHOUT pooling. @@ -183,16 +191,20 @@ EMBED_API int embed_pool_batch(const embed_config_t *cfg, const float *states, * Useful for contextual (late-chunking) models where you need * per-token embeddings before splitting by separator positions. */ -EMBED_API float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens); +EMBED_API float *embed_model_forward(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens); /* * Run the transformer forward pass into caller-provided * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error. */ -EMBED_API int embed_model_forward_into(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_states); +EMBED_API int embed_model_forward_into(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states); /* * Pool embeddings from final hidden states. @@ -201,18 +213,24 @@ EMBED_API int embed_model_forward_into(const embed_model_t *model, embed_workspa * embed_model_forward_into(). Each span is mean pooled into * out_embeddings[n_spans, hidden_size] without L2 normalization. */ -EMBED_API int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_tokens, - const embed_span_t *spans, int n_spans, - float *out_embeddings); +EMBED_API int embed_pool_spans(const embed_config_t *cfg, + const float *states, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings); /* * Run one contextual sequence and pool selected token spans. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_model_encode_spans(const embed_model_t *model, embed_workspace_t *ws, - const int *token_ids, int n_tokens, - const embed_span_t *spans, int n_spans, - float *out_embeddings); +EMBED_API int embed_model_encode_spans(const embed_model_t *model, + embed_workspace_t *ws, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings); /* * Run a packed/ragged contextual document batch and pool every selected span. @@ -221,9 +239,10 @@ EMBED_API int embed_model_encode_spans(const embed_model_t *model, embed_workspa * and out_embeddings contains spans in document order. */ EMBED_API int embed_model_encode_spans_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_context_input_t *inputs, int batch, - float *out_embeddings); + embed_workspace_t *ws, + const embed_context_input_t *inputs, + int batch, + float *out_embeddings); /* * L2-normalize vec[dim] in place. @@ -261,16 +280,20 @@ EMBED_API int embed_late_model_token_dim(const embed_late_model_t *model); * If normalize is non-zero, every non-zero token vector is L2-normalized. */ EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors); + embed_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors); /* * Sum over query tokens of max dot product against document tokens. */ -EMBED_API float embed_late_maxsim(const float *query_vectors, int query_tokens, - const float *doc_vectors, int doc_tokens, - int dim); +EMBED_API float embed_late_maxsim(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + int doc_tokens, + int dim); /* * Score one query against a packed/ragged batch of documents. @@ -279,10 +302,13 @@ EMBED_API float embed_late_maxsim(const float *query_vectors, int query_tokens, * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim). * scores is caller-provided [docs]. */ -EMBED_API int embed_late_maxsim_batch(const float *query_vectors, int query_tokens, - const float *doc_vectors, - const int *doc_offsets, int docs, - int dim, float *scores); +EMBED_API int embed_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores); /* Verbose level: 0=quiet, 1=info, 2=debug */ EMBED_API extern int embed_verbose; diff --git a/embed_cli.c b/embed_cli.c index caa586a..9baa104 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -18,68 +18,66 @@ #include #include -static double now_ms(void) -{ +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; } -static void print_usage(const char *prog) -{ +static void print_usage(const char *prog) { fprintf(stderr, - "Usage: %s -d [options] [text...]\n" - "\n" - "Options:\n" - " -d Model directory (required)\n" + "Usage: %s -d [options] [text...]\n" + "\n" + "Options:\n" + " -d Model directory (required)\n" #ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" + " --mlx Use Apple MLX GPU backend\n" #endif #ifdef USE_CUDA - " --cuda Use CUDA/cuBLAS GPU backend\n" + " --cuda Use CUDA/cuBLAS GPU backend\n" #endif - " --stream Read lines from stdin, write one JSON embedding per line\n" - " --json Emit JSON instead of plain text (embedding or matrix)\n" + " --stream Read lines from stdin, write one JSON embedding per line\n" + " --json Emit JSON instead of plain text (embedding or matrix)\n" #ifdef USE_MLX - " --mlx-quant-bits N\n" - " Quantize MLX linear weights to 8 bits at load time\n" - " --mlx-quant-group-size N\n" - " MLX quantization group size (default: 64)\n" + " --mlx-quant-bits N\n" + " Quantize MLX linear weights to 8 bits at load time\n" + " --mlx-quant-group-size N\n" + " MLX quantization group size (default: 64)\n" #endif #ifdef USE_CUDA - " --cuda-gemm-mode MODE\n" - " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" - " --cuda-weight-dtype DTYPE\n" - " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" - " weight memory and uses BF16 tensor cores\n" + " --cuda-gemm-mode MODE\n" + " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" + " --cuda-weight-dtype DTYPE\n" + " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" + " weight memory and uses BF16 tensor cores\n" #endif - " -b, --batch-size N\n" - " Max texts per engine batch (default: all; --stream: 1)\n" - " -t, --threads N\n" - " CPU threads (default: all cores)\n" - " -e, --embeddings\n" - " Print raw embeddings (with multiple texts)\n" - " -v, --verbose\n" - " Verbose (-vv for debug)\n" - " -V, --version\n" - " Print version and exit\n" - " --build-info Print build details and exit\n" - " -h Show this help\n" - "\n" - "Modes:\n" - " 1 text arg Embedding as space-separated floats (JSON with --json)\n" - " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n" - " no args Batch: read all stdin lines, then similarity matrix\n" - " --stream Streaming: read stdin lines, write JSON per line\n" - "\n" - "Examples:\n" - " %s -d ./model \"what is AI?\"\n" + " -b, --batch-size N\n" + " Max texts per engine batch (default: all; --stream: 1)\n" + " -t, --threads N\n" + " CPU threads (default: all cores)\n" + " -e, --embeddings\n" + " Print raw embeddings (with multiple texts)\n" + " -v, --verbose\n" + " Verbose (-vv for debug)\n" + " -V, --version\n" + " Print version and exit\n" + " --build-info Print build details and exit\n" + " -h Show this help\n" + "\n" + "Modes:\n" + " 1 text arg Embedding as space-separated floats (JSON with --json)\n" + " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n" + " no args Batch: read all stdin lines, then similarity matrix\n" + " --stream Streaming: read stdin lines, write JSON per line\n" + "\n" + "Examples:\n" + " %s -d ./model \"what is AI?\"\n" #ifdef USE_MLX - " %s -d ./model --mlx --stream < texts.txt\n", - prog, prog, prog); + " %s -d ./model --mlx --stream < texts.txt\n", + prog, prog, prog); #else - , - prog, prog); + , + prog, prog); #endif } @@ -88,29 +86,28 @@ static void print_usage(const char *prog) * ======================================================================== */ typedef struct { - embed_model_t *model; + embed_model_t *model; embed_workspace_t *workspace; #ifdef USE_MLX - embed_mlx_ctx_t *mlx_ctx; + embed_mlx_ctx_t *mlx_ctx; #endif #ifdef USE_CUDA - embed_cuda_ctx_t *cuda_ctx; + embed_cuda_ctx_t *cuda_ctx; #endif qwen_tokenizer_t *tok; qwen_tokenizer_workspace_t *tok_ws; - int dim; - int terminal_token_id; - int append_terminal_token; + int dim; + int terminal_token_id; + int append_terminal_token; } engine_t; typedef struct { int *ids; - int n_tokens; + int n_tokens; } token_buf_t; -static int engine_embed_batch(engine_t *e, const embed_input_t *inputs, - int batch, float *out_embeddings) -{ +static int +engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) { #ifdef USE_MLX if (e->mlx_ctx) return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings); @@ -119,15 +116,12 @@ static int engine_embed_batch(engine_t *e, const embed_input_t *inputs, if (e->cuda_ctx) return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings); #endif - return embed_model_encode_batch(e->model, e->workspace, - inputs, batch, out_embeddings); + return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings); } -static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) -{ +static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { memset(out, 0, sizeof(*out)); - out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, - text, &out->n_tokens); + out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens); if (!out->ids || out->n_tokens == 0) { fprintf(stderr, "tokenization failed: %s\n", text); free(out->ids); @@ -141,8 +135,7 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) memset(out, 0, sizeof(*out)); return -1; } - int *ids = (int *)realloc( - out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); + int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); if (!ids) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -156,15 +149,16 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) fprintf(stderr, "tokens (%d): ", out->n_tokens); for (int i = 0; i < out->n_tokens && i < 20; i++) fprintf(stderr, "%d ", out->ids[i]); - if (out->n_tokens > 20) fprintf(stderr, "..."); + if (out->n_tokens > 20) + fprintf(stderr, "..."); fprintf(stderr, "\n"); } return 0; } -static void free_tokens(token_buf_t *tokens, int n) -{ - if (!tokens) return; +static void free_tokens(token_buf_t *tokens, int n) { + if (!tokens) + return; for (int i = 0; i < n; i++) free(tokens[i].ids); } @@ -173,41 +167,40 @@ static void free_tokens(token_buf_t *tokens, int n) * Output helpers * ======================================================================== */ -static void print_embedding_raw(const float *emb, int dim) -{ +static void print_embedding_raw(const float *emb, int dim) { for (int i = 0; i < dim; i++) { - if (i > 0) putchar(' '); + if (i > 0) + putchar(' '); printf("%.8f", (double)emb[i]); } putchar('\n'); } /* Single embedding as a JSON array of floats. */ -static void print_embedding_array(const float *emb, int dim) -{ +static void print_embedding_array(const float *emb, int dim) { putchar('['); for (int i = 0; i < dim; i++) { - if (i) putchar(','); + if (i) + putchar(','); printf("%.8f", (double)emb[i]); } puts("]"); } -static size_t engine_workspace_nbytes(const engine_t *e) -{ +static size_t engine_workspace_nbytes(const engine_t *e) { return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0; } -static void print_embedding_json(const float *emb, int dim, int n_tokens, - double ms, size_t workspace_bytes) -{ +static void +print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) { printf("{\"embedding\":["); for (int i = 0; i < dim; i++) { - if (i > 0) putchar(','); + if (i > 0) + putchar(','); printf("%.8f", (double)emb[i]); } - printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", - dim, n_tokens, ms, workspace_bytes); + printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms, + workspace_bytes); fflush(stdout); } @@ -215,9 +208,9 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens, * Stdin mode: read lines from stdin, write JSON to stdout * ======================================================================== */ -static int process_stdin_batch(engine_t *e, char **lines, int n_lines) -{ - if (n_lines <= 0) return 0; +static int process_stdin_batch(engine_t *e, char **lines, int n_lines) { + if (n_lines <= 0) + return 0; token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t)); @@ -225,7 +218,10 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines) float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); if (!tokens || !inputs || !input_to_line || !embs) { fprintf(stderr, "OOM\n"); - free(tokens); free(inputs); free(input_to_line); free(embs); + free(tokens); + free(inputs); + free(input_to_line); + free(embs); for (int i = 0; i < n_lines; i++) printf("{\"error\":\"embedding failed\"}\n"); fflush(stdout); @@ -268,26 +264,27 @@ static int process_stdin_batch(engine_t *e, char **lines, int n_lines) printf("{\"error\":\"embedding failed\"}\n"); continue; } - print_embedding_json(embs + (size_t)next_valid * e->dim, - e->dim, tokens[i].n_tokens, ms, + print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms, engine_workspace_nbytes(e)); next_valid++; } fflush(stdout); free_tokens(tokens, n_lines); - free(tokens); free(inputs); free(input_to_line); free(embs); + free(tokens); + free(inputs); + free(input_to_line); + free(embs); return rc == 0 ? 0 : 1; } -static int run_stdin(engine_t *e, int batch_size) -{ +static int run_stdin(engine_t *e, int batch_size) { char line[65536]; - if (batch_size <= 0) batch_size = 1; + if (batch_size <= 0) + batch_size = 1; if (embed_verbose >= 1) - fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", - batch_size); + fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size); char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); if (!batch_lines) { @@ -300,13 +297,14 @@ static int run_stdin(engine_t *e, int batch_size) while (fgets(line, sizeof(line), stdin)) { /* Strip newline */ size_t len = strlen(line); - while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r')) + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) line[--len] = '\0'; - if (len == 0) continue; + if (len == 0) + continue; if (embed_verbose >= 1) - fprintf(stderr, "stdin: \"%.*s%s\"\n", - (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); + fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line, + len > 60 ? "..." : ""); batch_lines[n_batch] = strdup(line); if (!batch_lines[n_batch]) { @@ -345,18 +343,19 @@ static int run_stdin(engine_t *e, int batch_size) * Batch mode: embed args or stdin lines, then print similarity or vectors * ======================================================================== */ -static int append_text(char ***texts, int *n_texts, int *cap, const char *s) -{ +static int append_text(char ***texts, int *n_texts, int *cap, const char *s) { if (*n_texts == *cap) { int new_cap = *cap ? *cap * 2 : 8; char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); - if (!new_texts) return -1; + if (!new_texts) + return -1; *texts = new_texts; *cap = new_cap; } (*texts)[*n_texts] = strdup(s); - if (!(*texts)[*n_texts]) return -1; + if (!(*texts)[*n_texts]) + return -1; (*n_texts)++; return 0; } @@ -364,18 +363,18 @@ static int append_text(char ***texts, int *n_texts, int *cap, const char *s) /* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one * row of space-separated values per line. With print_embs the raw embeddings * follow, one per line after a blank line (plain-text output only). */ -static void print_matrix(const float *embs, int n, int dim, - int json, int print_embs) -{ +static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) { if (json) { putchar('['); for (int i = 0; i < n; i++) { - if (i) putchar(','); + if (i) + putchar(','); putchar('['); for (int j = 0; j < n; j++) { - if (j) putchar(','); - printf("%.6f", (double)embed_cosine_similarity( - embs + (size_t)i * dim, embs + (size_t)j * dim, dim)); + if (j) + putchar(','); + printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim, + embs + (size_t)j * dim, dim)); } putchar(']'); } @@ -385,9 +384,10 @@ static void print_matrix(const float *embs, int n, int dim, for (int i = 0; i < n; i++) { for (int j = 0; j < n; j++) { - if (j) putchar(' '); - printf("%.6f", (double)embed_cosine_similarity( - embs + (size_t)i * dim, embs + (size_t)j * dim, dim)); + if (j) + putchar(' '); + printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim, + embs + (size_t)j * dim, dim)); } putchar('\n'); } @@ -398,12 +398,11 @@ static void print_matrix(const float *embs, int n, int dim, } } -static int run_batch(engine_t *e, int argc, char **argv, int arg_start, - int print_embs, int batch_size, int json) -{ - int n_texts = 0, cap = 0; - char **texts = NULL; - int rc = 0; +static int run_batch( + engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) { + int n_texts = 0, cap = 0; + char **texts = NULL; + int rc = 0; if (arg_start < argc) { for (int i = arg_start; i < argc; i++) { @@ -414,9 +413,10 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, char line[65536]; while (fgets(line, sizeof(line), stdin)) { size_t l = strlen(line); - while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) + while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r')) line[--l] = '\0'; - if (l == 0) continue; + if (l == 0) + continue; if (append_text(&texts, &n_texts, &cap, line) != 0) goto oom_texts; } @@ -434,15 +434,18 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); if (!tokens || !inputs || !embs) { fprintf(stderr, "OOM\n"); - free(tokens); free(inputs); free(embs); - for (int i = 0; i < n_texts; i++) free(texts[i]); + free(tokens); + free(inputs); + free(embs); + for (int i = 0; i < n_texts; i++) + free(texts[i]); free(texts); return 1; } for (int i = 0; i < n_texts; i++) { if (embed_verbose >= 1) - fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); + fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]); if (tokenize_text(e, texts[i], &tokens[i]) != 0) { rc = 1; goto done; @@ -454,36 +457,43 @@ static int run_batch(engine_t *e, int argc, char **argv, int arg_start, int max_batch = batch_size > 0 ? batch_size : n_texts; for (int start = 0; start < n_texts; start += max_batch) { int cur = n_texts - start; - if (cur > max_batch) cur = max_batch; + if (cur > max_batch) + cur = max_batch; double t0 = now_ms(); - if (engine_embed_batch(e, inputs + start, cur, - embs + (size_t)start * dim) != 0) { + if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) { fprintf(stderr, "forward pass failed\n"); rc = 1; goto done; } if (embed_verbose >= 1) - fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", - start, start + cur - 1, cur, now_ms() - t0); + fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1, + cur, now_ms() - t0); } if (n_texts == 1) { - if (json) print_embedding_array(embs, dim); - else print_embedding_raw(embs, dim); + if (json) + print_embedding_array(embs, dim); + else + print_embedding_raw(embs, dim); } else { print_matrix(embs, n_texts, dim, json, print_embs); } done: free_tokens(tokens, n_texts); - for (int i = 0; i < n_texts; i++) free(texts[i]); - free(texts); free(tokens); free(inputs); free(embs); + for (int i = 0; i < n_texts; i++) + free(texts[i]); + free(texts); + free(tokens); + free(inputs); + free(embs); return rc; oom_texts: fprintf(stderr, "OOM\n"); - for (int i = 0; i < n_texts; i++) free(texts[i]); + for (int i = 0; i < n_texts; i++) + free(texts[i]); free(texts); return 1; } @@ -492,15 +502,14 @@ oom_texts: * Main * ======================================================================== */ -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]) { const char *model_dir = NULL; - int n_threads = 0; + int n_threads = 0; int print_embs = 0; - int verbose = 0; + int verbose = 0; int stdin_mode = 0; int batch_size = 0; - int json = 0; + int json = 0; const char *prog = embed_prog_name(argv[0]); #ifdef USE_MLX int use_mlx = 0; @@ -516,22 +525,21 @@ int main(int argc, char *argv[]) int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { const char *f = argv[arg_start]; - if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } - else if (!strcmp(f, "-V") || !strcmp(f, "--version")) { + if (!strcmp(f, "-d")) { + model_dir = argv[++arg_start]; + } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) { embed_print_version(prog); return 0; - } - else if (!strcmp(f, "--build-info")) { + } else if (!strcmp(f, "--build-info")) { embed_print_build_info(prog); return 0; - } - else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { n_threads = atoi(argv[++arg_start]); - } - else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); + } else if (!strcmp(f, "--json")) { + json = 1; } - else if (!strcmp(f, "--json")) { json = 1; } #ifdef USE_MLX else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg_start]); @@ -539,8 +547,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); return 1; } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { + } else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg_start]); if (mlx_quantize_group_size <= 0) { fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); @@ -551,28 +558,34 @@ int main(int argc, char *argv[]) #ifdef USE_CUDA else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg_start]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { + } else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg_start]; } #endif else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { print_embs = 1; + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "--stream")) { + stdin_mode = 1; } - else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } - else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "--stream")) { stdin_mode = 1; } #ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { use_mlx = 1; } + else if (!strcmp(f, "--mlx")) { + use_mlx = 1; + } #endif #ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { use_cuda = 1; } + else if (!strcmp(f, "--cuda")) { + use_cuda = 1; + } #endif else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(prog); return 0; - } - else break; + } else + break; arg_start++; } @@ -620,18 +633,22 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (use_mlx) { - if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + if (verbose >= 1) + fprintf(stderr, "Using MLX GPU backend\n"); } else #endif #ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); + if (use_cuda) { + if (verbose >= 1) + fprintf(stderr, "Using CUDA GPU backend\n"); } else #endif { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + if (n_threads <= 0) + n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + if (verbose >= 1) + fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } /* Tokenizer */ @@ -639,8 +656,12 @@ int main(int argc, char *argv[]) snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); double t0 = now_ms(); qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); - if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; } - if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); + if (!tok) { + fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); + return 1; + } + if (verbose >= 1) + fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); /* Model */ engine_t e = {0}; @@ -665,7 +686,7 @@ int main(int argc, char *argv[]) } else #endif #ifdef USE_CUDA - if (use_cuda) { + if (use_cuda) { e.cuda_ctx = embed_cuda_load(model_dir); if (!e.cuda_ctx) { fprintf(stderr, "failed to load CUDA model\n"); @@ -699,30 +720,33 @@ int main(int argc, char *argv[]) * wins; otherwise the released-model family constant applies. */ if (e.append_terminal_token) { int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>"); - e.terminal_token_id = - eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; } if (verbose >= 1) - fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", - e.dim, now_ms() - t0, + fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, #ifdef USE_MLX use_mlx ? " (MLX)" : #endif #ifdef USE_CUDA - use_cuda ? " (CUDA)" : + use_cuda ? " (CUDA)" + : #endif - ""); + ""); e.tok_ws = qwen_tokenizer_workspace_new(); if (!e.tok_ws) { fprintf(stderr, "failed to allocate tokenizer workspace\n"); - if (e.workspace) embed_workspace_free(e.workspace); - if (e.model) embed_model_free(e.model); + if (e.workspace) + embed_workspace_free(e.workspace); + if (e.model) + embed_model_free(e.model); #ifdef USE_MLX - if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx); + if (e.mlx_ctx) + embed_mlx_free(e.mlx_ctx); #endif #ifdef USE_CUDA - if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx); + if (e.cuda_ctx) + embed_cuda_free(e.cuda_ctx); #endif qwen_tokenizer_free(tok); return 1; @@ -736,13 +760,17 @@ int main(int argc, char *argv[]) rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json); /* Cleanup */ - if (e.workspace) embed_workspace_free(e.workspace); - if (e.model) embed_model_free(e.model); + if (e.workspace) + embed_workspace_free(e.workspace); + if (e.model) + embed_model_free(e.model); #ifdef USE_MLX - if (e.mlx_ctx) embed_mlx_free(e.mlx_ctx); + if (e.mlx_ctx) + embed_mlx_free(e.mlx_ctx); #endif #ifdef USE_CUDA - if (e.cuda_ctx) embed_cuda_free(e.cuda_ctx); + if (e.cuda_ctx) + embed_cuda_free(e.cuda_ctx); #endif qwen_tokenizer_workspace_free(e.tok_ws); qwen_tokenizer_free(tok); diff --git a/embed_cuda.cu b/embed_cuda.cu index 1ec0573..3955907 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -15,10 +15,10 @@ extern "C" { #include typedef struct { - float *d; // device buffer; holds __nv_bfloat16 elements when bf16 + float *d; // device buffer; holds __nv_bfloat16 elements when bf16 int rows; int cols; - int bf16; // 1 if d holds __nv_bfloat16, 0 if F32 + int bf16; // 1 if d holds __nv_bfloat16, 0 if F32 } cuda_matrix_t; typedef struct { @@ -47,7 +47,7 @@ struct embed_cuda_ctx { float *qkv; float *attn_out; float *ffn_gate_up; - void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) + void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) float *pooled_out; float *rope_cos; float *rope_sin; @@ -58,16 +58,16 @@ struct embed_cuda_ctx { float *kexp; float *vexp; float *attn_scores; - void *attn_probs; // BF16 softmax output (bf16 weights) + void *attn_probs; // BF16 softmax output (bf16 weights) long long attn_scores_elems; long long attn_probs_elems; // Equal-length micro-batch attention: pointer arrays for batched GEMMs, // built once per forward pass (layer-invariant buffer addresses). // attn_G is the number of sequences whose score tensors fit the scratch // budget at once; 0 disables the batched path for this forward. - const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads + const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads const void **attn_ptrs_host; - int attn_ptrs_cap; // capacity in pointer entries + int attn_ptrs_cap; // capacity in pointer entries int attn_G; int attn_L; int *span_starts; @@ -83,26 +83,25 @@ enum { CUDA_ATTN_SCORES_BUDGET = 64 * 1024 * 1024 }; // >=256; both paths are identical precision). enum { CUDA_ATTN_BATCHED_MAX_LEN = 192 }; -#define CUDA_CHECK(expr) do { \ - cudaError_t _e = (expr); \ - if (_e != cudaSuccess) { \ - fprintf(stderr, "cuda: %s failed: %s\n", #expr, \ - cudaGetErrorString(_e)); \ - return -1; \ - } \ -} while (0) - -#define CUBLAS_CHECK(expr) do { \ - cublasStatus_t _s = (expr); \ - if (_s != CUBLAS_STATUS_SUCCESS) { \ - fprintf(stderr, "cuda: %s failed: cublas status %d\n", \ - #expr, (int)_s); \ - return -1; \ - } \ -} while (0) - -static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) -{ +#define CUDA_CHECK(expr) \ + do { \ + cudaError_t _e = (expr); \ + if (_e != cudaSuccess) { \ + fprintf(stderr, "cuda: %s failed: %s\n", #expr, cudaGetErrorString(_e)); \ + return -1; \ + } \ + } while (0) + +#define CUBLAS_CHECK(expr) \ + do { \ + cublasStatus_t _s = (expr); \ + if (_s != CUBLAS_STATUS_SUCCESS) { \ + fprintf(stderr, "cuda: %s failed: cublas status %d\n", #expr, (int)_s); \ + return -1; \ + } \ + } while (0) + +static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) { if (!mode || !mode[0] || !strcmp(mode, "f32")) { *out = CUBLAS_COMPUTE_32F; return 0; @@ -125,8 +124,7 @@ static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; static int g_gemm_compute_set = 0; -int embed_cuda_set_fast_gemm(const char *mode) -{ +int embed_cuda_set_fast_gemm(const char *mode) { cublasComputeType_t ct; if (parse_gemm_compute(mode, &ct) != 0) return -1; @@ -135,15 +133,14 @@ int embed_cuda_set_fast_gemm(const char *mode) return 0; } -static void cuda_matrix_free(cuda_matrix_t *m) -{ - if (!m) return; +static void cuda_matrix_free(cuda_matrix_t *m) { + if (!m) + return; cudaFree(m->d); memset(m, 0, sizeof(*m)); } -static float bf16_to_f32(uint16_t v) -{ +static float bf16_to_f32(uint16_t v) { uint32_t u = ((uint32_t)v) << 16; float f; memcpy(&f, &u, sizeof(f)); @@ -152,13 +149,12 @@ static float bf16_to_f32(uint16_t v) // Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are // preserved. BF16 shares F32's 8-bit exponent, so this never overflows. -static uint16_t f32_to_bf16(float f) -{ +static uint16_t f32_to_bf16(float f) { uint32_t x; memcpy(&x, &f, sizeof(x)); - if ((x & 0x7fffffffu) > 0x7f800000u) // NaN + if ((x & 0x7fffffffu) > 0x7f800000u) // NaN return (uint16_t)((x >> 16) | 0x0040u); - x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even + x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even return (uint16_t)(x >> 16); } @@ -169,17 +165,15 @@ static uint16_t f32_to_bf16(float f) static int g_weights_bf16 = 0; static int g_weights_bf16_set = 0; -int embed_cuda_set_weights_bf16(int on) -{ +int embed_cuda_set_weights_bf16(int on) { g_weights_bf16 = on ? 1 : 0; g_weights_bf16_set = 1; return 0; } -static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, - size_t count) -{ - if (!dst || !w || !w->data) return -1; +static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) { + if (!dst || !w || !w->data) + return -1; if (w->dtype == DTYPE_F32) { memcpy(dst, w->data, count * sizeof(float)); return 0; @@ -195,13 +189,12 @@ static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, // Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is // set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16. -static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, - int rows, int cols) -{ +static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int rows, int cols) { cudaError_t e; if (g_weights_bf16) { uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t)); - if (!tmp16) return -1; + if (!tmp16) + return -1; for (size_t i = 0; i < count; i++) tmp16[i] = f32_to_bf16(src[i]); e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16)); @@ -210,8 +203,7 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, free(tmp16); return -1; } - e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), - cudaMemcpyHostToDevice); + e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice); free(tmp16); m->bf16 = 1; } else { @@ -233,13 +225,13 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, return 0; } -static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, - int rows, int cols) -{ - if (!m || !w || rows <= 0 || cols <= 0) return -1; +static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) { + if (!m || !w || rows <= 0 || cols <= 0) + return -1; size_t count = (size_t)rows * (size_t)cols; float *tmp = (float *)malloc(count * sizeof(float)); - if (!tmp) return -1; + if (!tmp) + return -1; if (copy_weight_host_f32(tmp, w, count) != 0) { free(tmp); return -1; @@ -249,17 +241,17 @@ static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, return r; } -static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, - const embed_config_t *c) -{ - if (!m || !src || !c) return -1; +static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { + if (!m || !src || !c) + return -1; int rows = c->q_dim + 2 * c->kv_dim; int cols = c->hidden_size; size_t q_count = (size_t)c->q_dim * cols; size_t kv_count = (size_t)c->kv_dim * cols; size_t total = q_count + 2 * kv_count; float *tmp = (float *)malloc(total * sizeof(float)); - if (!tmp) return -1; + if (!tmp) + return -1; if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 || copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 || copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) { @@ -271,16 +263,17 @@ static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, return r; } -static int load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, - const embed_config_t *c) -{ - if (!m || !src || !c) return -1; +static int +load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { + if (!m || !src || !c) + return -1; int rows = 2 * c->intermediate_size; int cols = c->hidden_size; size_t proj_count = (size_t)c->intermediate_size * cols; size_t total = 2 * proj_count; float *tmp = (float *)malloc(total * sizeof(float)); - if (!tmp) return -1; + if (!tmp) + return -1; if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 || copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) { free(tmp); @@ -291,9 +284,9 @@ static int load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, return r; } -static int load_vector(float **out, const float *src, int n) -{ - if (!out || !src || n <= 0) return -1; +static int load_vector(float **out, const float *src, int n) { + if (!out || !src || n <= 0) + return -1; cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float)); if (e != cudaSuccess) { fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e)); @@ -309,13 +302,12 @@ static int load_vector(float **out, const float *src, int n) return 0; } -__global__ static void embed_lookup_kernel(float *x, const int *ids, - const void *emb, int emb_bf16, - int total, int hidden, int vocab) -{ +__global__ static void embed_lookup_kernel( + float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int count = total * hidden; - if (idx >= count) return; + if (idx >= count) + return; int tok = idx / hidden; int d = idx - tok * hidden; int id = ids[tok]; @@ -324,20 +316,18 @@ __global__ static void embed_lookup_kernel(float *x, const int *ids, return; } size_t off = (size_t)id * hidden + d; - x[idx] = emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) - : ((const float *)emb)[off]; + x[idx] = + emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) : ((const float *)emb)[off]; } // Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs). -__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, - const float *in, size_t n) -{ +__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float *in, size_t n) { size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) out[i] = __float2bfloat16(in[i]); + if (i < n) + out[i] = __float2bfloat16(in[i]); } -__device__ static float block_sum(float v) -{ +__device__ static float block_sum(float v) { __shared__ float partial[8]; __shared__ float total; int lane = threadIdx.x & 31; @@ -346,36 +336,32 @@ __device__ static float block_sum(float v) for (int offset = 16; offset > 0; offset >>= 1) v += __shfl_down_sync(0xffffffff, v, offset); - if (lane == 0) partial[warp] = v; + if (lane == 0) + partial[warp] = v; __syncthreads(); if (warp == 0) { v = lane < warps ? partial[lane] : 0.0f; for (int offset = 16; offset > 0; offset >>= 1) v += __shfl_down_sync(0xffffffff, v, offset); - if (lane == 0) total = v; + if (lane == 0) + total = v; } __syncthreads(); return total; } -__device__ static inline void store_act(float *out, size_t i, float v) -{ - out[i] = v; -} +__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; } -__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) -{ +__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) { out[i] = __float2bfloat16(v); } // OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight // projection GEMM directly (same rounding as a separate cast kernel). template -__global__ static void rms_norm_kernel(OUT_T *out, const float *x, - const float *weight, int rows, - int dim, float eps) -{ +__global__ static void +rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int dim, float eps) { int row = blockIdx.x; int tid = threadIdx.x; float sum = 0.0f; @@ -386,24 +372,25 @@ __global__ static void rms_norm_kernel(OUT_T *out, const float *x, float total = block_sum(sum); float inv = rsqrtf(total / (float)dim + eps); for (int d = tid; d < dim; d += blockDim.x) - store_act(out, (size_t)row * dim + d, - x[(size_t)row * dim + d] * inv * weight[d]); + store_act(out, (size_t)row * dim + d, x[(size_t)row * dim + d] * inv * weight[d]); } // One grid covers Q and K: blockIdx.y < n_heads selects the Q head with // q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset. -__global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm, +__global__ static void rms_norm_rope_qk_kernel(float *x, + const float *q_norm, const float *k_norm, const int *positions, const float *cosv, const float *sinv, - int rows, int n_heads, + int rows, + int n_heads, int n_kv_heads, int head_dim, int row_stride, - int q_offset, int k_offset, - float eps) -{ + int q_offset, + int k_offset, + float eps) { __shared__ float partial[4]; __shared__ float total; int row = blockIdx.x; @@ -416,7 +403,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm, float sign = tid < half ? -1.0f : 1.0f; const float *weight = head < n_heads ? q_norm : k_norm; int base_offset = head < n_heads ? q_offset : k_offset; - if (head >= n_heads) head -= n_heads; + if (head >= n_heads) + head -= n_heads; size_t base = (size_t)row * row_stride + base_offset + head * head_dim; float v = tid < head_dim ? x[base + tid] : 0.0f; float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; @@ -424,13 +412,15 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm, for (int offset = 16; offset > 0; offset >>= 1) sum += __shfl_down_sync(0xffffffff, sum, offset); - if (lane == 0) partial[warp] = sum; + if (lane == 0) + partial[warp] = sum; __syncthreads(); if (warp == 0) { sum = lane < 4 ? partial[lane] : 0.0f; for (int offset = 16; offset > 0; offset >>= 1) sum += __shfl_down_sync(0xffffffff, sum, offset); - if (lane == 0) total = sum; + if (lane == 0) + total = sum; } __syncthreads(); @@ -449,10 +439,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, const float *q_norm, // over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16 // path writes a packed [rows x intermediate] BF16 matrix for the down GEMM. template -__global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, - const float *gate_up, int rows, - int intermediate) -{ +__global__ static void silu_mul_packed_kernel( + OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) { int i = blockIdx.x * blockDim.x + threadIdx.x; int n = rows * intermediate; if (i < n) { @@ -461,8 +449,7 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, size_t base = (size_t)row * (2 * intermediate); float g = gate_up[base + d]; float u = gate_up[base + intermediate + d]; - store_act(out, (size_t)row * out_stride + d, - (g / (1.0f + expf(-g))) * u); + store_act(out, (size_t)row * out_stride + d, (g / (1.0f + expf(-g))) * u); } } @@ -471,14 +458,21 @@ __global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, // so the attention GEMMs can stride uniformly over query heads. // V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly. template -__global__ static void attn_expand_kv_kernel( - float *kexp, V_T *vexp, const float *qkv, int total, int n_heads, - int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset) -{ +__global__ static void attn_expand_kv_kernel(float *kexp, + V_T *vexp, + const float *qkv, + int total, + int n_heads, + int n_kv_heads, + int head_dim, + int qkv_dim, + int k_offset, + int v_offset) { int q_dim = n_heads * head_dim; size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; size_t count = (size_t)total * q_dim; - if (idx >= count) return; + if (idx >= count) + return; int t = (int)(idx / q_dim); int rem = (int)(idx - (size_t)t * q_dim); int h = rem / head_dim; @@ -495,19 +489,20 @@ __global__ static void attn_expand_kv_kernel( // P_T is float with probs == scores (in place), or __nv_bfloat16 when the // probabilities feed a BF16 @V GEMM. template -__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) -{ +__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) { float *s = scores + (size_t)blockIdx.x * L; P_T *p = probs + (size_t)blockIdx.x * L; int tid = threadIdx.x; __shared__ float red[256]; float m = -3.402823466e+38F; - for (int j = tid; j < L; j += blockDim.x) m = fmaxf(m, s[j]); + for (int j = tid; j < L; j += blockDim.x) + m = fmaxf(m, s[j]); red[tid] = m; __syncthreads(); for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]); + if (tid < o) + red[tid] = fmaxf(red[tid], red[tid + o]); __syncthreads(); } m = red[0]; @@ -522,7 +517,8 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) red[tid] = sum; __syncthreads(); for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] += red[tid + o]; + if (tid < o) + red[tid] += red[tid + o]; __syncthreads(); } float inv = 1.0f / red[0]; @@ -531,9 +527,7 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) } template -__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, - int L) -{ +__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int L) { float *s = scores + (size_t)blockIdx.x * L; P_T *p = probs + (size_t)blockIdx.x * L; int query = blockIdx.x % L; @@ -542,11 +536,13 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, __shared__ float red[256]; float m = -3.402823466e+38F; - for (int j = tid; j < keys; j += blockDim.x) m = fmaxf(m, s[j]); + for (int j = tid; j < keys; j += blockDim.x) + m = fmaxf(m, s[j]); red[tid] = m; __syncthreads(); for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]); + if (tid < o) + red[tid] = fmaxf(red[tid], red[tid + o]); __syncthreads(); } m = red[0]; @@ -561,7 +557,8 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, red[tid] = sum; __syncthreads(); for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] += red[tid + o]; + if (tid < o) + red[tid] += red[tid + o]; __syncthreads(); } float inv = 1.0f / red[0]; @@ -569,10 +566,8 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f); } -__global__ static void mean_pool_kernel(float *out, const float *x, - const int *offsets, int batch, - int hidden) -{ +__global__ static void +mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { int b = blockIdx.x; int tid = threadIdx.x; int start = offsets[b]; @@ -586,10 +581,8 @@ __global__ static void mean_pool_kernel(float *out, const float *x, } } -__global__ static void last_pool_kernel(float *out, const float *x, - const int *offsets, int batch, - int hidden) -{ +__global__ static void +last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { int b = blockIdx.x; int tid = threadIdx.x; int last = offsets[b + 1] - 1; @@ -597,10 +590,8 @@ __global__ static void last_pool_kernel(float *out, const float *x, out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d]; } -__global__ static void span_pool_kernel(float *out, const float *x, - const int *starts, const int *lens, - int n_spans, int hidden) -{ +__global__ static void span_pool_kernel( + float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) { int s = blockIdx.x; int tid = threadIdx.x; int start = starts[s]; @@ -613,8 +604,7 @@ __global__ static void span_pool_kernel(float *out, const float *x, } } -static int launch_check(void) -{ +static int launch_check(void) { cudaError_t e = cudaGetLastError(); if (e != cudaSuccess) { fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e)); @@ -627,8 +617,7 @@ static int launch_check(void) // EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core // matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands // internally. -static cublasComputeType_t gemm_compute(void) -{ +static cublasComputeType_t gemm_compute(void) { static int env_init = 0; if (!g_gemm_compute_set && !env_init) { const char *e = getenv("EMBED_CUDA_GEMM_MODE"); @@ -644,80 +633,100 @@ static cublasComputeType_t gemm_compute(void) // already produced in BF16 by the upstream kernel, otherwise the F32 operand // is cast into ctx->act_bf16 first. With F32 weights this is the exact path, // honoring gemm_compute(). -static int linear_ex(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const void *x, int x_is_bf16, float *y, int rows, - int in_dim, int out_dim, int x_stride, float beta) -{ +static int linear_ex(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const void *x, + int x_is_bf16, + float *y, + int rows, + int in_dim, + int out_dim, + int x_stride, + float beta) { const float alpha = 1.0f; if (w->bf16) { const void *xb = x; if (!x_is_bf16) { size_t n = (size_t)rows * (size_t)x_stride; int threads = 256; - cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, - ctx->stream>>>( + cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, ctx->stream>>>( (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n); - if (launch_check() != 0) return -1; + if (launch_check() != 0) + return -1; xb = ctx->act_bf16; } - CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, CUDA_R_16BF, in_dim, - xb, CUDA_R_16BF, x_stride, - &beta, y, CUDA_R_32F, out_dim, - CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, + &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride, + &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F, + CUBLAS_GEMM_DEFAULT)); return 0; } - CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, CUDA_R_32F, in_dim, - x, CUDA_R_32F, x_stride, - &beta, y, CUDA_R_32F, out_dim, - gemm_compute(), CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha, + w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y, + CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT)); return 0; } -static int linear(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const float *x, float *y, int rows, int in_dim, int out_dim) -{ +static int linear(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const float *x, + float *y, + int rows, + int in_dim, + int out_dim) { return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f); } // x points at activations already stored as BF16 (e.g. ctx->act_bf16). -static int linear_bf16x(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const void *x, float *y, int rows, int in_dim, - int out_dim, int x_stride, float beta) -{ +static int linear_bf16x(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const void *x, + float *y, + int rows, + int in_dim, + int out_dim, + int x_stride, + float beta) { return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta); } -static int linear_accum(embed_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const float *x, float *y, - int rows, int in_dim, int out_dim) -{ +static int linear_accum(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const float *x, + float *y, + int rows, + int in_dim, + int out_dim) { return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f); } -static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, - cublasOperation_t transb, int m, int n, int k, - const float *alpha, const float *A, int lda, - long long strideA, const float *B, int ldb, - long long strideB, const float *beta, - float *C, int ldc, long long strideC, - int batch_count) -{ +static int gemm_strided_batched(cublasHandle_t blas, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, + const float *A, + int lda, + long long strideA, + const float *B, + int ldb, + long long strideB, + const float *beta, + float *C, + int ldc, + long long strideC, + int batch_count) { cublasComputeType_t ct = gemm_compute(); if (ct == CUBLAS_COMPUTE_32F) { - CUBLAS_CHECK(cublasSgemmStridedBatched( - blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, - strideB, beta, C, ldc, strideC, batch_count)); + CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda, + strideA, B, ldb, strideB, beta, C, ldc, strideC, + batch_count)); } else { CUBLAS_CHECK(cublasGemmStridedBatchedEx( - blas, transa, transb, m, n, k, - alpha, A, CUDA_R_32F, lda, strideA, - B, CUDA_R_32F, ldb, strideB, - beta, C, CUDA_R_32F, ldc, strideC, - batch_count, ct, CUBLAS_GEMM_DEFAULT)); + blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb, + strideB, beta, C, CUDA_R_32F, ldc, strideC, batch_count, ct, CUBLAS_GEMM_DEFAULT)); } return 0; } @@ -733,22 +742,26 @@ static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, // layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged // batches, single sequences, or when one sequence's scores exceed the // budget. -static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, - int qkv_dim) -{ +static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) { const embed_config_t *c = &ctx->config; ctx->attn_G = 0; - if (batch < 2) return 0; + if (batch < 2) + return 0; int L = ctx->offsets_host[1] - ctx->offsets_host[0]; - if (L > CUDA_ATTN_BATCHED_MAX_LEN) return 0; + if (L > CUDA_ATTN_BATCHED_MAX_LEN) + return 0; for (int b = 1; b < batch; b++) - if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L) return 0; + if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L) + return 0; int H = c->n_heads, hd = c->head_dim, q_dim = c->q_dim; long long per = (long long)H * L * L; - if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2) return 0; + if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2) + return 0; int G = (int)(CUDA_ATTN_SCORES_BUDGET / per); - if (G > batch) G = batch; - if (G < 2) return 0; + if (G > batch) + G = batch; + if (G < 2) + return 0; int bf16 = g_weights_bf16; long long want = (long long)G * per; @@ -756,16 +769,14 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; ctx->attn_scores_elems = 0; - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, - (size_t)want * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)want * sizeof(float))); ctx->attn_scores_elems = want; } if (bf16 && want > ctx->attn_probs_elems) { cudaFree(ctx->attn_probs); ctx->attn_probs = NULL; ctx->attn_probs_elems = 0; - CUDA_CHECK(cudaMalloc(&ctx->attn_probs, - (size_t)want * sizeof(__nv_bfloat16))); + CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)want * sizeof(__nv_bfloat16))); ctx->attn_probs_elems = want; } @@ -775,11 +786,10 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, free((void *)ctx->attn_ptrs_host); ctx->attn_ptrs = NULL; ctx->attn_ptrs_cap = 0; - ctx->attn_ptrs_host = - (const void **)malloc((size_t)entries * sizeof(void *)); - if (!ctx->attn_ptrs_host) return -1; - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, - (size_t)entries * sizeof(void *))); + ctx->attn_ptrs_host = (const void **)malloc((size_t)entries * sizeof(void *)); + if (!ctx->attn_ptrs_host) + return -1; + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, (size_t)entries * sizeof(void *))); ctx->attn_ptrs_cap = entries; } @@ -813,8 +823,7 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, } } CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host, - (size_t)entries * sizeof(void *), - cudaMemcpyHostToDevice)); + (size_t)entries * sizeof(void *), cudaMemcpyHostToDevice)); ctx->attn_G = G; ctx->attn_L = L; return 0; @@ -823,11 +832,15 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, // out_bf16 != NULL selects the BF16-weight layout: V and the softmax // probabilities are stored BF16 and the @V GEMM emits BF16 attention output // at out_bf16 (stride q_dim), ready for the wo projection without a cast. -static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, - int batch, int q_offset, int k_offset, - int v_offset, int qkv_dim, float scale, - __nv_bfloat16 *out_bf16) -{ +static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, + const int *h_offsets, + int batch, + int q_offset, + int k_offset, + int v_offset, + int qkv_dim, + float scale, + __nv_bfloat16 *out_bf16) { const embed_config_t *c = &ctx->config; int q_dim = c->q_dim; int hd = c->head_dim; @@ -839,17 +852,16 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, int threads = 256; size_t exp_count = (size_t)total * q_dim; if (out_bf16) { - attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, - 0, ctx->stream>>>( - ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, - c->n_kv_heads, hd, qkv_dim, k_offset, v_offset); + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, + k_offset, v_offset); } else { - attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, - 0, ctx->stream>>>( - ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, - qkv_dim, k_offset, v_offset); + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset, + v_offset); } - if (launch_check() != 0) return -1; + if (launch_check() != 0) + return -1; if (ctx->attn_G >= 2) { int G = ctx->attn_G, L = ctx->attn_L; @@ -860,44 +872,37 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, const void *const *P = V + (size_t)batch * H; void *const *O = (void *const *)(P + (size_t)batch * H); cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F; - cublasComputeType_t av_ct = - out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute(); + cublasComputeType_t av_ct = out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute(); for (int s = 0; s < batch; s += G) { int g = batch - s < G ? batch - s : G; int n = g * H; size_t off = (size_t)s * H; /* scores = scale * Q @ K^T, batched over (seq, head) */ - CUBLAS_CHECK(cublasGemmBatchedEx( - ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, - &alpha, K + off, CUDA_R_32F, q_dim, - Q + off, CUDA_R_32F, qkv_dim, - &beta, S + off, CUDA_R_32F, L, - n, gemm_compute(), CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, + K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F, + qkv_dim, &beta, S + off, CUDA_R_32F, L, n, + gemm_compute(), CUBLAS_GEMM_DEFAULT)); if (out_bf16) { if (causal) attn_causal_softmax_kernel<<stream>>>( - ctx->attn_scores, - (__nv_bfloat16 *)ctx->attn_probs, L); + ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L); else attn_softmax_kernel<<stream>>>( - ctx->attn_scores, - (__nv_bfloat16 *)ctx->attn_probs, L); + ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L); } else { if (causal) - attn_causal_softmax_kernel<<stream>>>( - ctx->attn_scores, ctx->attn_scores, L); + attn_causal_softmax_kernel<<stream>>>(ctx->attn_scores, + ctx->attn_scores, L); else - attn_softmax_kernel<<stream>>>( - ctx->attn_scores, ctx->attn_scores, L); + attn_softmax_kernel<<stream>>>(ctx->attn_scores, + ctx->attn_scores, L); } - if (launch_check() != 0) return -1; + if (launch_check() != 0) + return -1; /* O = P @ V, batched over (seq, head) */ - CUBLAS_CHECK(cublasGemmBatchedEx( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V + off, pv, q_dim, - P + off, pv, L, - &beta, O + off, pv, q_dim, - n, av_ct, CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, + V + off, pv, q_dim, P + off, pv, L, &beta, O + off, pv, + q_dim, n, av_ct, CUBLAS_GEMM_DEFAULT)); } return 0; } @@ -908,53 +913,47 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; const float *K = ctx->kexp + (size_t)start * q_dim; /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ - if (gemm_strided_batched( - ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, - &alpha, K, q_dim, hd, Q, qkv_dim, hd, - &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) + if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, + hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L, + H) != 0) return -1; if (out_bf16) { __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs; - const __nv_bfloat16 *V = - (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim; + const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim; __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim; if (causal) - attn_causal_softmax_kernel<<stream>>>( - ctx->attn_scores, P, L); + attn_causal_softmax_kernel<<stream>>>(ctx->attn_scores, P, L); else - attn_softmax_kernel<<stream>>>( - ctx->attn_scores, P, L); - if (launch_check() != 0) return -1; + attn_softmax_kernel<<stream>>>(ctx->attn_scores, P, L); + if (launch_check() != 0) + return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ CUBLAS_CHECK(cublasGemmStridedBatchedEx( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V, CUDA_R_16BF, q_dim, hd, - P, CUDA_R_16BF, L, (long long)L * L, - &beta, O, CUDA_R_16BF, q_dim, hd, - H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P, + CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); } else { const float *V = ctx->vexp + (size_t)start * q_dim; float *O = ctx->attn_out + (size_t)start * q_dim; if (causal) - attn_causal_softmax_kernel<<stream>>>( - ctx->attn_scores, ctx->attn_scores, L); + attn_causal_softmax_kernel<<stream>>>(ctx->attn_scores, + ctx->attn_scores, L); else - attn_softmax_kernel<<stream>>>( - ctx->attn_scores, ctx->attn_scores, L); - if (launch_check() != 0) return -1; + attn_softmax_kernel<<stream>>>(ctx->attn_scores, + ctx->attn_scores, L); + if (launch_check() != 0) + return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ - if (gemm_strided_batched( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V, q_dim, hd, ctx->attn_scores, L, - (long long)L * L, &beta, O, q_dim, hd, H) != 0) + if (gemm_strided_batched(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, q_dim, + hd, ctx->attn_scores, L, (long long)L * L, &beta, O, q_dim, hd, + H) != 0) return -1; } } return 0; } -static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) -{ +static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) { const embed_config_t *c = &ctx->config; if (total > ctx->seq_cap) { cudaFree(ctx->x); @@ -972,9 +971,11 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s ctx->token_ids = ctx->positions = NULL; ctx->kexp = ctx->vexp = NULL; ctx->act_bf16 = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, + (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, (size_t)total * 2 * c->intermediate_size * sizeof(float))); @@ -983,8 +984,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); if (g_weights_bf16) - CUDA_CHECK(cudaMalloc(&ctx->act_bf16, - (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16))); + CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size * + sizeof(__nv_bfloat16))); ctx->seq_cap = total; } if (batch + 1 > ctx->batch_cap) { @@ -992,7 +993,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s free(ctx->offsets_host); CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int)); - if (!ctx->offsets_host) return -1; + if (!ctx->offsets_host) + return -1; ctx->batch_cap = batch + 1; } if (max_seq > ctx->max_seq_cap) { @@ -1003,17 +1005,14 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; ctx->attn_scores_elems = 0; - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, - (size_t)score_elems * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float))); ctx->attn_scores_elems = score_elems; } if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) { cudaFree(ctx->attn_probs); ctx->attn_probs = NULL; ctx->attn_probs_elems = 0; - CUDA_CHECK(cudaMalloc(&ctx->attn_probs, - (size_t)score_elems * - sizeof(__nv_bfloat16))); + CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(__nv_bfloat16))); ctx->attn_probs_elems = score_elems; } float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); @@ -1026,8 +1025,7 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s int half = c->head_dim / 2; for (int pos = 0; pos < max_seq; pos++) { for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(c->rope_theta, - (float)(2 * d) / (float)c->head_dim); + float inv = 1.0f / powf(c->rope_theta, (float)(2 * d) / (float)c->head_dim); float angle = (float)pos * inv; float cv = cosf(angle); float sv = sinf(angle); @@ -1037,10 +1035,10 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s hsin[(size_t)pos * c->head_dim + half + d] = sv; } } - CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos, - (size_t)max_seq * c->head_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin, - (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float))); CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, (size_t)max_seq * c->head_dim * sizeof(float), cudaMemcpyHostToDevice, ctx->stream)); @@ -1054,50 +1052,43 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s return 0; } -static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) -{ - if (rows <= ctx->pooled_rows_cap) return 0; +static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) { + if (rows <= ctx->pooled_rows_cap) + return 0; const embed_config_t *c = &ctx->config; cudaFree(ctx->pooled_out); ctx->pooled_out = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, - (size_t)rows * c->hidden_size * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float))); ctx->pooled_rows_cap = rows; return 0; } -static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) -{ - if (n_spans <= ctx->span_cap) return 0; +static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) { + if (n_spans <= ctx->span_cap) + return 0; cudaFree(ctx->span_starts); cudaFree(ctx->span_lens); ctx->span_starts = NULL; ctx->span_lens = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, - (size_t)n_spans * sizeof(int))); - CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, - (size_t)n_spans * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, (size_t)n_spans * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, (size_t)n_spans * sizeof(int))); ctx->span_cap = n_spans; return 0; } -static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, - const embed_config_t *c) -{ - return - load_qkv_matrix(&dst->qkv, src, c) || - load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || - load_vector(&dst->q_norm, src->q_norm, c->head_dim) || - load_vector(&dst->k_norm, src->k_norm, c->head_dim) || - load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || - load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || - load_gate_up_matrix(&dst->gate_up_proj, src, c) || - load_matrix(&dst->down_proj, &src->down_proj, - c->hidden_size, c->intermediate_size); +static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { + return load_qkv_matrix(&dst->qkv, src, c) || + load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || + load_vector(&dst->q_norm, src->q_norm, c->head_dim) || + load_vector(&dst->k_norm, src->k_norm, c->head_dim) || + load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || + load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || + load_gate_up_matrix(&dst->gate_up_proj, src, c) || + load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size); } -static void free_layer(cuda_layer_t *l) -{ +static void free_layer(cuda_layer_t *l) { cuda_matrix_free(&l->qkv); cuda_matrix_free(&l->wo); cudaFree(l->q_norm); @@ -1108,10 +1099,10 @@ static void free_layer(cuda_layer_t *l) cuda_matrix_free(&l->down_proj); } -embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) -{ +embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { embed_model_t *cpu = embed_model_load(model_dir); - if (!cpu) return NULL; + if (!cpu) + return NULL; embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx)); if (!ctx) { @@ -1144,8 +1135,8 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) embed_cuda_free(ctx); return NULL; } - if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, - c->vocab_size, c->hidden_size) != 0 || + if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size, + c->hidden_size) != 0 || load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { embed_cuda_free(ctx); return NULL; @@ -1161,9 +1152,9 @@ embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) return ctx; } -void embed_cuda_free(embed_cuda_ctx_t *ctx) -{ - if (!ctx) return; +void embed_cuda_free(embed_cuda_ctx_t *ctx) { + if (!ctx) + return; cuda_matrix_free(&ctx->embed_tokens); if (ctx->layers) { for (int i = 0; i < ctx->config.n_layers; i++) @@ -1192,24 +1183,25 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) free((void *)ctx->attn_ptrs_host); cudaFree(ctx->span_starts); cudaFree(ctx->span_lens); - if (ctx->blas) cublasDestroy(ctx->blas); - if (ctx->stream) cudaStreamDestroy(ctx->stream); + if (ctx->blas) + cublasDestroy(ctx->blas); + if (ctx->stream) + cudaStreamDestroy(ctx->stream); embed_model_free(ctx->cpu); free(ctx); } -const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) -{ +const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } -static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, - int batch) -{ - if (!ctx || !inputs || batch <= 0) return -1; +static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) { + if (!ctx || !inputs || batch <= 0) + return -1; const embed_config_t *c = &ctx->config; int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); - if (!h_offsets) return -1; + if (!h_offsets) + return -1; int total = 0, max_seq = 0; for (int b = 0; b < batch; b++) { if (!inputs[b].ids || inputs[b].n_tokens <= 0) { @@ -1218,40 +1210,43 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs } h_offsets[b] = total; total += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; } h_offsets[batch] = total; int *h_ids = (int *)malloc((size_t)total * sizeof(int)); int *h_pos = (int *)malloc((size_t)total * sizeof(int)); if (!h_offsets || !h_ids || !h_pos) { - free(h_offsets); free(h_ids); free(h_pos); + free(h_offsets); + free(h_ids); + free(h_pos); return -1; } for (int b = 0; b < batch; b++) { int off = h_offsets[b]; - memcpy(h_ids + off, inputs[b].ids, - (size_t)inputs[b].n_tokens * sizeof(int)); + memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int)); for (int i = 0; i < inputs[b].n_tokens; i++) h_pos[off + i] = i; } if (ensure_buffers(ctx, total, batch, max_seq) != 0) { - free(h_offsets); free(h_ids); free(h_pos); + free(h_offsets); + free(h_ids); + free(h_pos); return -1; } memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int)); - cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, - (size_t)total * sizeof(int), + cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, (size_t)total * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->positions, h_pos, - (size_t)total * sizeof(int), + ce = cudaMemcpyAsync(ctx->positions, h_pos, (size_t)total * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->offsets, h_offsets, - (size_t)(batch + 1) * sizeof(int), + ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); - free(h_offsets); free(h_ids); free(h_pos); + free(h_offsets); + free(h_ids); + free(h_pos); if (ce != cudaSuccess) { fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); return -1; @@ -1260,9 +1255,10 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs int threads = 256; int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; embed_lookup_kernel<<stream>>>( - ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, - total, c->hidden_size, c->vocab_size); - if (launch_check() != 0) return -1; + ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size, + c->vocab_size); + if (launch_check() != 0) + return -1; float scale = 1.0f / sqrtf((float)c->head_dim); int q_offset = 0; @@ -1280,110 +1276,106 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs cuda_layer_t *l = &ctx->layers[layer]; int wbf16 = l->qkv.bf16; if (wbf16) { - rms_norm_kernel<<stream>>>( - act, ctx->x, l->input_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, - total, c->hidden_size, qkv_dim, + rms_norm_kernel<<stream>>>(act, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, total, c->hidden_size, qkv_dim, c->hidden_size, 0.0f) != 0) return -1; } else { - rms_norm_kernel<<stream>>>( - ctx->x_norm, ctx->x, l->input_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, - total, c->hidden_size, qkv_dim) != 0) + rms_norm_kernel<<stream>>>(ctx->x_norm, ctx->x, l->input_norm, + total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0) return -1; } - rms_norm_rope_qk_kernel<<n_heads + c->n_kv_heads), - 128, 0, ctx->stream>>>( - ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, - ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim, - qkv_dim, q_offset, k_offset, c->rms_norm_eps); - if (launch_check() != 0) return -1; + rms_norm_rope_qk_kernel<<n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>( + ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total, + c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->rms_norm_eps); + if (launch_check() != 0) + return -1; - if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, - k_offset, v_offset, qkv_dim, scale, - wbf16 ? act : NULL) != 0) + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, + qkv_dim, scale, wbf16 ? act : NULL) != 0) return -1; if (wbf16) { - if (linear_bf16x(ctx, &l->wo, act, ctx->x, - total, c->q_dim, c->hidden_size, - c->q_dim, 1.0f) != 0) + if (linear_bf16x(ctx, &l->wo, act, ctx->x, total, c->q_dim, c->hidden_size, c->q_dim, + 1.0f) != 0) return -1; } else { - if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, - total, c->q_dim, c->hidden_size) != 0) + if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, c->hidden_size) != + 0) return -1; } int inter_count = total * c->intermediate_size; if (wbf16) { - rms_norm_kernel<<stream>>>( - act, ctx->x, l->post_attn_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, - total, c->hidden_size, 2 * c->intermediate_size, - c->hidden_size, 0.0f) != 0) + rms_norm_kernel<<stream>>>(act, ctx->x, l->post_attn_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, total, c->hidden_size, + 2 * c->intermediate_size, c->hidden_size, 0.0f) != 0) return -1; - silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, - threads, 0, ctx->stream>>>( - act, c->intermediate_size, ctx->ffn_gate_up, total, - c->intermediate_size); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, - total, c->intermediate_size, c->hidden_size, - c->intermediate_size, 1.0f) != 0) + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0, + ctx->stream>>>(act, c->intermediate_size, ctx->ffn_gate_up, + total, c->intermediate_size); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, total, c->intermediate_size, + c->hidden_size, c->intermediate_size, 1.0f) != 0) return -1; } else { - rms_norm_kernel<<stream>>>( - ctx->x_norm, ctx->x, l->post_attn_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear(ctx, &l->gate_up_proj, ctx->x_norm, - ctx->ffn_gate_up, total, c->hidden_size, + rms_norm_kernel<<stream>>>(ctx->x_norm, ctx->x, l->post_attn_norm, + total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear(ctx, &l->gate_up_proj, ctx->x_norm, ctx->ffn_gate_up, total, c->hidden_size, 2 * c->intermediate_size) != 0) return -1; - silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, - threads, 0, ctx->stream>>>( - ctx->ffn_gate_up, 2 * c->intermediate_size, ctx->ffn_gate_up, - total, c->intermediate_size); - if (launch_check() != 0) return -1; - if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, - total, c->intermediate_size, c->hidden_size, - 2 * c->intermediate_size, 1.0f) != 0) + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0, + ctx->stream>>>(ctx->ffn_gate_up, 2 * c->intermediate_size, + ctx->ffn_gate_up, total, c->intermediate_size); + if (launch_check() != 0) + return -1; + if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total, + c->intermediate_size, c->hidden_size, 2 * c->intermediate_size, + 1.0f) != 0) return -1; } } - rms_norm_kernel<<stream>>>( - ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; + rms_norm_kernel<<stream>>>(ctx->x, ctx->x, ctx->norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; return 0; } -int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; +int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; const embed_config_t *c = &ctx->config; if (cuda_forward_batch(ctx, inputs, batch) != 0) return -1; if (ensure_pooled_rows(ctx, batch) != 0) return -1; if (c->pooling_mode == EMBED_POOL_LAST_TOKEN) - last_pool_kernel<<stream>>>( - ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); + last_pool_kernel<<stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); else - mean_pool_kernel<<stream>>>( - ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); - if (launch_check() != 0) return -1; + mean_pool_kernel<<stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); + if (launch_check() != 0) + return -1; cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, (size_t)batch * c->hidden_size * sizeof(float), cudaMemcpyDeviceToHost, ctx->stream); @@ -1395,9 +1387,8 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, } if (c->normalize_embeddings) { for (int b = 0; b < batch; b++) { - if (embed_l2_normalize( - out_embeddings + (size_t)b * c->hidden_size, - c->hidden_size) != 0) + if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) != + 0) return -1; } } @@ -1405,25 +1396,24 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, } int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; const embed_config_t *c = &ctx->config; int total_spans = 0; int total_tokens = 0; for (int b = 0; b < batch; b++) { const embed_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans || - total_tokens > INT_MAX - n_tokens) + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens) return -1; for (int s = 0; s < input->n_spans; s++) { int start = input->spans[s].start; int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || - len > n_tokens - start) + if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) return -1; } total_spans += input->n_spans; @@ -1457,23 +1447,19 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, cudaError_t ce; if (cuda_forward_batch(ctx, packed, batch) != 0) goto cleanup; - if (ensure_pooled_rows(ctx, total_spans) != 0 || - ensure_span_buffers(ctx, total_spans) != 0) + if (ensure_pooled_rows(ctx, total_spans) != 0 || ensure_span_buffers(ctx, total_spans) != 0) goto cleanup; - ce = cudaMemcpyAsync(ctx->span_starts, h_starts, - (size_t)total_spans * sizeof(int), + ce = cudaMemcpyAsync(ctx->span_starts, h_starts, (size_t)total_spans * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->span_lens, h_lens, - (size_t)total_spans * sizeof(int), + ce = cudaMemcpyAsync(ctx->span_lens, h_lens, (size_t)total_spans * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); if (ce != cudaSuccess) { fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce)); goto cleanup; } span_pool_kernel<<stream>>>( - ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, - total_spans, c->hidden_size); + ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, total_spans, c->hidden_size); if (launch_check() != 0) goto cleanup; ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, @@ -1482,8 +1468,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, if (ce == cudaSuccess) ce = cudaStreamSynchronize(ctx->stream); if (ce != cudaSuccess) { - fprintf(stderr, "cuda span output copy failed: %s\n", - cudaGetErrorString(ce)); + fprintf(stderr, "cuda span output copy failed: %s\n", cudaGetErrorString(ce)); goto cleanup; } rc = 0; @@ -1495,20 +1480,21 @@ cleanup: return rc; } -int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - embed_input_t input = { token_ids, n_tokens }; +int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; return embed_cuda_encode_batch(ctx, &input, 1, out_embedding); } -float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens) -{ - if (!ctx) return NULL; +float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) { + if (!ctx) + return NULL; int hidden = ctx->config.hidden_size; float *out = (float *)malloc((size_t)hidden * sizeof(float)); - if (!out) return NULL; + if (!out) + return NULL; if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) { free(out); return NULL; diff --git a/embed_cuda.h b/embed_cuda.h index b3d2c3c..b3c9582 100644 --- a/embed_cuda.h +++ b/embed_cuda.h @@ -28,14 +28,18 @@ EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx); EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx); -EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding); -EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens); -EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, - int batch, float *out_embeddings); +EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding); +EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens); +EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings); EMBED_API int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, float *out_embeddings); + const embed_context_input_t *inputs, + int batch, + float *out_embeddings); #endif /* EMBED_CUDA_H */ diff --git a/embed_internal.h b/embed_internal.h index 60fc581..d2c1a30 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -29,42 +29,50 @@ typedef struct { const float *norm; /* [hidden] */ } embed_weights_t; -typedef void (*embed_attention_fn)( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes); +typedef void (*embed_attention_fn)(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); struct embed_model { - embed_config_t config; + embed_config_t config; embed_weights_t weights; - void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ + void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ embed_attention_fn attention; }; struct embed_workspace { - const embed_model_t *model; /* immutable model this workspace belongs to */ - int buf_seq_cap; - float *x; /* [seq, hidden] */ - float *x_norm; /* [seq, hidden] */ - float *q; /* [seq, q_dim] */ - float *k; /* [seq, kv_dim] */ - float *v; /* [seq, kv_dim] */ - float *attn_out; /* [seq, q_dim] */ - float *proj_out; /* [seq, hidden] */ - float *ffn_gate; /* [seq, intermediate] */ - float *ffn_up; /* [seq, intermediate] */ - float *attn_scores; /* reusable BLAS attention score tiles */ + const embed_model_t *model; /* immutable model this workspace belongs to */ + int buf_seq_cap; + float *x; /* [seq, hidden] */ + float *x_norm; /* [seq, hidden] */ + float *q; /* [seq, q_dim] */ + float *k; /* [seq, kv_dim] */ + float *v; /* [seq, kv_dim] */ + float *attn_out; /* [seq, q_dim] */ + float *proj_out; /* [seq, hidden] */ + float *ffn_gate; /* [seq, intermediate] */ + float *ffn_up; /* [seq, intermediate] */ + float *attn_scores; /* reusable BLAS attention score tiles */ size_t attn_scores_bytes; - int *offsets; /* [batch + 1] packed sequence offsets */ - int offsets_cap; + int *offsets; /* [batch + 1] packed sequence offsets */ + int offsets_cap; /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */ float *rope_cos; float *rope_sin; - int rope_cache_cap; + int rope_cache_cap; /* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the * largest projection. Workspace-owned so concurrent workspaces on one diff --git a/embed_mlx.c b/embed_mlx.c index cb3b737..9f20026 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -35,12 +35,12 @@ typedef struct { struct embed_mlx_ctx { embed_config_t config; mlx_array embed_tokens; - mlx_layer_t *layers; /* heap-allocated [n_layers] */ + mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; mlx_stream stream; multi_safetensors_t *ms; - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ int quantize_bits; int quantize_group_size; mlx_dtype weight_dtype; @@ -55,7 +55,7 @@ struct embed_mlx_late_ctx { struct embed_mlx_late_vectors { embed_mlx_late_ctx_t *owner; - mlx_array vectors; /* [tokens, dim] */ + mlx_array vectors; /* [tokens, dim] */ int tokens; int dim; }; @@ -66,47 +66,46 @@ struct embed_mlx_late_vectors { static int arr_ok(mlx_array a) { return a.ctx != NULL; } -static int mlx_mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; +static int mlx_mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; *out = a * b; return 0; } -static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) -{ - return ctx->weight_dtype; -} +static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; } -static size_t safetensor_dtype_size(safetensor_dtype_t dtype) -{ +static size_t safetensor_dtype_size(safetensor_dtype_t dtype) { switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; + case DTYPE_F32: + return sizeof(float); + case DTYPE_BF16: + return sizeof(uint16_t); + default: + return 0; } } static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, - const char *name, const int *shape, - int ndim) -{ + const char *name, + const int *shape, + int ndim) { size_t elem_size = safetensor_dtype_size(t->dtype); if (elem_size == 0) { fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); return 0; } if (t->ndim != ndim) { - fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); + fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } size_t numel = 1; for (int i = 0; i < ndim; i++) { if (shape[i] <= 0 || t->shape[i] != shape[i]) { - fprintf(stderr, "mlx: bad shape for %s at dim %d: " + fprintf(stderr, + "mlx: bad shape for %s at dim %d: " "got %lld, expected %d\n", name, i, (long long)t->shape[i], shape[i]); return 0; @@ -123,15 +122,15 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, return 0; } if (t->data_size != bytes) { - fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); + fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + bytes); return 0; } - if (sf->header_size > SIZE_MAX - 8) return 0; + if (sf->header_size > SIZE_MAX - 8) + return 0; size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); return 0; @@ -139,11 +138,10 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, - const int *shape, int ndim) -{ +static mlx_array +load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { fprintf(stderr, "mlx: tensor not found: %s\n", name); return (mlx_array){0}; @@ -162,22 +160,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); } -static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, - const char *name, - const int *shape, int ndim) -{ +static mlx_linear_t +load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { mlx_linear_t l = {0}; l.w = load_tensor(ms, name, shape, ndim); return l; } -static int linear_ok(mlx_linear_t l) -{ - return arr_ok(l.w) || arr_ok(l.wt); -} +static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); } -static void free_linear(mlx_linear_t *l) -{ +static void free_linear(mlx_linear_t *l) { mlx_array_free(l->w); mlx_array_free(l->wt); mlx_array_free(l->scales); @@ -185,8 +177,7 @@ static void free_linear(mlx_linear_t *l) memset(l, 0, sizeof(*l)); } -static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) -{ +static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) { if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) return 0; @@ -214,15 +205,15 @@ static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) return 0; } -static int quantize_linear(mlx_linear_t *l, int bits, int group_size, - mlx_stream s) -{ - if (!l || !arr_ok(l->w) || bits == 0) return 0; +static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) { + if (!l || !arr_ok(l->w) || bits == 0) + return 0; if (bits != 8) { fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); return -1; } - if (group_size <= 0) group_size = 64; + if (group_size <= 0) + group_size = 64; mlx_optional_int gs = {.value = group_size, .has_value = true}; mlx_optional_int qb = {.value = bits, .has_value = true}; @@ -253,7 +244,8 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size, } mlx_array_eval(qw); mlx_array_eval(scales); - if (arr_ok(biases)) mlx_array_eval(biases); + if (arr_ok(biases)) + mlx_array_eval(biases); mlx_synchronize(s); mlx_array_free(l->w); @@ -269,16 +261,15 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size, } /* y = x @ W^T */ -static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) -{ +static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { if (W->quantized) { - if (!arr_ok(W->w)) return (mlx_array){0}; + if (!arr_ok(W->w)) + return (mlx_array){0}; mlx_array res = mlx_array_new(); mlx_optional_int gs = {.value = W->group_size, .has_value = true}; mlx_optional_int bits = {.value = W->bits, .has_value = true}; mlx_array null_arr = (mlx_array){0}; - mlx_quantized_matmul(&res, x, W->w, W->scales, - arr_ok(W->biases) ? W->biases : null_arr, + mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, true, gs, bits, "affine", s); return res; } @@ -294,8 +285,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) return (mlx_array){0}; } mlx_array res = mlx_array_new(); - if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && - mlx_array_dtype(x) == MLX_FLOAT32) { + if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) { /* F32 activations with BF16 weights: upcast the weight operand and run * the matmul in F32 (so accumulation is F32). Only the weights are * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), @@ -307,21 +297,21 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) } else { mlx_matmul(&res, x, Wt, s); } - if (free_wt) mlx_array_free(Wt); + if (free_wt) + mlx_array_free(Wt); return res; } -const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) -{ +const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } /* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the * residual stream in F32 while keeping the large matmul weights in BF16. */ -static void upcast_f32_inplace(mlx_array *a, mlx_stream s) -{ - if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return; +static void upcast_f32_inplace(mlx_array *a, mlx_stream s) { + if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) + return; mlx_array f = mlx_array_new(); mlx_astype(&f, *a, MLX_FLOAT32, s); mlx_array_free(*a); @@ -335,14 +325,13 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s) /* We duplicate a minimal config parser here so MLX can load standalone. * (Same logic as embed.c:parse_config, kept inline for simplicity.) */ -static const char *skip_ws_m(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; +static const char *skip_ws_m(const char *p) { + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + p++; return p; } -static const char *json_find_m(const char *json, const char *key) -{ +static const char *json_find_m(const char *json, const char *key) { size_t klen = strlen(key); const char *p = json; while ((p = strstr(p, key)) != NULL) { @@ -350,7 +339,8 @@ static const char *json_find_m(const char *json, const char *key) const char *a = p + klen; if (*a == '"') { a = skip_ws_m(a + 1); - if (*a == ':') return skip_ws_m(a + 1); + if (*a == ':') + return skip_ws_m(a + 1); } } p += klen; @@ -358,18 +348,16 @@ static const char *json_find_m(const char *json, const char *key) return NULL; } -static int json_string_equals_m(const char *json, const char *key, - const char *expected) -{ +static int json_string_equals_m(const char *json, const char *key, const char *expected) { const char *v = json_find_m(json, key); - if (!v || *v != '"') return 0; + if (!v || *v != '"') + return 0; v++; size_t len = strlen(expected); return strncmp(v, expected, len) == 0 && v[len] == '"'; } -static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) -{ +static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) { char path[1024]; int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir); if (path_len < 0 || (size_t)path_len >= sizeof(path)) { @@ -377,8 +365,14 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) return -1; } FILE *f = fopen(path, "rb"); - if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } - if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; } + if (!f) { + fprintf(stderr, "mlx: cannot open %s\n", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } long sz = ftell(f); if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { fclose(f); @@ -386,22 +380,32 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) } char *buf = (char *)malloc((size_t)sz + 1); if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; + fclose(f); + free(buf); + return -1; } fclose(f); buf[sz] = '\0'; -#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0) -#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) - GI("hidden_size", hidden_size, 0); - GI("num_hidden_layers", n_layers, 0); - GI("num_attention_heads", n_heads, 0); - GI("num_key_value_heads", n_kv_heads, 0); - GI("head_dim", head_dim, EMBED_HEAD_DIM); - GI("intermediate_size", intermediate_size, 0); - GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE); - GF("rms_norm_eps", rms_norm_eps, 1e-6f); - GF("rope_theta", rope_theta, 1000000.0f); +#define GI(k, fb, def) \ + do { \ + const char *v = json_find_m(buf, k); \ + cfg->fb = v ? atoi(v) : def; \ + } while (0) +#define GF(k, fb, def) \ + do { \ + const char *v = json_find_m(buf, k); \ + cfg->fb = v ? (float)atof(v) : def; \ + } while (0) + GI("hidden_size", hidden_size, 0); + GI("num_hidden_layers", n_layers, 0); + GI("num_attention_heads", n_heads, 0); + GI("num_key_value_heads", n_kv_heads, 0); + GI("head_dim", head_dim, EMBED_HEAD_DIM); + GI("intermediate_size", intermediate_size, 0); + GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE); + GF("rms_norm_eps", rms_norm_eps, 1e-6f); + GF("rope_theta", rope_theta, 1000000.0f); #undef GI #undef GF cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; @@ -417,58 +421,53 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) cfg->q_dim = 0; cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim) cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim) cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; free(buf); - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "mlx: invalid config in %s " + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || + cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || + cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || + cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || + cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + fprintf(stderr, + "mlx: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim, + cfg->intermediate_size); return -1; } if (cfg->n_layers > EMBED_MAX_LAYERS) { - fprintf(stderr, "mlx: too many layers (%d > %d)\n", - cfg->n_layers, EMBED_MAX_LAYERS); + fprintf(stderr, "mlx: too many layers (%d > %d)\n", cfg->n_layers, EMBED_MAX_LAYERS); return -1; } return 0; } -static int mlx_model_dir_has_file(const char *model_dir, const char *rel) -{ +static int mlx_model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; FILE *f = fopen(path, "rb"); - if (!f) return 0; + if (!f) + return 0; fclose(f); return 1; } -static int mlx_model_dir_has_late_projection(const char *model_dir) -{ +static int mlx_model_dir_has_late_projection(const char *model_dir) { return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) -{ +static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) { embed_mlx_options_t out = {0}; - if (opts) out = *opts; - if (out.quantize_bits != 0 && - out.quantize_bits != 8) { + if (opts) + out = *opts; + if (out.quantize_bits != 0 && out.quantize_bits != 8) { fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); out.quantize_bits = -1; } @@ -477,10 +476,9 @@ static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts return out; } -static int quantize_layer(mlx_layer_t *l, int bits, int group_size, - mlx_stream s) -{ - if (bits == 0) return 0; +static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) { + if (bits == 0) + return 0; return quantize_linear(&l->wq, bits, group_size, s) || quantize_linear(&l->wk, bits, group_size, s) || quantize_linear(&l->wv, bits, group_size, s) || @@ -490,45 +488,45 @@ static int quantize_layer(mlx_layer_t *l, int bits, int group_size, quantize_linear(&l->down_proj, bits, group_size, s); } -static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) -{ - return prepare_linear_transpose(&l->wq, s) || - prepare_linear_transpose(&l->wk, s) || - prepare_linear_transpose(&l->wv, s) || - prepare_linear_transpose(&l->wo, s) || - prepare_linear_transpose(&l->gate_proj, s) || - prepare_linear_transpose(&l->up_proj, s) || +static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { + return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) || + prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) || + prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) || prepare_linear_transpose(&l->down_proj, s); } static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - const embed_mlx_options_t *opts, - int allow_late) -{ + int layer_start, + int layer_end, + const embed_mlx_options_t *opts, + int allow_late) { embed_mlx_options_t options = normalize_mlx_options(opts); - if (options.quantize_bits < 0) return NULL; + if (options.quantize_bits < 0) + return NULL; if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx: late-interaction models require token-level " - "MaxSim support and are not valid pooled embedding models " - "yet\n"); + "MaxSim support and are not valid pooled embedding models " + "yet\n"); return NULL; } embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t)); - if (!ctx) return NULL; + if (!ctx) + return NULL; ctx->quantize_bits = options.quantize_bits; ctx->quantize_group_size = options.quantize_group_size; - if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } + if (mlx_parse_config(&ctx->config, model_dir) != 0) { + free(ctx); + return NULL; + } const embed_config_t *c = &ctx->config; if (layer_end == -1) layer_end = c->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > c->n_layers) { - fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", - layer_start, layer_end, c->n_layers); + if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) { + fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start, + layer_end, c->n_layers); free(ctx); return NULL; } @@ -538,7 +536,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, ctx->ms = multi_safetensors_open(model_dir); if (!ctx->ms) { fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); - free(ctx); return NULL; + free(ctx); + return NULL; } int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; @@ -547,64 +546,73 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, /* Only the first stage needs token embeddings. */ if (layer_start == 0) { int emb_shape[] = {c->vocab_size, h}; - ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", - emb_shape, 2); - if (!arr_ok(ctx->embed_tokens)) goto fail; + ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) + goto fail; } /* Layers */ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); - if (!ctx->layers) goto fail; + if (!ctx->layers) + goto fail; char name[256]; for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); - LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); - LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); - LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) + + LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); + LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); + LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); + LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); #undef LD -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!arr_ok(l->fld)) goto fail; \ -} while (0) - - LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); - LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); - LD(input_norm, "layers.%d.input_layernorm.weight", h); +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!arr_ok(l->fld)) \ + goto fail; \ + } while (0) + + LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); + LD(input_norm, "layers.%d.input_layernorm.weight", h); LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); #undef LD -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); - LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); - LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) + + LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); + LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); + LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); #undef LD - if (embed_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); + if (embed_verbose >= 2) + fprintf(stderr, " mlx layer %d loaded\n", i); } if (layer_end == c->n_layers) { int ns[] = {h}; ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); - if (!arr_ok(ctx->norm)) goto fail; + if (!arr_ok(ctx->norm)) + goto fail; } ctx->stream = mlx_default_gpu_stream_new(); @@ -624,13 +632,14 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, } if (ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { - if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, - ctx->quantize_group_size, ctx->stream) != 0) + if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size, + ctx->stream) != 0) goto fail; } mlx_clear_cache(); if (embed_verbose >= 1) - fprintf(stderr, "mlx: quantized linear weights to %d-bit " + fprintf(stderr, + "mlx: quantized linear weights to %d-bit " "(group_size=%d)\n", ctx->quantize_bits, ctx->quantize_group_size); } else { @@ -645,7 +654,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, mlx_get_active_memory(&active); mlx_get_cache_memory(&cache); mlx_get_peak_memory(&peak); - fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " + fprintf(stderr, + "mlx: memory active=%.2f GiB cache=%.2f GiB " "peak=%.2f GiB\n", (double)active / (1024.0 * 1024.0 * 1024.0), (double)cache / (1024.0 * 1024.0 * 1024.0), @@ -653,8 +663,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, } if (embed_verbose >= 1) - fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", - layer_start, layer_end, h); + fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h); return ctx; @@ -663,14 +672,12 @@ fail: return NULL; } -embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) -{ +embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) { return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); } embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts) -{ + const embed_mlx_options_t *opts) { return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } @@ -678,19 +685,23 @@ embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, * Free * ======================================================================== */ -static void free_mlx_layer(mlx_layer_t *l) -{ - free_linear(&l->wq); free_linear(&l->wk); - free_linear(&l->wv); free_linear(&l->wo); - mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); - mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); - free_linear(&l->gate_proj); free_linear(&l->up_proj); +static void free_mlx_layer(mlx_layer_t *l) { + free_linear(&l->wq); + free_linear(&l->wk); + free_linear(&l->wv); + free_linear(&l->wo); + mlx_array_free(l->q_norm); + mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); + mlx_array_free(l->post_attn_norm); + free_linear(&l->gate_proj); + free_linear(&l->up_proj); free_linear(&l->down_proj); } -void embed_mlx_free(embed_mlx_ctx_t *ctx) -{ - if (!ctx) return; +void embed_mlx_free(embed_mlx_ctx_t *ctx) { + if (!ctx) + return; mlx_array_free(ctx->embed_tokens); if (ctx->layers) { for (int i = 0; i < ctx->config.n_layers; i++) @@ -698,24 +709,27 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) free(ctx->layers); } mlx_array_free(ctx->norm); - if (ctx->stream.ctx) mlx_stream_free(ctx->stream); - if (ctx->ms) multi_safetensors_close(ctx->ms); + if (ctx->stream.ctx) + mlx_stream_free(ctx->stream); + if (ctx->ms) + multi_safetensors_close(ctx->ms); free(ctx); } -embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( - const char *model_dir, const embed_mlx_options_t *opts) -{ +embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, + const embed_mlx_options_t *opts) { if (!mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx late: missing 1_Dense projection\n"); return NULL; } embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); - if (!late) return NULL; + if (!late) + return NULL; late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); - if (!late->base) goto fail; + if (!late->base) + goto fail; char dense_dir[1024]; int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); @@ -726,13 +740,11 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( late->dense_ms = multi_safetensors_open(dense_dir); if (!late->dense_ms) { - fprintf(stderr, "mlx late: failed to open safetensors in %s\n", - dense_dir); + fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir); goto fail; } - const safetensor_t *t = multi_safetensors_find(late->dense_ms, - "linear.weight", NULL); + const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL); if (!t) { fprintf(stderr, "mlx late: missing linear.weight\n"); goto fail; @@ -743,17 +755,16 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( goto fail; } - int shape[] = { (int)t->shape[0], late->base->config.hidden_size }; - late->projection = load_linear_tensor(late->dense_ms, "linear.weight", - shape, 2); - if (!linear_ok(late->projection)) goto fail; + int shape[] = {(int)t->shape[0], late->base->config.hidden_size}; + late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2); + if (!linear_ok(late->projection)) + goto fail; if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) goto fail; late->token_dim = shape[0]; if (embed_verbose >= 1) - fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", - late->token_dim); + fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim); return late; fail: @@ -761,39 +772,38 @@ fail: return NULL; } -embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) -{ +embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) { return embed_mlx_late_load_with_options(model_dir, NULL); } -void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) -{ - if (!ctx) return; +void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) { + if (!ctx) + return; free_linear(&ctx->projection); - if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); + if (ctx->dense_ms) + multi_safetensors_close(ctx->dense_ms); embed_mlx_free(ctx->base); free(ctx); } -const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) -{ +const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) { return ctx && ctx->base ? &ctx->base->config : NULL; } -int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) -{ - return ctx ? ctx->token_dim : 0; -} +int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } /* ======================================================================== * Forward pass * ======================================================================== */ -static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, - int batch, int max_seq, int has_padding, +static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, + mlx_array x, + int batch, + int max_seq, + int has_padding, mlx_array attn_mask, - int layer_start, int layer_end) -{ + int layer_start, + int layer_end) { const embed_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int n_heads = c->n_heads; @@ -802,8 +812,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, int q_dim = c->q_dim; float scale = 1.0f / sqrtf((float)head_dim); mlx_array null_arr = (mlx_array){0}; - const char *mask_mode = - c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; + const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; /* For non-quantized BF16 weights, run the residual stream, norms, and * attention in F32; matmuls downcast their operands to BF16 internally @@ -812,8 +821,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, * models are unaffected. */ mlx_array local_mask = attn_mask; int free_mask = 0; - int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && - !ctx->layers[layer_start].wq.quantized; + int mixed_bf16 = + mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized; if (mixed_bf16) { if (mlx_array_dtype(x) != MLX_FLOAT32) { mlx_array xf = mlx_array_new(); @@ -821,8 +830,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array_free(x); x = xf; } - if (has_padding && arr_ok(attn_mask) && - mlx_array_dtype(attn_mask) != MLX_FLOAT32) { + if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) { local_mask = mlx_array_new(); mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); free_mask = 1; @@ -840,7 +848,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array v_flat = linear(xn, &l->wv, S); mlx_array_free(xn); - int q_shape[] = {batch, max_seq, n_heads, head_dim}; + int q_shape[] = {batch, max_seq, n_heads, head_dim}; int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; mlx_array q = mlx_array_new(); mlx_array k = mlx_array_new(); @@ -848,33 +856,40 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_reshape(&q, q_flat, q_shape, 4, S); mlx_reshape(&k, k_flat, k_shape, 4, S); mlx_reshape(&v, v_flat, k_shape, 4, S); - mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); + mlx_array_free(q_flat); + mlx_array_free(k_flat); + mlx_array_free(v_flat); mlx_array qn = mlx_array_new(); mlx_array kn = mlx_array_new(); mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); - mlx_array_free(q); mlx_array_free(k); + mlx_array_free(q); + mlx_array_free(k); /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ int perm[] = {0, 2, 1, 3}; mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); mlx_transpose_axes(&qt, qn, perm, 4, S); mlx_transpose_axes(&kt, kn, perm, 4, S); - mlx_transpose_axes(&vt, v, perm, 4, S); - mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); + mlx_array_free(kn); + mlx_array_free(v); mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; mlx_array qr = mlx_array_new(), kr = mlx_array_new(); mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_array_free(qt); mlx_array_free(kt); + mlx_array_free(qt); + mlx_array_free(kt); mlx_array attn = mlx_array_new(); - mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, mask_mode, - has_padding ? local_mask : null_arr, null_arr, S); - mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); + mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode, + has_padding ? local_mask : null_arr, null_arr, S); + mlx_array_free(qr); + mlx_array_free(kr); + mlx_array_free(vt); mlx_array attn_t = mlx_array_new(); mlx_transpose_axes(&attn_t, attn, perm, 4, S); @@ -889,44 +904,49 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array_free(attn_flat); mlx_array x2 = mlx_array_new(); mlx_add(&x2, x, proj, S); - mlx_array_free(x); mlx_array_free(proj); + mlx_array_free(x); + mlx_array_free(proj); x = x2; mlx_array xn2 = mlx_array_new(); mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); mlx_array gate = linear(xn2, &l->gate_proj, S); - mlx_array up = linear(xn2, &l->up_proj, S); + mlx_array up = linear(xn2, &l->up_proj, S); mlx_array_free(xn2); mlx_array gate_sig = mlx_array_new(); mlx_sigmoid(&gate_sig, gate, S); mlx_array silu_gate = mlx_array_new(); mlx_multiply(&silu_gate, gate, gate_sig, S); - mlx_array_free(gate); mlx_array_free(gate_sig); + mlx_array_free(gate); + mlx_array_free(gate_sig); mlx_array mid = mlx_array_new(); mlx_multiply(&mid, silu_gate, up, S); - mlx_array_free(silu_gate); mlx_array_free(up); + mlx_array_free(silu_gate); + mlx_array_free(up); mlx_array ffn = linear(mid, &l->down_proj, S); mlx_array_free(mid); mlx_array x3 = mlx_array_new(); mlx_add(&x3, x, ffn, S); - mlx_array_free(x); mlx_array_free(ffn); + mlx_array_free(x); + mlx_array_free(ffn); x = x3; } - if (free_mask) mlx_array_free(local_mask); + if (free_mask) + mlx_array_free(local_mask); return x; } -embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( - embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize) -{ - if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || - ctx->token_dim <= 0 || !linear_ok(ctx->projection)) +embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize) { + if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 || + !linear_ok(ctx->projection)) return NULL; embed_mlx_ctx_t *base = ctx->base; @@ -938,8 +958,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_stream S = base->stream; for (int i = 0; i < n_tokens; i++) { if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { - fprintf(stderr, "mlx late: invalid token id %d at position %d\n", - token_ids[i], i); + fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i); return NULL; } } @@ -949,13 +968,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( return NULL; int *ids_data = (int *)malloc(id_bytes); - if (!ids_data) return NULL; + if (!ids_data) + return NULL; memcpy(ids_data, token_ids, id_bytes); int ids_shape[] = {1, n_tokens}; mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); free(ids_data); - if (!arr_ok(ids)) return NULL; + if (!arr_ok(ids)) + return NULL; embed_mlx_late_vectors_t *result = NULL; mlx_array x = mlx_array_new(); @@ -965,22 +986,25 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_take_axis(&x, base->embed_tokens, ids, 0, S); mlx_array_free(ids); - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; - x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, - 0, c->n_layers); - if (!arr_ok(x)) goto cleanup; + x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers); + if (!arr_ok(x)) + goto cleanup; x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); mlx_array_free(x); x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; + if (!arr_ok(x_normed)) + goto cleanup; projected = linear(x_normed, &ctx->projection, S); mlx_array_free(x_normed); x_normed = (mlx_array){0}; - if (!arr_ok(projected)) goto cleanup; + if (!arr_ok(projected)) + goto cleanup; if (normalize) { mlx_array sq = mlx_array_new(); @@ -1011,16 +1035,19 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_array_free(inv_norm); mlx_array_free(projected); projected = normalized; - if (!arr_ok(projected)) goto cleanup; + if (!arr_ok(projected)) + goto cleanup; } int vec_shape[] = {n_tokens, ctx->token_dim}; reshaped = mlx_array_new(); mlx_reshape(&reshaped, projected, vec_shape, 2, S); - if (!arr_ok(reshaped)) goto cleanup; + if (!arr_ok(reshaped)) + goto cleanup; result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); - if (!result) goto cleanup; + if (!result) + goto cleanup; result->owner = ctx; result->vectors = reshaped; result->tokens = n_tokens; @@ -1035,28 +1062,23 @@ cleanup: return result; } -void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) -{ - if (!vecs) return; +void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) { + if (!vecs) + return; mlx_array_free(vecs->vectors); free(vecs); } -int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) -{ +int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) { return vecs ? vecs->tokens : 0; } -int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) -{ +int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) { return vecs ? vecs->dim : 0; } -int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, - float *out_vectors) -{ - if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || - vecs->tokens <= 0 || vecs->dim <= 0) +int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) { + if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0) return -1; size_t values, bytes; @@ -1084,32 +1106,33 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, } embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( - embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *const *items, int count) -{ - if (!ctx || !ctx->base || !items || count <= 0) return NULL; + embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) { + if (!ctx || !ctx->base || !items || count <= 0) + return NULL; int dim = 0; int total = 0; for (int i = 0; i < count; i++) { const embed_mlx_late_vectors_t *v = items[i]; - if (!v || v->owner != ctx || !arr_ok(v->vectors) || - v->tokens <= 0 || v->dim <= 0) + if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0) return NULL; - if (i == 0) dim = v->dim; + if (i == 0) + dim = v->dim; if (v->dim != dim || total > INT_MAX - v->tokens) return NULL; total += v->tokens; } mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); - if (!parts) return NULL; + if (!parts) + return NULL; for (int i = 0; i < count; i++) parts[i] = items[i]->vectors; mlx_stream S = ctx->base->stream; mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); free(parts); - if (!vec.ctx) return NULL; + if (!vec.ctx) + return NULL; mlx_array joined = mlx_array_new(); mlx_concatenate_axis(&joined, vec, 0, S); @@ -1119,8 +1142,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return NULL; } - embed_mlx_late_vectors_t *out = - (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(joined); return NULL; @@ -1132,13 +1154,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return out; } -embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs, - const int *token_indices, int count) -{ - if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || - !arr_ok(vecs->vectors) || !token_indices || count <= 0 || - vecs->tokens <= 0 || vecs->dim <= 0) +embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *vecs, + const int *token_indices, + int count) { + if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) || + !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0) return NULL; for (int i = 0; i < count; i++) { @@ -1150,13 +1171,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) return NULL; int *idx_data = (int *)malloc(idx_bytes); - if (!idx_data) return NULL; + if (!idx_data) + return NULL; memcpy(idx_data, token_indices, idx_bytes); int idx_shape[] = {count}; mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); free(idx_data); - if (!arr_ok(idx)) return NULL; + if (!arr_ok(idx)) + return NULL; mlx_stream S = ctx->base->stream; mlx_array selected = mlx_array_new(); @@ -1167,8 +1190,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( return NULL; } - embed_mlx_late_vectors_t *out = - (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(selected); return NULL; @@ -1180,21 +1202,21 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( return out; } -int embed_mlx_late_maxsim_batch_device( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query, - const embed_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores) -{ - if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || - !scores || query->owner != ctx || docs->owner != ctx || - query->dim <= 0 || query->dim != docs->dim || - query->tokens <= 0 || docs->tokens <= 0 || - !arr_ok(query->vectors) || !arr_ok(docs->vectors) || - doc_offsets[0] != 0) +int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *query, + const embed_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores) { + if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores || + query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim || + query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) || + !arr_ok(docs->vectors) || doc_offsets[0] != 0) return -1; int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); - if (!lengths) return -1; + if (!lengths) + return -1; int max_doc_tokens = 0; for (int i = 0; i < docs_count; i++) { if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || @@ -1204,7 +1226,8 @@ int embed_mlx_late_maxsim_batch_device( } int n = doc_offsets[i + 1] - doc_offsets[i]; lengths[i] = n; - if (n > max_doc_tokens) max_doc_tokens = n; + if (n > max_doc_tokens) + max_doc_tokens = n; } if (max_doc_tokens <= 0) { free(lengths); @@ -1214,8 +1237,7 @@ int embed_mlx_late_maxsim_batch_device( mlx_stream S = ctx->base->stream; int dim = query->dim; int rc = -1; - mlx_array *doc_rows = - (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); + mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); if (!doc_rows) { free(lengths); return -1; @@ -1250,95 +1272,114 @@ int embed_mlx_late_maxsim_batch_device( mlx_array padded = (mlx_array){0}; mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); - if (!arr_ok(doc)) goto cleanup; + if (!arr_ok(doc)) + goto cleanup; if (n < max_doc_tokens) { int axes[] = {0}; int low[] = {0}; int high[] = {max_doc_tokens - n}; padded = mlx_array_new(); - mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, - "constant", S); + mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S); mlx_array_free(doc); doc = padded; padded = (mlx_array){0}; - if (!arr_ok(doc)) goto cleanup; + if (!arr_ok(doc)) + goto cleanup; } doc_rows[i] = mlx_array_new(); mlx_expand_dims(&doc_rows[i], doc, 0, S); mlx_array_free(doc); - if (!arr_ok(doc_rows[i])) goto cleanup; + if (!arr_ok(doc_rows[i])) + goto cleanup; } doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); - if (!doc_vec.ctx) goto cleanup; + if (!doc_vec.ctx) + goto cleanup; doc_batch = mlx_array_new(); mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); - if (!arr_ok(doc_batch)) goto cleanup; + if (!arr_ok(doc_batch)) + goto cleanup; query_3d = mlx_array_new(); mlx_expand_dims(&query_3d, query->vectors, 0, S); - if (!arr_ok(query_3d)) goto cleanup; + if (!arr_ok(query_3d)) + goto cleanup; int q_shape[] = {docs_count, query->tokens, dim}; query_batched = mlx_array_new(); mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); - if (!arr_ok(query_batched)) goto cleanup; + if (!arr_ok(query_batched)) + goto cleanup; int perm[] = {0, 2, 1}; docs_t = mlx_array_new(); mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); - if (!arr_ok(docs_t)) goto cleanup; + if (!arr_ok(docs_t)) + goto cleanup; sims = mlx_array_new(); mlx_matmul(&sims, query_batched, docs_t, S); - if (!arr_ok(sims)) goto cleanup; + if (!arr_ok(sims)) + goto cleanup; int len_shape[] = {docs_count}; lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); - if (!arr_ok(lengths_arr)) goto cleanup; + if (!arr_ok(lengths_arr)) + goto cleanup; positions = mlx_array_new(); mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); - if (!arr_ok(positions)) goto cleanup; + if (!arr_ok(positions)) + goto cleanup; lengths_2d = mlx_array_new(); mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); - if (!arr_ok(lengths_2d)) goto cleanup; + if (!arr_ok(lengths_2d)) + goto cleanup; positions_2d = mlx_array_new(); mlx_expand_dims(&positions_2d, positions, 0, S); - if (!arr_ok(positions_2d)) goto cleanup; + if (!arr_ok(positions_2d)) + goto cleanup; valid = mlx_array_new(); mlx_less(&valid, positions_2d, lengths_2d, S); - if (!arr_ok(valid)) goto cleanup; + if (!arr_ok(valid)) + goto cleanup; valid_3d = mlx_array_new(); mlx_expand_dims(&valid_3d, valid, 1, S); - if (!arr_ok(valid_3d)) goto cleanup; + if (!arr_ok(valid_3d)) + goto cleanup; masked_sims = mlx_array_new(); mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); - if (!arr_ok(masked_sims)) goto cleanup; + if (!arr_ok(masked_sims)) + goto cleanup; maxes = mlx_array_new(); mlx_max_axis(&maxes, masked_sims, 2, false, S); - if (!arr_ok(maxes)) goto cleanup; + if (!arr_ok(maxes)) + goto cleanup; sums = mlx_array_new(); mlx_sum_axis(&sums, maxes, 1, false, S); - if (!arr_ok(sums)) goto cleanup; + if (!arr_ok(sums)) + goto cleanup; sums_f32 = mlx_array_new(); mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); - if (!arr_ok(sums_f32)) goto cleanup; + if (!arr_ok(sums_f32)) + goto cleanup; mlx_array_eval(sums_f32); const float *data = mlx_array_data_float32(sums_f32); - if (!data) goto cleanup; + if (!data) + goto cleanup; memcpy(scores, data, (size_t)docs_count * sizeof(float)); rc = 0; @@ -1359,7 +1400,8 @@ cleanup: mlx_array_free(query_batched); mlx_array_free(query_3d); mlx_array_free(doc_batch); - if (doc_vec.ctx) mlx_vector_array_free(doc_vec); + if (doc_vec.ctx) + mlx_vector_array_free(doc_vec); mlx_array_free(pad_value); for (int i = 0; i < docs_count; i++) mlx_array_free(doc_rows[i]); @@ -1369,31 +1411,32 @@ cleanup: } int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { embed_mlx_late_vectors_t *vecs = - embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, - normalize); - if (!vecs) return -1; + embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); + if (!vecs) + return -1; int rc = embed_mlx_late_vectors_copy(vecs, out_vectors); embed_mlx_late_vectors_free(vecs); return rc; } static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, - const embed_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers) return -1; const embed_config_t *c = &ctx->config; mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; + int hidden = c->hidden_size; + int max_seq = 0; int has_padding = 0; int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN; @@ -1401,13 +1444,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, return -1; for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return -1; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; } - if (max_seq <= 0) return -1; + if (max_seq <= 0) + return -1; for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; + if (inputs[b].n_tokens != max_seq) + has_padding = 1; } int needs_pool_mask = has_padding && !last_token_pool; int needs_attn_mask = has_padding && !causal; @@ -1420,24 +1467,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 || - (last_token_pool && - mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) + (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) return -1; if (ids_bytes == 0 || out_bytes == 0 || (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) || - (needs_attn_mask && mask_bytes == 0) || - (last_token_pool && last_index_bytes == 0)) + (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0)) return -1; int *padded_ids = (int *)malloc(ids_bytes); - float *pool_mask_data = - needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; - float *attn_mask_data = - needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; + float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL; - int *last_index_data = - last_token_pool ? (int *)malloc(last_index_bytes) : NULL; - if (!padded_ids || - (needs_pool_mask && (!pool_mask_data || !len_data)) || + int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL; + if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) || (needs_attn_mask && !attn_mask_data)) { free(padded_ids); free(pool_mask_data); @@ -1455,7 +1496,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, } for (int b = 0; b < batch; b++) { - if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens; + if (needs_pool_mask) + len_data[b] = (float)inputs[b].n_tokens; if (last_token_pool) { int index = inputs[b].n_tokens - 1; for (int d = 0; d < hidden; d++) @@ -1466,8 +1508,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (t < inputs[b].n_tokens) { int id = inputs[b].ids[t]; if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t); free(padded_ids); free(pool_mask_data); free(attn_mask_data); @@ -1498,25 +1539,26 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); mlx_array pool_mask = needs_pool_mask - ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) - : (mlx_array){0}; + ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) + : (mlx_array){0}; mlx_array attn_mask = needs_attn_mask - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array lengths = needs_pool_mask - ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array last_indices = last_token_pool - ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) - : (mlx_array){0}; - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array lengths = + needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; + mlx_array last_indices = + last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); free(last_index_data); - if (!arr_ok(ids) || - (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || - (needs_attn_mask && !arr_ok(attn_mask)) || - (last_token_pool && !arr_ok(last_indices))) { - mlx_array_free(ids); mlx_array_free(pool_mask); + if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || + (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) { + mlx_array_free(ids); + mlx_array_free(pool_mask); mlx_array_free(attn_mask); mlx_array_free(lengths); mlx_array_free(last_indices); @@ -1528,7 +1570,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(attn_mask); attn_mask = attn_mask_bf16; if (!arr_ok(attn_mask)) { - mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); + mlx_array_free(ids); + mlx_array_free(pool_mask); + mlx_array_free(lengths); mlx_array_free(last_indices); return -1; } @@ -1540,9 +1584,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(ids); /* 2. Transformer layers. */ - x = mlx_forward_layers(ctx, x, batch, max_seq, - needs_attn_mask, attn_mask, - 0, c->n_layers); + x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); /* 3. Final RMSNorm. */ mlx_array x_normed = mlx_array_new(); @@ -1573,7 +1615,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(masked); mlx_divide(&emb, emb_sum, lengths, S); - mlx_array_free(emb_sum); mlx_array_free(lengths); + mlx_array_free(emb_sum); + mlx_array_free(lengths); mlx_array_free(attn_mask); } else { /* Equal-length batches do not need masks or a separate divide. */ @@ -1595,8 +1638,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (c->normalize_embeddings) { rc = 0; for (int b = 0; b < batch; b++) { - if (embed_l2_normalize( - out_embeddings + (size_t)b * hidden, hidden) != 0) { + if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) { rc = -1; break; } @@ -1614,8 +1656,7 @@ typedef struct { const embed_input_t *input; } mlx_batch_item_t; -static int mlx_batch_item_cmp(const void *a, const void *b) -{ +static int mlx_batch_item_cmp(const void *a, const void *b) { const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; int da = ia->input->n_tokens; @@ -1623,27 +1664,33 @@ static int mlx_batch_item_cmp(const void *a, const void *b) return (da > db) - (da < db); } -static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) -{ +static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) { if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) return 1; return padded * 5 > tokens * 11; } -static int mlx_batch_should_split(const embed_input_t *inputs, int batch, - int *out_total_tokens, int *out_max_seq) -{ +static int mlx_batch_should_split(const embed_input_t *inputs, + int batch, + int *out_total_tokens, + int *out_max_seq) { int max_seq = 0; int total_tokens = 0; for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0; + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return 0; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + if (total_tokens > INT_MAX - inputs[b].n_tokens) + return 0; total_tokens += inputs[b].n_tokens; } - if (out_total_tokens) *out_total_tokens = total_tokens; - if (out_max_seq) *out_max_seq = max_seq; - if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0; + if (out_total_tokens) + *out_total_tokens = total_tokens; + if (out_max_seq) + *out_max_seq = max_seq; + if (batch < 4 || max_seq < 64 || total_tokens <= 0) + return 0; size_t padded; if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) @@ -1653,9 +1700,9 @@ static int mlx_batch_should_split(const embed_input_t *inputs, int batch, } static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, - int group_count, int group_max, - int next_tokens) -{ + int group_count, + int group_max, + int next_tokens) { if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) return 1; @@ -1674,9 +1721,10 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, return merged <= split_total; } -int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, - int batch, float *out_embeddings) -{ +int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -1684,19 +1732,20 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; - mlx_batch_item_t *items = (mlx_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - embed_input_t *group_inputs = (embed_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); + mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); + free(items); + free(group_inputs); return -1; } float *group_out = (float *)malloc(group_bytes); if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return -1; } for (int i = 0; i < batch; i++) { @@ -1706,7 +1755,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { + for (int start = 0; start < batch && rc == 0;) { int group_max = 0; int group_tokens = 0; int end = start; @@ -1717,12 +1766,9 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, int next_total = group_tokens + next_tokens; if (end > start) { size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) break; } group_max = next_max; @@ -1733,26 +1779,28 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, - group_out); - if (rc != 0) break; + rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; for (int i = 0; i < group_count; i++) { int dst = items[start + i].index; - memcpy(out_embeddings + (size_t)dst * hidden, - group_out + (size_t)i * hidden, + memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden, (size_t)hidden * sizeof(float)); } start = end; } - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return rc; } -int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - embed_input_t input = { token_ids, n_tokens }; +int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; return embed_mlx_encode_batch(ctx, &input, 1, out_embedding); } @@ -1762,8 +1810,7 @@ typedef struct { const embed_context_input_t *input; } mlx_context_batch_item_t; -static int mlx_context_batch_item_cmp(const void *a, const void *b) -{ +static int mlx_context_batch_item_cmp(const void *a, const void *b) { const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; int da = ia->input->input.n_tokens; @@ -1771,33 +1818,33 @@ static int mlx_context_batch_item_cmp(const void *a, const void *b) return (da > db) - (da < db); } -static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) -{ +static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) { if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) return 1; return padded * 4 > tokens * 5; } static int mlx_context_batch_should_split(const embed_context_input_t *inputs, - int batch, int *out_total_spans) -{ + int batch, + int *out_total_spans) { int max_seq = 0; int total_tokens = 0; int total_spans = 0; for (int b = 0; b < batch; b++) { int n_tokens = inputs[b].input.n_tokens; - if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || - inputs[b].n_spans <= 0) + if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0) return 0; - if (n_tokens > max_seq) max_seq = n_tokens; - if (total_tokens > INT_MAX - n_tokens || - total_spans > INT_MAX - inputs[b].n_spans) + if (n_tokens > max_seq) + max_seq = n_tokens; + if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans) return 0; total_tokens += n_tokens; total_spans += inputs[b].n_spans; } - if (out_total_spans) *out_total_spans = total_spans; - if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0; + if (out_total_spans) + *out_total_spans = total_spans; + if (batch < 2 || max_seq < 64 || total_tokens <= 0) + return 0; size_t padded; if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) @@ -1806,13 +1853,11 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs, } static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || - ctx->layer_end != ctx->config.n_layers || - !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) return -1; const embed_config_t *c = &ctx->config; @@ -1824,29 +1869,29 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, for (int b = 0; b < batch; b++) { const embed_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_spans > INT_MAX - input->n_spans) return -1; total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; + if (n_tokens > max_seq) + max_seq = n_tokens; for (int t = 0; t < n_tokens; t++) { - if (input->input.ids[t] < 0 || - input->input.ids[t] >= c->vocab_size) { + if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) { fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", input->input.ids[t], b, t); return -1; } } for (int s = 0; s < input->n_spans; s++) { - if (input->spans[s].start < 0 || - input->spans[s].start > n_tokens || + if (input->spans[s].start < 0 || input->spans[s].start > n_tokens || input->spans[s].n_tokens <= 0 || input->spans[s].n_tokens > n_tokens - input->spans[s].start) return -1; } } for (int b = 0; b < batch; b++) - if (inputs[b].input.n_tokens != max_seq) has_padding = 1; + if (inputs[b].input.n_tokens != max_seq) + has_padding = 1; size_t rows, ids_bytes, mask_bytes, index_bytes; size_t span_float_bytes, out_values, out_bytes; @@ -1858,10 +1903,10 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) return -1; - if (rows > INT_MAX) return -1; + if (rows > INT_MAX) + return -1; int *padded_ids = (int *)malloc(ids_bytes); - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; if (!padded_ids || (has_padding && !attn_mask_data)) { free(padded_ids); free(attn_mask_data); @@ -1873,10 +1918,12 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, size_t row = (size_t)b * max_seq + t; if (t < input->n_tokens) { padded_ids[row] = input->ids[t]; - if (has_padding) attn_mask_data[row] = 0.0f; + if (has_padding) + attn_mask_data[row] = 0.0f; } else { padded_ids[row] = 0; - if (has_padding) attn_mask_data[row] = -1.0e9f; + if (has_padding) + attn_mask_data[row] = -1.0e9f; } } } @@ -1908,44 +1955,49 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int ids_shape[] = {batch, max_seq}; int attn_mask_shape[] = {batch, 1, 1, max_seq}; ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; + attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; free(padded_ids); free(attn_mask_data); - if (!arr_ok(ids)) goto cleanup; - if (has_padding && !arr_ok(attn_mask)) goto cleanup; + if (!arr_ok(ids)) + goto cleanup; + if (has_padding && !arr_ok(attn_mask)) + goto cleanup; if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { mlx_array attn_mask_bf16 = mlx_array_new(); mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); mlx_array_free(attn_mask); attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) goto cleanup; + if (!arr_ok(attn_mask)) + goto cleanup; } x = mlx_array_new(); mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); mlx_array_free(ids); ids = (mlx_array){0}; - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers); if (has_padding) { mlx_array_free(attn_mask); attn_mask = (mlx_array){0}; } - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); mlx_array_free(x); x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; + if (!arr_ok(x_normed)) + goto cleanup; if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); - if (!pooled) goto cleanup; + if (!pooled) + goto cleanup; int pooled_count = 0; for (int b = 0; b < batch; b++) { @@ -1964,23 +2016,25 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, pooled[pooled_count] = mlx_array_new(); mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); mlx_array_free(slice); - if (!arr_ok(pooled[pooled_count])) goto cleanup; + if (!arr_ok(pooled[pooled_count])) + goto cleanup; pooled_count++; } } pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); - if (!pooled_vec.ctx) goto cleanup; + if (!pooled_vec.ctx) + goto cleanup; embeddings = mlx_array_new(); mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); - if (!arr_ok(embeddings)) goto cleanup; + if (!arr_ok(embeddings)) + goto cleanup; } else { end_idx_data = (int *)malloc(index_bytes); start_idx_data = (int *)malloc(index_bytes); lengths_data = (float *)malloc(span_float_bytes); start_mask_data = (float *)malloc(span_float_bytes); - if (!end_idx_data || !start_idx_data || !lengths_data || - !start_mask_data) + if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data) goto cleanup; int span_count = 0; @@ -2001,57 +2055,65 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int index_shape[] = {total_spans}; int scale_shape[] = {total_spans, 1}; end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); - start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, - MLX_INT32); - lengths = mlx_array_new_data(lengths_data, scale_shape, 2, - MLX_FLOAT32); - start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, - MLX_FLOAT32); - free(end_idx_data); end_idx_data = NULL; - free(start_idx_data); start_idx_data = NULL; - free(lengths_data); lengths_data = NULL; - free(start_mask_data); start_mask_data = NULL; - if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || - !arr_ok(start_mask)) + start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32); + lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32); + start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32); + free(end_idx_data); + end_idx_data = NULL; + free(start_idx_data); + start_idx_data = NULL; + free(lengths_data); + lengths_data = NULL; + free(start_mask_data); + start_mask_data = NULL; + if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask)) goto cleanup; prefix = mlx_array_new(); mlx_cumsum(&prefix, x_normed, 1, false, true, S); mlx_array_free(x_normed); x_normed = (mlx_array){0}; - if (!arr_ok(prefix)) goto cleanup; + if (!arr_ok(prefix)) + goto cleanup; int flat_shape[] = {(int)rows, hidden}; flat_prefix = mlx_array_new(); mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); mlx_array_free(prefix); prefix = (mlx_array){0}; - if (!arr_ok(flat_prefix)) goto cleanup; + if (!arr_ok(flat_prefix)) + goto cleanup; end_values = mlx_array_new(); mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); - if (!arr_ok(end_values)) goto cleanup; + if (!arr_ok(end_values)) + goto cleanup; start_values = mlx_array_new(); mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); - if (!arr_ok(start_values)) goto cleanup; + if (!arr_ok(start_values)) + goto cleanup; masked_start = mlx_array_new(); mlx_multiply(&masked_start, start_values, start_mask, S); - if (!arr_ok(masked_start)) goto cleanup; + if (!arr_ok(masked_start)) + goto cleanup; span_sums = mlx_array_new(); mlx_subtract(&span_sums, end_values, masked_start, S); - if (!arr_ok(span_sums)) goto cleanup; + if (!arr_ok(span_sums)) + goto cleanup; embeddings = mlx_array_new(); mlx_divide(&embeddings, span_sums, lengths, S); - if (!arr_ok(embeddings)) goto cleanup; + if (!arr_ok(embeddings)) + goto cleanup; } embeddings_f32 = mlx_array_new(); mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); - if (!arr_ok(embeddings_f32)) goto cleanup; + if (!arr_ok(embeddings_f32)) + goto cleanup; mlx_array_eval(embeddings_f32); const float *data = mlx_array_data_float32(embeddings_f32); @@ -2067,9 +2129,11 @@ cleanup: free(end_idx_data); mlx_array_free(embeddings_f32); mlx_array_free(embeddings); - if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); + if (pooled_vec.ctx) + mlx_vector_array_free(pooled_vec); if (pooled) { - for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); + for (int s = 0; s < total_spans; s++) + mlx_array_free(pooled[s]); free(pooled); } mlx_array_free(span_sums); @@ -2090,31 +2154,33 @@ cleanup: } int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, int batch, - float *out_embeddings) -{ + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; int total_spans = 0; if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) - return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, - out_embeddings); + return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; - mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - embed_context_input_t *group_inputs = (embed_context_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); + mlx_context_batch_item_t *items = + (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_context_input_t *group_inputs = + (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); + free(items); + free(group_inputs); return -1; } float *group_out = (float *)malloc(group_bytes); if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return -1; } @@ -2128,7 +2194,7 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { + for (int start = 0; start < batch && rc == 0;) { int group_max = 0; int group_tokens = 0; int end = start; @@ -2139,12 +2205,9 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, int next_total = group_tokens + next_tokens; if (end > start) { size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_context_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) break; } group_max = next_max; @@ -2155,47 +2218,51 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, - group_count, group_out); - if (rc != 0) break; + rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; int src_span = 0; for (int i = 0; i < group_count; i++) { int n_spans = group_inputs[i].n_spans; int dst_span = items[start + i].span_offset; memcpy(out_embeddings + (size_t)dst_span * hidden, - group_out + (size_t)src_span * hidden, - (size_t)n_spans * hidden * sizeof(float)); + group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float)); src_span += n_spans; } start = end; } - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return rc; } -int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const embed_span_t *spans, - int n_spans, float *out_embeddings) -{ +int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings) { embed_context_input_t input = { - .input = { token_ids, n_tokens }, + .input = {token_ids, n_tokens}, .spans = spans, .n_spans = n_spans, }; return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings); } -float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) -{ - if (!ctx || !token_ids || n_tokens <= 0) return NULL; +float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { + if (!ctx || !token_ids || n_tokens <= 0) + return NULL; int hidden = ctx->config.hidden_size; size_t out_bytes; if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) return NULL; float *out = (float *)malloc(out_bytes); - if (!out) return NULL; + if (!out) + return NULL; if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) { free(out); diff --git a/embed_mlx.h b/embed_mlx.h index 5f6e2c9..87fbda6 100644 --- a/embed_mlx.h +++ b/embed_mlx.h @@ -38,7 +38,7 @@ typedef struct { */ EMBED_API embed_mlx_ctx_t *embed_mlx_load(const char *model_dir); EMBED_API embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts); + const embed_mlx_options_t *opts); EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx); @@ -48,8 +48,8 @@ EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx); * scoring, not one pooled document vector. */ EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir); -EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( - const char *model_dir, const embed_mlx_options_t *opts); +EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, + const embed_mlx_options_t *opts); EMBED_API void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx); EMBED_API const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx); @@ -62,8 +62,10 @@ EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx); * before copying back to CPU memory. */ EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors); + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors); /* * Device-resident late-interaction token vectors. @@ -72,24 +74,27 @@ EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, * outlive ctx. Copying is optional: MaxSim can consume these handles directly * and copy back only final candidate scores. */ -EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( - embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize); +EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize); EMBED_API void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs); EMBED_API int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs); EMBED_API int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs); -EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, - float *out_vectors); +EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors); EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( - embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *const *items, int count); -EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs, - const int *token_indices, int count); -EMBED_API int embed_mlx_late_maxsim_batch_device( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query, - const embed_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores); + embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count); +EMBED_API embed_mlx_late_vectors_t * +embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *vecs, + const int *token_indices, + int count); +EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *query, + const embed_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores); /* * Compute embedding for token_ids[0..n_tokens-1]. @@ -101,8 +106,10 @@ EMBED_API float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, in * Compute one embedding into caller-provided out[hidden_size]. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding); +EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding); /* * Compute a true padded dense batch on MLX. @@ -111,24 +118,30 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids, * Pooling and attention masking follow the loaded model configuration. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, - int batch, float *out_embeddings); +EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings); /* * Run one contextual sequence and pool selected token spans. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const embed_span_t *spans, - int n_spans, float *out_embeddings); +EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings); /* * Run a padded dense contextual document batch and pool every selected span. * out_embeddings contains spans in document order. */ EMBED_API int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, int batch, - float *out_embeddings); + const embed_context_input_t *inputs, + int batch, + float *out_embeddings); /* Get the config (so embed_cli.c can read hidden_size etc.) */ EMBED_API const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx); diff --git a/embed_server.c b/embed_server.c index 8c7743c..38da125 100644 --- a/embed_server.c +++ b/embed_server.c @@ -47,39 +47,39 @@ #include -#define EMBED_HTTP_IO_BUF 8192 -#define EMBED_HTTP_MAX_HEADER (64u * 1024u) -#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define EMBED_HTTP_MAX_PATH 255u -#define EMBED_HTTP_BACKLOG 128 -#define EMBED_HTTP_SETSIZE 10240 +#define EMBED_HTTP_IO_BUF 8192 +#define EMBED_HTTP_MAX_HEADER (64u * 1024u) +#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define EMBED_HTTP_MAX_PATH 255u +#define EMBED_HTTP_BACKLOG 128 +#define EMBED_HTTP_SETSIZE 10240 #define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 -#define EMBED_HTTP_SWEEP_MS 1000 +#define EMBED_HTTP_SWEEP_MS 1000 -#define EMBED_API_MAX_STANDARD_INPUTS 512 -#define EMBED_API_MAX_CONTEXT_DOCS 512 -#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 +#define EMBED_API_MAX_STANDARD_INPUTS 512 +#define EMBED_API_MAX_CONTEXT_DOCS 512 +#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 #define EMBED_API_MAX_RERANK_DOCUMENTS 1000 -#define EMBED_API_MAX_ITEM_TOKENS 32768 -#define EMBED_API_MAX_TOTAL_TOKENS 120000 -#define EMBED_LATE_QUERY_TOKENS 32 -#define EMBED_LATE_MASK_TOKEN_ID 151642 -#define EMBED_LATE_QUERY_PREFIX_ID 151669 -#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 +#define EMBED_API_MAX_ITEM_TOKENS 32768 +#define EMBED_API_MAX_TOTAL_TOKENS 120000 +#define EMBED_LATE_QUERY_TOKENS 32 +#define EMBED_LATE_MASK_TOKEN_ID 151642 +#define EMBED_LATE_QUERY_PREFIX_ID 151669 +#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ -#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 +#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 #define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 /* Microseconds the worker waits for more requests before dispatching a batch. * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU * busy; MLX and CPU dispatch immediately, where waiting only adds latency. * Override with --batch-wait-us. */ -#define EMBED_SERVER_BATCH_WAIT_US 0 -#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 +#define EMBED_SERVER_BATCH_WAIT_US 0 +#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 #define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 -#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 -#define EMBED_MLX_RESIDENT_MULTIPLIER 2 +#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 +#define EMBED_MLX_RESIDENT_MULTIPLIER 2 typedef struct { char *ptr; @@ -94,19 +94,22 @@ static void die_oom(void) { static void *xmalloc(size_t n) { void *p = malloc(n ? n : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } static void *xcalloc(size_t n, size_t sz) { void *p = calloc(n ? n : 1, sz ? sz : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } static void *xrealloc(void *p, size_t n) { p = realloc(p, n ? n : 1); - if (!p) die_oom(); + if (!p) + die_oom(); return p; } @@ -142,9 +145,7 @@ static uint64_t nstime(void) { return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u; } -static double ns_to_ms(uint64_t ns) { - return (double)ns / 1000000.0; -} +static double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; } static void server_log(const char *fmt, ...) { char ts[32]; @@ -161,12 +162,15 @@ static void server_log(const char *fmt, ...) { } static void sbuf_reserve(sbuf *b, size_t add) { - if (add > SIZE_MAX - b->len - 1) die_oom(); + if (add > SIZE_MAX - b->len - 1) + die_oom(); size_t need = b->len + add + 1; - if (need <= b->cap) return; + if (need <= b->cap) + return; size_t cap = b->cap ? b->cap * 2 : 256; while (cap < need) { - if (cap > SIZE_MAX / 2) die_oom(); + if (cap > SIZE_MAX / 2) + die_oom(); cap *= 2; } b->ptr = xrealloc(b->ptr, cap); @@ -185,7 +189,8 @@ static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } static void sbuf_clear(sbuf *b) { b->len = 0; - if (b->ptr) b->ptr[0] = '\0'; + if (b->ptr) + b->ptr[0] = '\0'; } static void sbuf_printf(sbuf *b, const char *fmt, ...) { @@ -194,11 +199,13 @@ static void sbuf_printf(sbuf *b, const char *fmt, ...) { va_copy(ap2, ap); int n = vsnprintf(NULL, 0, fmt, ap); va_end(ap); - if (n < 0) die_oom(); + if (n < 0) + die_oom(); sbuf_reserve(b, (size_t)n); int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2); va_end(ap2); - if (n2 < 0 || n2 != n) die_oom(); + if (n2 < 0 || n2 != n) + die_oom(); b->len += (size_t)n; } @@ -220,11 +227,7 @@ typedef enum { MODEL_UNKNOWN } model_slot; -typedef enum { - MODEL_KIND_STANDARD, - MODEL_KIND_CONTEXTUAL, - MODEL_KIND_LATE -} model_kind; +typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; /* Which embedding API a model speaks. pplx-embed models follow the Perplexity * API: the canonical output is the tanh int8 quantization, so the default @@ -251,28 +254,30 @@ typedef struct { } model_info; static const model_info k_models[] = { - {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, - {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, - EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, - {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, }; static model_slot model_slot_for_id(const char *id) { - if (!id) return MODEL_UNKNOWN; + if (!id) + return MODEL_UNKNOWN; for (int i = 0; i < MODEL_COUNT; i++) { - if (!strcmp(id, k_models[i].id)) return (model_slot)i; + if (!strcmp(id, k_models[i].id)) + return (model_slot)i; } return MODEL_UNKNOWN; } @@ -398,7 +403,8 @@ static volatile sig_atomic_t g_signal_wfd = -1; static void stop_signal_handler(int sig) { (void)sig; - if (g_stop_requested) _exit(130); + if (g_stop_requested) + _exit(130); int saved = errno; g_stop_requested = 1; int fd = (int)g_signal_wfd; @@ -411,7 +417,8 @@ static void stop_signal_handler(int sig) { static int set_nonblock(int fd) { int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0) return -1; + if (flags < 0) + return -1; return fcntl(fd, F_SETFL, flags | O_NONBLOCK); } @@ -422,7 +429,8 @@ static void http_request_free(http_request *r) { } static void client_free(client *c) { - if (!c) return; + if (!c) + return; http_request_free(&c->req); sbuf_free(&c->in); sbuf_free(&c->out); @@ -438,34 +446,45 @@ static void client_incref(client *c) { static void client_decref(client *c) { int free_now = 0; pthread_mutex_lock(&c->srv->mu); - if (--c->refcount == 0) free_now = 1; + if (--c->refcount == 0) + free_now = 1; pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); + if (free_now) + client_free(c); } static void client_decref_n(client *c, int n) { int free_now = 0; - if (n <= 0) return; + if (n <= 0) + return; pthread_mutex_lock(&c->srv->mu); c->refcount -= n; - if (c->refcount <= 0) free_now = 1; + if (c->refcount <= 0) + free_now = 1; pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); + if (free_now) + client_free(c); } static void client_unlink(client *c) { http_server *s = c->srv; - if (!c->linked) return; - if (c->prev) c->prev->next = c->next; - else s->clients = c->next; - if (c->next) c->next->prev = c->prev; + if (!c->linked) + return; + if (c->prev) + c->prev->next = c->next; + else + s->clients = c->next; + if (c->next) + c->next->prev = c->prev; c->prev = c->next = NULL; c->linked = 0; - if (s->n_clients > 0) s->n_clients--; + if (s->n_clients > 0) + s->n_clients--; } static int close_client_unlink(client *c) { - if (!c || c->cancelled) return 0; + if (!c || c->cancelled) + return 0; c->cancelled = 1; if (c->fd >= 0) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); @@ -478,40 +497,52 @@ static int close_client_unlink(client *c) { } static void close_client(client *c) { - if (!close_client_unlink(c)) return; + if (!close_client_unlink(c)) + return; client_decref(c); } -static void append_http_response_ex(client *c, int status, const char *ctype, +static void append_http_response_ex(client *c, + int status, + const char *ctype, const char *extra_headers, - const char *body, size_t body_len) { + const char *body, + size_t body_len) { const char *reason = "OK"; - if (status == 204) reason = "No Content"; - else if (status == 400) reason = "Bad Request"; - else if (status == 401) reason = "Unauthorized"; - else if (status == 404) reason = "Not Found"; - else if (status == 405) reason = "Method Not Allowed"; - else if (status == 422) reason = "Unprocessable Entity"; - else if (status == 503) reason = "Service Unavailable"; - else if (status >= 500) reason = "Internal Server Error"; + if (status == 204) + reason = "No Content"; + else if (status == 400) + reason = "Bad Request"; + else if (status == 401) + reason = "Unauthorized"; + else if (status == 404) + reason = "Not Found"; + else if (status == 405) + reason = "Method Not Allowed"; + else if (status == 422) + reason = "Unprocessable Entity"; + else if (status == 503) + reason = "Service Unavailable"; + else if (status >= 500) + reason = "Internal Server Error"; sbuf_printf(&c->out, "HTTP/1.1 %d %s\r\n" "Content-Length: %zu\r\n" "Connection: %s\r\n", - status, reason, body_len, - c->close_after_write ? "close" : "keep-alive"); + status, reason, body_len, c->close_after_write ? "close" : "keep-alive"); if (ctype) sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); if (extra_headers) sbuf_puts(&c->out, extra_headers); sbuf_puts(&c->out, "\r\n"); - if (body_len) sbuf_append(&c->out, body, body_len); + if (body_len) + sbuf_append(&c->out, body, body_len); } -static void append_http_response(client *c, int status, const char *ctype, - const char *body, size_t body_len) { +static void +append_http_response(client *c, int status, const char *ctype, const char *body, size_t body_len) { append_http_response_ex(c, status, ctype, NULL, body, body_len); } @@ -534,13 +565,27 @@ static void append_json_string(sbuf *b, const char *s) { sbuf_putc(b, '"'); for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) { switch (*p) { - case '"': sbuf_puts(b, "\\\""); break; - case '\\': sbuf_puts(b, "\\\\"); break; - case '\b': sbuf_puts(b, "\\b"); break; - case '\f': sbuf_puts(b, "\\f"); break; - case '\n': sbuf_puts(b, "\\n"); break; - case '\r': sbuf_puts(b, "\\r"); break; - case '\t': sbuf_puts(b, "\\t"); break; + case '"': + sbuf_puts(b, "\\\""); + break; + case '\\': + sbuf_puts(b, "\\\\"); + break; + case '\b': + sbuf_puts(b, "\\b"); + break; + case '\f': + sbuf_puts(b, "\\f"); + break; + case '\n': + sbuf_puts(b, "\\n"); + break; + case '\r': + sbuf_puts(b, "\\r"); + break; + case '\t': + sbuf_puts(b, "\\t"); + break; default: if (*p < 0x20) sbuf_printf(b, "\\u%04x", (unsigned)*p); @@ -566,8 +611,10 @@ static void job_set_error(job *j, int status, const char *message, const char *t } static bool auth_ok(http_server *s, const char *auth) { - if (!s->api_key || !s->api_key[0]) return true; - if (!auth) return false; + if (!s->api_key || !s->api_key[0]) + return true; + if (!auth) + return false; const char prefix[] = "Bearer "; return !strncmp(auth, prefix, sizeof(prefix) - 1) && !strcmp(auth + sizeof(prefix) - 1, s->api_key); @@ -575,12 +622,12 @@ static bool auth_ok(http_server *s, const char *auth) { static ssize_t find_header_end(const char *p, size_t n) { for (size_t i = 3; i < n; i++) { - if (p[i - 3] == '\r' && p[i - 2] == '\n' && - p[i - 1] == '\r' && p[i] == '\n') + if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n') return (ssize_t)(i + 1); } for (size_t i = 1; i < n; i++) { - if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); + if (p[i - 1] == '\n' && p[i] == '\n') + return (ssize_t)(i + 1); } return -1; } @@ -590,18 +637,22 @@ static char *header_value_dup(const char *h, size_t n, const char *name) { const char *p = h, *end = h + n; while (p < end) { const char *line = p; - while (p < end && *p != '\n') p++; + while (p < end && *p != '\n') + p++; size_t len = (size_t)(p - line); - if (len && line[len - 1] == '\r') len--; - if (len > name_len && !strncasecmp(line, name, name_len) && - line[name_len] == ':') { + if (len && line[len - 1] == '\r') + len--; + if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') { const char *v = line + name_len + 1; - while (v < line + len && isspace((unsigned char)*v)) v++; + while (v < line + len && isspace((unsigned char)*v)) + v++; const char *e = line + len; - while (e > v && isspace((unsigned char)e[-1])) e--; + while (e > v && isspace((unsigned char)e[-1])) + e--; return xstrndup(v, (size_t)(e - v)); } - if (p < end) p++; + if (p < end) + p++; } return NULL; } @@ -610,13 +661,15 @@ static int header_has_token(const char *value, const char *token) { size_t token_len = strlen(token); const char *p = value; while (p && *p) { - while (*p == ',' || isspace((unsigned char)*p)) p++; + while (*p == ',' || isspace((unsigned char)*p)) + p++; const char *start = p; - while (*p && *p != ',') p++; + while (*p && *p != ',') + p++; const char *end = p; - while (end > start && isspace((unsigned char)end[-1])) end--; - if ((size_t)(end - start) == token_len && - !strncasecmp(start, token, token_len)) + while (end > start && isspace((unsigned char)end[-1])) + end--; + if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len)) return 1; } return 0; @@ -630,14 +683,14 @@ static int parse_request_headers(client *c) { i++; } line[i] = '\0'; - int fields = sscanf(line, "%7s %255s %15s", - c->req.method, c->req.path, c->req.version); + int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version); if (fields < 2) return -1; if (fields < 3) snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); char *q = strchr(c->req.path, '?'); - if (q) *q = '\0'; + if (q) + *q = '\0'; c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); @@ -669,15 +722,19 @@ static int complete_request(client *c) { if (!c->header_done) { ssize_t hend = find_header_end(c->in.ptr, c->in.len); if (hend < 0) { - if (c->in.len > EMBED_HTTP_MAX_HEADER) return -1; + if (c->in.len > EMBED_HTTP_MAX_HEADER) + return -1; return 0; } c->header_done = 1; c->header_len = (size_t)hend; - if (parse_request_headers(c) != 0) return -1; + if (parse_request_headers(c) != 0) + return -1; } - if (c->content_length > EMBED_HTTP_MAX_BODY) return -1; - if (c->in.len < c->header_len + c->content_length) return 0; + if (c->content_length > EMBED_HTTP_MAX_BODY) + return -1; + if (c->in.len < c->header_len + c->content_length) + return 0; c->req.body_len = c->content_length; c->req.body = xmalloc(c->content_length + 1); memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); @@ -687,23 +744,25 @@ static int complete_request(client *c) { static void send_bad_http_request(client *c) { size_t len; - char *body = json_error_body("bad HTTP request", "invalid_request_error", - &len); + char *body = json_error_body("bad HTTP request", "invalid_request_error", &len); c->close_after_write = 1; append_http_response(c, 400, "application/json", body, len); free(body); aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); } static void reset_client_for_next_request(client *c) { size_t consumed = c->header_len + c->content_length; - if (consumed > c->in.len) consumed = c->in.len; + if (consumed > c->in.len) + consumed = c->in.len; size_t remain = c->in.len - consumed; if (remain) memmove(c->in.ptr, c->in.ptr + consumed, remain); c->in.len = remain; - if (c->in.ptr) c->in.ptr[remain] = '\0'; + if (c->in.ptr) + c->in.ptr[remain] = '\0'; http_request_free(&c->req); sbuf_clear(&c->out); @@ -716,7 +775,8 @@ static void reset_client_for_next_request(client *c) { static void arm_next_request(client *c) { reset_client_for_next_request(c); - if (c->cancelled || c->fd < 0) return; + if (c->cancelled || c->fd < 0) + return; if (c->in.len) { int done = complete_request(c); @@ -735,7 +795,8 @@ static void arm_next_request(client *c) { } static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; client *c = clientData; while (c->sent < c->out.len) { ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); @@ -744,26 +805,33 @@ static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { c->last_active_ms = mstime(); continue; } - if (n < 0 && errno == EINTR) continue; - if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; + if (n < 0 && errno == EINTR) + continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + return; close_client(c); return; } aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); - if (c->close_after_write) close_client(c); - else arm_next_request(c); + if (c->close_after_write) + close_client(c); + else + arm_next_request(c); } static int queue_write(client *c) { - if (!c || c->cancelled || c->fd < 0) return -1; + if (!c || c->cancelled || c->fd < 0) + return -1; return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); } static void enqueue_done(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); - if (s->done_tail) s->done_tail->next = j; - else s->done_head = j; + if (s->done_tail) + s->done_tail->next = j; + else + s->done_head = j; s->done_tail = j; pthread_mutex_unlock(&s->mu); const unsigned char byte = 1; @@ -774,16 +842,17 @@ static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { s->job_head = j->next; - if (!s->job_head) s->job_tail = NULL; + if (!s->job_head) + s->job_tail = NULL; j->next = NULL; } return j; } -static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, - uint64_t deadline_ns) { +static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) { uint64_t now_ns = nstime(); - if (now_ns >= deadline_ns) return ETIMEDOUT; + if (now_ns >= deadline_ns) + return ETIMEDOUT; uint64_t remaining_ns = deadline_ns - now_ns; struct timespec abs; @@ -803,7 +872,8 @@ static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, } static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { - if (max_jobs <= 0) return 0; + if (max_jobs <= 0) + return 0; pthread_mutex_lock(&s->mu); while (!s->job_head && !s->stopping) @@ -817,9 +887,8 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { int n = 1; jobs[0] = first; - int batchable = s->batch_size > 1 && - (!strcmp(first->path, "/v1/embeddings") || - !strcmp(first->path, "/v1/contextualizedembeddings")); + int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") || + !strcmp(first->path, "/v1/contextualizedembeddings")); if (!batchable) { pthread_mutex_unlock(&s->mu); return n; @@ -828,12 +897,13 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { /* Exact item and token budgets are applied after worker-side tokenization; * this queue stage only defines which arrival window may be grouped. */ uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u; - uint64_t deadline_ns = first->created_ns > UINT64_MAX - wait_ns - ? UINT64_MAX : first->created_ns + wait_ns; + uint64_t deadline_ns = + first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns; while (n < max_jobs) { while (!s->job_head && !s->stopping && wait_ns > 0) { int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns); - if (rc == ETIMEDOUT) break; + if (rc == ETIMEDOUT) + break; } if (!s->job_head) break; @@ -850,8 +920,10 @@ static int collect_job_batch(http_server *s, job **jobs, int max_jobs) { static void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); - if (s->job_tail) s->job_tail->next = j; - else s->job_head = j; + if (s->job_tail) + s->job_tail->next = j; + else + s->job_head = j; s->job_tail = j; pthread_cond_signal(&s->cv); pthread_mutex_unlock(&s->mu); @@ -862,7 +934,8 @@ static job *pop_done(http_server *s) { job *j = s->done_head; if (j) { s->done_head = j->next; - if (!s->done_head) s->done_tail = NULL; + if (!s->done_head) + s->done_tail = NULL; j->next = NULL; } pthread_mutex_unlock(&s->mu); @@ -870,7 +943,8 @@ static job *pop_done(http_server *s) { } static void job_free(job *j) { - if (!j) return; + if (!j) + return; free(j->body); free(j->auth); free(j->extra_headers); @@ -879,18 +953,19 @@ static void job_free(job *j) { } static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; http_server *s = clientData; unsigned char tmp[128]; - while (read(fd, tmp, sizeof(tmp)) > 0) {} + while (read(fd, tmp, sizeof(tmp)) > 0) { + } job *j; while ((j = pop_done(s)) != NULL) { client *c = j->c; int refs_to_drop = 1; /* job reference */ if (!c->cancelled) { - append_http_response_ex(c, j->status, j->content_type, - j->extra_headers, + append_http_response_ex(c, j->status, j->content_type, j->extra_headers, j->response ? j->response : "", j->response ? j->response_len : 0); if (queue_write(c) != AE_OK) @@ -903,8 +978,7 @@ static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) static bool route_is_inference(const char *method, const char *path) { return !strcmp(method, "POST") && - (!strcmp(path, "/v1/embeddings") || - !strcmp(path, "/v1/contextualizedembeddings") || + (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") || !strcmp(path, "/v1/rerank")); } @@ -913,7 +987,8 @@ static void dispatch_request(client *c) { if (!strcmp(c->req.method, "OPTIONS")) { append_http_response(c, 204, NULL, "", 0); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } @@ -922,17 +997,19 @@ static void dispatch_request(client *c) { char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); append_http_response(c, 404, "application/json", body, len); free(body); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } if (!auth_ok(c->srv, c->req.auth)) { size_t len; - char *body = json_error_body("missing or invalid bearer token", - "authentication_error", &len); + char *body = + json_error_body("missing or invalid bearer token", "authentication_error", &len); append_http_response(c, 401, "application/json", body, len); free(body); - if (queue_write(c) != AE_OK) close_client(c); + if (queue_write(c) != AE_OK) + close_client(c); return; } @@ -950,7 +1027,8 @@ static void dispatch_request(client *c) { } static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; client *c = clientData; char tmp[EMBED_HTTP_IO_BUF]; for (;;) { @@ -973,23 +1051,28 @@ static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { close_client(c); return; } - if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EWOULDBLOCK) return; + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; close_client(c); return; } } static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; + (void)loop; + (void)mask; http_server *s = clientData; char cip[64]; int cport = 0; for (;;) { int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); if (cfd == ANET_ERR) { - if (errno == EAGAIN || errno == EWOULDBLOCK) return; - if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + if (errno == EINTR) + continue; return; } set_nonblock(cfd); @@ -1000,7 +1083,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { c->last_active_ms = mstime(); c->linked = 1; c->next = s->clients; - if (s->clients) s->clients->prev = c; + if (s->clients) + s->clients->prev = c; s->clients = c; s->n_clients++; if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) @@ -1009,7 +1093,8 @@ static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { } static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { - (void)loop; (void)id; + (void)loop; + (void)id; http_server *s = clientData; uint64_t now = mstime(); client *c = s->clients; @@ -1027,15 +1112,16 @@ static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { } static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)fd; (void)mask; + (void)fd; + (void)mask; http_server *s = clientData; unsigned char tmp[128]; - while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {} + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { + } aeStop(loop); } -static bool ve_add(cJSON *detail, const char *loc_json, - const char *msg, const char *type) { +static bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) { cJSON *obj = cJSON_CreateObject(); cJSON *loc = cJSON_Parse(loc_json); if (!obj || !loc) { @@ -1067,21 +1153,19 @@ static void job_set_422(job *j, cJSON *detail) { } static void job_set_timing_header(job *j) { - if (!j || j->extra_headers || !j->created_ns) return; + if (!j || j->extra_headers || !j->created_ns) + return; uint64_t done_ns = nstime(); - uint64_t queue_ns = j->started_ns > j->created_ns - ? j->started_ns - j->created_ns : 0; - uint64_t worker_ns = j->started_ns && done_ns > j->started_ns - ? done_ns - j->started_ns : 0; + uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0; + uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0; char buf[384]; int n = snprintf(buf, sizeof(buf), - "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " - "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " - "worker;dur=%.3f\r\n", - ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), - ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns), - ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); + "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " + "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " + "worker;dur=%.3f\r\n", + ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns), + ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); if (n > 0 && (size_t)n < sizeof(buf)) j->extra_headers = xstrdup(buf); } @@ -1098,7 +1182,8 @@ static cJSON *parse_json_body(job *j, cJSON *detail) { return NULL; } const char *body_end = j->body + j->body_len; - while (end && end < body_end && isspace((unsigned char)*end)) end++; + while (end && end < body_end && isspace((unsigned char)*end)) + end++; if (!end || end != body_end) { ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); cJSON_Delete(root); @@ -1113,30 +1198,30 @@ static cJSON *parse_json_body(job *j, cJSON *detail) { } static bool cjson_is_integer(cJSON *item) { - if (!cJSON_IsNumber(item)) return false; + if (!cJSON_IsNumber(item)) + return false; double d = item->valuedouble; int i = item->valueint; return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; } -static const char *encoding_from_root(cJSON *root, cJSON *detail, - embedding_api_t api) { +static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) { /* Default and accepted encodings follow the model's API family: Perplexity * defaults to base64_int8 and accepts {base64_int8, base64_binary, float}; * OpenAI/DashScope (Qwen3) defaults to float and accepts {float, base64}. */ const char *dflt = api == EMBED_API_OPENAI ? "float" : "base64_int8"; cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); - if (!encoding) return dflt; + if (!encoding) + return dflt; if (!cJSON_IsString(encoding) || !encoding->valuestring) { - ve_add(detail, "[\"body\",\"encoding_format\"]", - "encoding_format must be a string", "type_error.string"); + ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string", + "type_error.string"); return dflt; } const char *v = encoding->valuestring; - int ok = api == EMBED_API_OPENAI - ? (!strcmp(v, "float") || !strcmp(v, "base64")) - : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || - !strcmp(v, "float")); + int ok = api == EMBED_API_OPENAI ? (!strcmp(v, "float") || !strcmp(v, "base64")) + : (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || + !strcmp(v, "float")); if (!ok) { ve_add(detail, "[\"body\",\"encoding_format\"]", api == EMBED_API_OPENAI @@ -1149,8 +1234,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail, return v; } -static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, - int max_dim, const char *encoding) { +static int +dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) { int dims = max_dim; cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); if (dimensions) { @@ -1161,31 +1246,30 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, dims = dimensions->valueint; if (dims < min_dim || dims > max_dim) { char msg[96]; - snprintf(msg, sizeof(msg), - "dimensions must be between %d and %d", - min_dim, max_dim); + snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, + max_dim); ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); } } } if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) ve_add(detail, "[\"body\",\"dimensions\"]", - "dimensions must be divisible by 8 for base64_binary", - "value_error.divisible"); + "dimensions must be divisible by 8 for base64_binary", "value_error.divisible"); return dims; } static char *base64_encode(const unsigned char *src, size_t n) { - static const char tbl[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; size_t out_len = ((n + 2) / 3) * 4; char *out = xmalloc(out_len + 1); size_t o = 0; for (size_t i = 0; i < n; i += 3) { unsigned v = (unsigned)src[i] << 16; int remain = (int)(n - i); - if (remain > 1) v |= (unsigned)src[i + 1] << 8; - if (remain > 2) v |= (unsigned)src[i + 2]; + if (remain > 1) + v |= (unsigned)src[i + 1] << 8; + if (remain > 2) + v |= (unsigned)src[i + 2]; out[o++] = tbl[(v >> 18) & 63]; out[o++] = tbl[(v >> 12) & 63]; out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '='; @@ -1197,8 +1281,10 @@ static char *base64_encode(const unsigned char *src, size_t n) { static signed char quantize_int8_tanh(float x) { int v = (int)lrintf(tanhf(x) * 127.0f); - if (v > 127) v = 127; - if (v < -128) v = -128; + if (v > 127) + v = 127; + if (v < -128) + v = -128; return (signed char)v; } @@ -1295,54 +1381,62 @@ typedef struct { } rerank_request; static void free_token_bufs(token_buf *t, int n) { - if (!t) return; - for (int i = 0; i < n; i++) free(t[i].ids); + if (!t) + return; + for (int i = 0; i < n; i++) + free(t[i].ids); } static void embedding_request_free(embedding_request *r) { - if (!r) return; + if (!r) + return; free_token_bufs(r->tokens, r->n_inputs); free(r->tokens); free(r->inputs); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void contextual_request_free(contextual_request *r) { - if (!r) return; + if (!r) + return; for (int i = 0; i < r->n_docs; i++) { free(r->docs[i].ids); free(r->docs[i].spans); } free(r->docs); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } static void late_tokens_free(late_tokens *t) { - if (!t) return; + if (!t) + return; free(t->ids); free(t->keep); memset(t, 0, sizeof(*t)); } static void rerank_request_free(rerank_request *r) { - if (!r) return; + if (!r) + return; late_tokens_free(&r->query); for (int i = 0; i < r->n_documents; i++) late_tokens_free(&r->documents[i]); free(r->documents); - if (r->root) cJSON_Delete(r->root); + if (r->root) + cJSON_Delete(r->root); memset(r, 0, sizeof(*r)); } -static int tokenize_one(loaded_model *m, job *j, const char *text, - token_buf *out) { +static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); uint64_t t0 = nstime(); - out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, - text, &out->n_tokens); - if (j) j->tokenize_ns += nstime() - t0; + out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); + if (j) + j->tokenize_ns += nstime() - t0; if (!out->ids || out->n_tokens <= 0) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1354,8 +1448,7 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, memset(out, 0, sizeof(*out)); return -1; } - int *ids = realloc( - out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); + int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); if (!ids) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1369,15 +1462,17 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, static int late_id_is_skipped(const loaded_model *m, int id) { for (int i = 0; i < m->n_late_skip_ids; i++) { - if (m->late_skip_ids[i] == id) return 1; + if (m->late_skip_ids[i] == id) + return 1; } return 0; } -static int tokenize_late_text(loaded_model *m, job *j, const char *text, - int is_query, late_tokens *out) { +static int +tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { token_buf raw = {0}; - if (tokenize_one(m, j, text, &raw) != 0) return -1; + if (tokenize_one(m, j, text, &raw) != 0) + return -1; int raw_tokens = raw.n_tokens; int target = raw_tokens + 1; @@ -1386,11 +1481,9 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text, out->ids = xmalloc((size_t)target * sizeof(*out->ids)); out->ids[0] = raw.ids[0]; - out->ids[1] = is_query ? m->late_query_prefix_id - : m->late_document_prefix_id; + out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id; if (raw_tokens > 1) { - memcpy(out->ids + 2, raw.ids + 1, - (size_t)(raw_tokens - 1) * sizeof(*out->ids)); + memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids)); } out->n_tokens = raw_tokens + 1; if (is_query) { @@ -1408,8 +1501,7 @@ static int tokenize_late_text(loaded_model *m, job *j, const char *text, return out->n_keep > 0 ? 0 : -1; } -static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, - int batch, float *out) { +static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out); @@ -1423,7 +1515,8 @@ static int model_embed_batch(loaded_model *m, const embed_input_t *inputs, static int model_embed_spans_batch(loaded_model *m, const embed_context_input_t *inputs, - int batch, float *out) { + int batch, + float *out) { #ifdef USE_MLX if (m->mlx_ctx) return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out); @@ -1432,8 +1525,7 @@ static int model_embed_spans_batch(loaded_model *m, if (m->cuda_ctx) return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out); #endif - return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, - out); + return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } static int model_uses_dense_batches(const loaded_model *m) { @@ -1447,10 +1539,10 @@ static int model_uses_dense_batches(const loaded_model *m) { /* Inputs are sorted by length before chunking. CPU packs real tokens while MLX * pads each dense row to the longest input in the chunk. */ -static int inference_batch_accepts_input(const loaded_model *m, int batch, - int packed_tokens, int next_tokens, - int max_batch_tokens) { - if (batch == 0) return 1; +static int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { + if (batch == 0) + return 1; if (model_uses_dense_batches(m)) return next_tokens <= max_batch_tokens / (batch + 1); return next_tokens <= max_batch_tokens - packed_tokens; @@ -1466,17 +1558,14 @@ static void append_embedding_object(sbuf *b, int index, const char *embedding) { * "float" renders a JSON array: the true float32 vector for OpenAI/DashScope * (Qwen3), or the int8-decoded view (int8/128) for Perplexity (pplx). Every * other encoding renders a base64 string via encode_embedding(). */ -static void append_embedding_value(sbuf *b, int index, const float *emb, - int dims, const char *encoding, - embedding_api_t api) { +static void append_embedding_value( + sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { if (!strcmp(encoding, "float")) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", - index); + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); for (int i = 0; i < dims; i++) { - if (i) sbuf_putc(b, ','); - float v = api == EMBED_API_OPENAI - ? emb[i] - : (float)quantize_int8_tanh(emb[i]) / 128.0f; + if (i) + sbuf_putc(b, ','); + float v = api == EMBED_API_OPENAI ? emb[i] : (float)quantize_int8_tanh(emb[i]) / 128.0f; sbuf_printf(b, "%.9g", (double)v); } sbuf_puts(b, "]}"); @@ -1500,12 +1589,10 @@ static int render_embedding_response(embedding_request *r, const float *embs) { const float *render_embs = embs; float *truncated = NULL; if (r->model->renormalize_truncated && r->dims < full_dim) { - truncated = xmalloc( - (size_t)r->n_inputs * r->dims * sizeof(*truncated)); + truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated)); for (int i = 0; i < r->n_inputs; i++) { float *dst = truncated + (size_t)i * r->dims; - memcpy(dst, embs + (size_t)i * full_dim, - (size_t)r->dims * sizeof(*dst)); + memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); if (embed_l2_normalize(dst, r->dims) != 0) { free(truncated); return -1; @@ -1519,25 +1606,23 @@ static int render_embedding_response(embedding_request *r, const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); for (int i = 0; i < r->n_inputs; i++) { - if (i) sbuf_putc(&b, ','); - append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, - r->dims, r->encoding, api); + if (i) + sbuf_putc(&b, ','); + append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding, + api); } sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); set_response_from_buf(r->j, &b); free(truncated); return 0; } -static int embedding_request_compatible(const embedding_request *a, - const embedding_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && +static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } @@ -1549,21 +1634,27 @@ typedef struct { static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; const embedding_batch_item *ib = b; - if (ia->input.n_tokens < ib->input.n_tokens) return -1; - if (ia->input.n_tokens > ib->input.n_tokens) return 1; + if (ia->input.n_tokens < ib->input.n_tokens) + return -1; + if (ia->input.n_tokens > ib->input.n_tokens) + return 1; return ia->output_index - ib->output_index; } static int embedding_request_fits_group(const embedding_request *r, - int group_inputs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_inputs == 0) return 1; + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_inputs == 0) + return 1; return r->n_inputs <= max_batch - group_inputs && r->total_tokens <= max_batch_tokens - group_tokens; } static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { - if (n_reqs <= 0) return; + if (n_reqs <= 0) + return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_inputs = 0; @@ -1583,9 +1674,9 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) } qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_inputs; - if (max_batch > total_inputs) max_batch = total_inputs; + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) + max_batch = total_inputs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); @@ -1597,8 +1688,7 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) int tokens = 0; while (start + cur < total_inputs && cur < max_batch) { int n_tokens = items[start + cur].input.n_tokens; - if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, - max_batch_tokens)) + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) break; inputs[cur] = items[start + cur].input; tokens += n_tokens; @@ -1612,8 +1702,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) break; } for (int i = 0; i < cur; i++) - memcpy(embs + (size_t)items[start + i].output_index * dim, - batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); + memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, + (size_t)dim * sizeof(float)); start += cur; } @@ -1624,10 +1714,8 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); } else { uint64_t t0 = nstime(); - if (render_embedding_response( - reqs[i], embs + (size_t)pos * dim) != 0) - job_set_error(reqs[i]->j, 500, "embedding normalization failed", - "server_error"); + if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) + job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); reqs[i]->j->encode_ns += nstime() - t0; } pos += reqs[i]->n_inputs; @@ -1639,41 +1727,40 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) free(items); } -static void render_contextual_response(contextual_request *r, - const float *embs) { +static void render_contextual_response(contextual_request *r, const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); int span_offset = 0; for (int di = 0; di < r->n_docs; di++) { - if (di) sbuf_putc(&b, ','); + if (di) + sbuf_putc(&b, ','); sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); for (int ci = 0; ci < r->docs[di].n_spans; ci++) { - if (ci) sbuf_putc(&b, ','); + if (ci) + sbuf_putc(&b, ','); const float *emb = embs + (size_t)span_offset * r->model->info->dim; - append_embedding_value(&b, ci, emb, r->dims, r->encoding, - r->model->info->api); + append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api); span_offset++; } sbuf_puts(&b, "]}"); } - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); + sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); set_response_from_buf(r->j, &b); } -static int contextual_request_compatible(const contextual_request *a, - const contextual_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && +static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); } static int contextual_request_fits_group(const contextual_request *r, - int group_docs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_docs == 0) return 1; + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_docs == 0) + return 1; return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; } @@ -1687,14 +1774,16 @@ typedef struct { static int contextual_batch_item_cmp(const void *a, const void *b) { const contextual_batch_item *ia = a; const contextual_batch_item *ib = b; - if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; - if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) + return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) + return 1; return ia->order - ib->order; } -static void execute_contextual_request_list(contextual_request **reqs, - int n_reqs) { - if (n_reqs <= 0) return; +static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; loaded_model *m = reqs[0]->model; int dim = m->info->dim; int total_docs = 0; @@ -1703,8 +1792,7 @@ static void execute_contextual_request_list(contextual_request **reqs, if (total_docs > INT_MAX - reqs[i]->n_docs || total_chunks > INT_MAX - reqs[i]->total_chunks) { for (int k = 0; k < n_reqs; k++) - job_set_error(reqs[k]->j, 500, "contextual batch is too large", - "server_error"); + job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); return; } total_docs += reqs[i]->n_docs; @@ -1712,13 +1800,11 @@ static void execute_contextual_request_list(contextual_request **reqs, } if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { for (int i = 0; i < n_reqs; i++) - job_set_error(reqs[i]->j, 500, "contextual batch is too large", - "server_error"); + job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); return; } - contextual_batch_item *items = - xmalloc((size_t)total_docs * sizeof(*items)); + contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); int pos = 0; int span_offset = 0; @@ -1737,12 +1823,11 @@ static void execute_contextual_request_list(contextual_request **reqs, } qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_docs; - if (max_batch > total_docs) max_batch = total_docs; + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) + max_batch = total_docs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_context_input_t *inputs = - xmalloc((size_t)max_batch * sizeof(*inputs)); + embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc(sizeof(*batch_embs)); size_t batch_emb_cap = 0; @@ -1754,8 +1839,7 @@ static void execute_contextual_request_list(contextual_request **reqs, int chunks = 0; while (start + cur < total_docs && cur < max_batch) { const embed_context_input_t *input = &items[start + cur].input; - if (!inference_batch_accepts_input(m, cur, tokens, - input->input.n_tokens, + if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, max_batch_tokens)) break; inputs[cur] = *input; @@ -1779,8 +1863,7 @@ static void execute_contextual_request_list(contextual_request **reqs, for (int i = 0; i < cur; i++) { int n_spans = inputs[i].n_spans; memcpy(embs + (size_t)items[start + i].output_span_index * dim, - batch_embs + (size_t)batch_span * dim, - (size_t)n_spans * dim * sizeof(*embs)); + batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); batch_span += n_spans; } start += cur; @@ -1790,12 +1873,10 @@ static void execute_contextual_request_list(contextual_request **reqs, for (int i = 0; i < n_reqs; i++) { reqs[i]->j->infer_ns += infer_ns; if (failed) { - job_set_error(reqs[i]->j, 500, "contextual embedding failed", - "server_error"); + job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); } else { uint64_t t0 = nstime(); - render_contextual_response(reqs[i], - embs + (size_t)span_offset * dim); + render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); reqs[i]->j->encode_ns += nstime() - t0; } span_offset += reqs[i]->total_chunks; @@ -1807,9 +1888,13 @@ static void execute_contextual_request_list(contextual_request **reqs, free(items); } -static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, - loaded_model **out_model, int *out_dims, - const char **out_encoding, http_server *s) { +static int validate_common(cJSON *root, + cJSON *detail, + model_kind expected_kind, + loaded_model **out_model, + int *out_dims, + const char **out_encoding, + http_server *s) { cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); const char *model_id = NULL; if (!model_item) { @@ -1832,8 +1917,7 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, } } model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; - embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api - : EMBED_API_PERPLEXITY; + embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; const char *enc = encoding_from_root(root, detail, api); int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; @@ -1841,11 +1925,12 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, *out_encoding = enc; *out_dims = dims; return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && - !s->models[model_slot_for_id(model_id)].info ? 503 : 0; + !s->models[model_slot_for_id(model_id)].info + ? 503 + : 0; } -static void prepare_embedding_request(job *j, cJSON *root, http_server *s, - embedding_request *out) { +static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -1854,8 +1939,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, - &m, &dims, &encoding, s); + int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; @@ -1866,8 +1950,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (cJSON_IsString(input)) { if (!input->valuestring || input->valuestring[0] == '\0') - ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", - "value_error.empty"); + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); n_inputs = 1; } else if (cJSON_IsArray(input)) { n_inputs = cJSON_GetArraySize(input); @@ -1890,13 +1973,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, i++; } } else { - ve_add(detail, "[\"body\",\"input\"]", - "input must be a string or an array of strings", "type_error"); + ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", + "type_error"); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -1916,8 +1998,7 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, out->inputs[0].n_tokens = out->tokens[0].n_tokens; out->total_tokens = out->tokens[0].n_tokens; if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", - "input exceeds 32768 token limit", + ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", "value_error.context_length"); } } else { @@ -1926,15 +2007,12 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, loc, "input item must be a string", - "type_error.string"); + ve_add(detail, loc, "input item must be a string", "type_error.string"); idx++; continue; } - if (tokenize_one(m, j, item->valuestring, - &out->tokens[idx]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_one(m, j, item->valuestring, &out->tokens[idx]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { out->inputs[idx].ids = out->tokens[idx].ids; out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; @@ -1962,8 +2040,8 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, cJSON_Delete(detail); } -static void prepare_contextual_request(job *j, cJSON *root, http_server *s, - contextual_request *out) { +static void +prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -1972,8 +2050,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, - &m, &dims, &encoding, s); + int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); out->model = m; out->dims = dims; out->encoding = encoding; @@ -1984,8 +2061,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, if (!input) { ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (!cJSON_IsArray(input)) { - ve_add(detail, "[\"body\",\"input\"]", - "input must be an array of document chunk arrays", "type_error.array"); + ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", + "type_error.array"); } else { n_docs = cJSON_GetArraySize(input); if (n_docs < 1) @@ -2000,8 +2077,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, if (!cJSON_IsArray(doc_arr)) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must be an array of strings", - "type_error.array"); + ve_add(detail, loc, "document must be an array of strings", "type_error.array"); di++; continue; } @@ -2036,8 +2112,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -2051,8 +2126,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(doc_arr, input) { contextual_doc *doc = &out->docs[di]; int n_chunks = cJSON_GetArraySize(doc_arr); - token_buf *chunk_tokens = - xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); doc->n_spans = n_chunks; int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; @@ -2063,10 +2137,8 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(chunk, doc_arr) { char loc[80]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (tokenize_one(m, j, chunk->valuestring, - &chunk_tokens[ci]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); doc_valid = 0; } else { doc_tokens += chunk_tokens[ci].n_tokens; @@ -2103,8 +2175,7 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, di++; } if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { - ve_add(detail, "[\"body\",\"input\"]", - "request exceeds 120000 token limit", + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } else { out->total_tokens = (int)request_tokens; @@ -2122,30 +2193,28 @@ static void prepare_contextual_request(job *j, cJSON *root, http_server *s, cJSON_Delete(detail); } -static int validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, - loaded_model **out_model) { +static int +validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); if (!item) { ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); return 0; } if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, "[\"body\",\"model\"]", "model must be a string", - "type_error.string"); + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); return 0; } model_slot slot = model_slot_for_id(item->valuestring); if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { - ve_add(detail, "[\"body\",\"model\"]", - "value is not a valid enum member for this endpoint", "enum"); + ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", + "enum"); return 0; } *out_model = s->models[slot].info ? &s->models[slot] : NULL; return *out_model ? 0 : 503; } -static void prepare_rerank_request(job *j, cJSON *root, http_server *s, - rerank_request *out) { +static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { memset(out, 0, sizeof(*out)); out->j = j; out->root = root; @@ -2157,28 +2226,24 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, if (!query) { ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); } else if (!cJSON_IsString(query)) { - ve_add(detail, "[\"body\",\"query\"]", "query must be a string", - "type_error.string"); + ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); } else if (!query->valuestring || !query->valuestring[0]) { - ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", - "value_error.empty"); + ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); } int n_documents = 0; if (!documents) { ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); } else if (!cJSON_IsArray(documents)) { - ve_add(detail, "[\"body\",\"documents\"]", - "documents must be an array of strings", "type_error.array"); + ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", + "type_error.array"); } else { n_documents = cJSON_GetArraySize(documents); if (n_documents < 1) - ve_add(detail, "[\"body\",\"documents\"]", - "documents must contain at least 1 item", + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", "value_error.list.min_items"); else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) - ve_add(detail, "[\"body\",\"documents\"]", - "documents must contain at most 1000 items", + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", "value_error.list.max_items"); cJSON *document; int i = 0; @@ -2186,51 +2251,43 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); if (!cJSON_IsString(document)) - ve_add(detail, loc, "document must be a string", - "type_error.string"); + ve_add(detail, loc, "document must be a string", "type_error.string"); else if (!document->valuestring || !document->valuestring[0]) - ve_add(detail, loc, "document must not be empty", - "value_error.empty"); + ve_add(detail, loc, "document must not be empty", "value_error.empty"); } } cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); if (top_n && top_k) { - ve_add(detail, "[\"body\",\"top_n\"]", - "top_n and top_k are aliases; provide only one", + ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", "value_error.conflict"); } cJSON *top = top_n ? top_n : top_k; out->top_n = n_documents; if (top) { if (!cjson_is_integer(top)) { - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" - : "[\"body\",\"top_k\"]", + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", "top_n must be an integer", "type_error.integer"); } else { out->top_n = top->valueint; if (out->top_n < 1 || out->top_n > n_documents) - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" - : "[\"body\",\"top_k\"]", - "top_n must be between 1 and the number of documents", - "value_error.range"); + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be between 1 and the number of documents", "value_error.range"); } } - cJSON *return_documents = - cJSON_GetObjectItemCaseSensitive(root, "return_documents"); + cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); if (return_documents) { if (!cJSON_IsBool(return_documents)) - ve_add(detail, "[\"body\",\"return_documents\"]", - "return_documents must be a boolean", "type_error.bool"); + ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", + "type_error.bool"); else out->return_documents = cJSON_IsTrue(return_documents); } if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); cJSON_Delete(detail); return; } @@ -2238,15 +2295,13 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, if (cJSON_GetArraySize(detail) == 0 && out->model) { out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); - if (tokenize_late_text(out->model, j, query->valuestring, 1, - &out->query) != 0) { + if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) { ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); } else { out->query_tokens = out->query.n_tokens; if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"query\"]", - "query exceeds 32768 token limit", + ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", "value_error.context_length"); } @@ -2255,10 +2310,9 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, cJSON_ArrayForEach(document, documents) { char loc[72]; snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); - if (tokenize_late_text(out->model, j, document->valuestring, 0, - &out->documents[i]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); + if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != + 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { int n = out->documents[i].n_tokens; if (n > EMBED_API_MAX_ITEM_TOKENS) @@ -2271,10 +2325,8 @@ static void prepare_rerank_request(job *j, cJSON *root, http_server *s, } i++; } - if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - - out->document_tokens) { - ve_add(detail, "[\"body\",\"documents\"]", - "request exceeds 120000 token limit", + if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { + ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", "value_error.context_length"); } } @@ -2297,8 +2349,10 @@ typedef struct { static int rerank_result_cmp(const void *a, const void *b) { const rerank_result *ra = a; const rerank_result *rb = b; - if (ra->score > rb->score) return -1; - if (ra->score < rb->score) return 1; + if (ra->score > rb->score) + return -1; + if (ra->score < rb->score) + return 1; return ra->index - rb->index; } @@ -2317,26 +2371,24 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); float *all = xmalloc((size_t)max_doc_tokens * dim * sizeof(*all)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - int rc = embed_late_model_encode_tokens( - m->cpu_late_model, m->cpu_late_ws, r->query.ids, r->query.n_tokens, - 1, query); + int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, + r->query.n_tokens, 1, query); int pos = 0; offsets[0] = 0; for (int i = 0; rc == 0 && i < r->n_documents; i++) { late_tokens *doc = &r->documents[i]; - rc = embed_late_model_encode_tokens( - m->cpu_late_model, m->cpu_late_ws, doc->ids, doc->n_tokens, 1, all); + rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, doc->ids, + doc->n_tokens, 1, all); for (int k = 0; rc == 0 && k < doc->n_keep; k++) { - memcpy(docs + (size_t)pos * dim, - all + (size_t)doc->keep[k] * dim, + memcpy(docs + (size_t)pos * dim, all + (size_t)doc->keep[k] * dim, (size_t)dim * sizeof(*docs)); pos++; } offsets[i + 1] = pos; } if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, - r->n_documents, dim, scores); + rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, + scores); free(offsets); free(all); free(docs); @@ -2347,25 +2399,22 @@ static int execute_rerank_cpu(rerank_request *r, float *scores) { #ifdef USE_MLX static int execute_rerank_mlx(rerank_request *r, float *scores) { loaded_model *m = r->model; - embed_mlx_late_vectors_t *query = embed_mlx_late_encode_tokens_device( - m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); - embed_mlx_late_vectors_t **docs = - xcalloc((size_t)r->n_documents, sizeof(*docs)); - const embed_mlx_late_vectors_t **doc_refs = - xcalloc((size_t)r->n_documents, sizeof(*doc_refs)); + embed_mlx_late_vectors_t *query = + embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); + embed_mlx_late_vectors_t **docs = xcalloc((size_t)r->n_documents, sizeof(*docs)); + const embed_mlx_late_vectors_t **doc_refs = xcalloc((size_t)r->n_documents, sizeof(*doc_refs)); int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); int rc = query ? 0 : -1; offsets[0] = 0; for (int i = 0; rc == 0 && i < r->n_documents; i++) { late_tokens *doc = &r->documents[i]; - embed_mlx_late_vectors_t *all = embed_mlx_late_encode_tokens_device( - m->mlx_late_ctx, doc->ids, doc->n_tokens, 1); + embed_mlx_late_vectors_t *all = + embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, doc->ids, doc->n_tokens, 1); if (!all) { rc = -1; break; } - docs[i] = embed_mlx_late_vectors_select( - m->mlx_late_ctx, all, doc->keep, doc->n_keep); + docs[i] = embed_mlx_late_vectors_select(m->mlx_late_ctx, all, doc->keep, doc->n_keep); embed_mlx_late_vectors_free(all); if (!docs[i]) { rc = -1; @@ -2374,14 +2423,13 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { doc_refs[i] = docs[i]; offsets[i + 1] = offsets[i] + doc->n_keep; } - embed_mlx_late_vectors_t *packed = rc == 0 - ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, - r->n_documents) - : NULL; - if (!packed) rc = -1; + embed_mlx_late_vectors_t *packed = + rc == 0 ? embed_mlx_late_vectors_concat(m->mlx_late_ctx, doc_refs, r->n_documents) : NULL; + if (!packed) + rc = -1; if (rc == 0) - rc = embed_mlx_late_maxsim_batch_device( - m->mlx_late_ctx, query, packed, offsets, r->n_documents, scores); + rc = embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, + r->n_documents, scores); embed_mlx_late_vectors_free(packed); for (int i = 0; i < r->n_documents; i++) embed_mlx_late_vectors_free(docs[i]); @@ -2396,29 +2444,24 @@ static int execute_rerank_mlx(rerank_request *r, float *scores) { static void execute_rerank_request(rerank_request *r) { if (!r || !r->model) { if (r && r->j) - job_set_error(r->j, 500, "late-interaction model is unavailable", - "server_error"); + job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); return; } float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); uint64_t t0 = nstime(); #ifdef USE_MLX - int rc = r->model->mlx_late_ctx - ? execute_rerank_mlx(r, scores) - : execute_rerank_cpu(r, scores); + int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); #else int rc = execute_rerank_cpu(r, scores); #endif r->j->infer_ns += nstime() - t0; if (rc != 0) { free(scores); - job_set_error(r->j, 500, "late-interaction reranking failed", - "server_error"); + job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); return; } - rerank_result *ranked = - xmalloc((size_t)r->n_documents * sizeof(*ranked)); + rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); for (int i = 0; i < r->n_documents; i++) { ranked[i].index = i; ranked[i].score = scores[i]; @@ -2427,14 +2470,13 @@ static void execute_rerank_request(rerank_request *r) { t0 = nstime(); sbuf b = {0}; - sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", - r->model->info->id); - cJSON *documents = - cJSON_GetObjectItemCaseSensitive(r->root, "documents"); + sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); for (int i = 0; i < r->top_n; i++) { - if (i) sbuf_putc(&b, ','); - sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", - ranked[i].index, (double)ranked[i].score); + if (i) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, + (double)ranked[i].score); if (r->return_documents) { cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); sbuf_puts(&b, ",\"document\":"); @@ -2443,10 +2485,9 @@ static void execute_rerank_request(rerank_request *r) { sbuf_putc(&b, '}'); } sbuf_printf(&b, - "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," - "\"total_tokens\":%d}}", - r->query_tokens, r->document_tokens, - r->query_tokens + r->document_tokens); + "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," + "\"total_tokens\":%d}}", + r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); set_response_from_buf(r->j, &b); r->j->encode_ns += nstime() - t0; free(ranked); @@ -2469,7 +2510,8 @@ static int parse_job_root(job *j, cJSON **out_root) { static void process_unknown_job(job *j) { cJSON *root = NULL; - if (!j->started_ns) j->started_ns = nstime(); + if (!j->started_ns) + j->started_ns = nstime(); uint64_t t0 = nstime(); int rc = parse_job_root(j, &root); j->parse_ns += nstime() - t0; @@ -2482,8 +2524,7 @@ static void process_unknown_job(job *j) { static void process_job_group(http_server *s, job **jobs, int n_jobs) { embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); - rerank_request *rerank_reqs = - xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); + rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs)); int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); int *done = xcalloc((size_t)n_jobs, sizeof(*done)); embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); @@ -2491,7 +2532,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int i = 0; i < n_jobs; i++) { cJSON *root = NULL; - if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime(); + if (!jobs[i]->started_ns) + jobs[i]->started_ns = nstime(); if (!strcmp(jobs[i]->path, "/v1/embeddings")) { kind[i] = 1; } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { @@ -2526,8 +2568,7 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { continue; } - if ((kind[i] == 1 && !std_reqs[i].ready) || - (kind[i] == 2 && !ctx_reqs[i].ready) || + if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) || (kind[i] == 3 && !rerank_reqs[i].ready)) { job_set_timing_header(jobs[i]); enqueue_done(jobs[i]); @@ -2548,8 +2589,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 1 && std_reqs[k].ready && embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && - embedding_request_fits_group(&std_reqs[k], group_inputs, - group_tokens, s->batch_size, s->max_batch_tokens)) { + embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens, + s->batch_size, s->max_batch_tokens)) { std_group[group_n++] = &std_reqs[k]; group_inputs += std_reqs[k].n_inputs; group_tokens += std_reqs[k].total_tokens; @@ -2570,8 +2611,8 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && - contextual_request_fits_group(&ctx_reqs[k], group_docs, - group_tokens, s->batch_size, s->max_batch_tokens)) { + contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens, + s->batch_size, s->max_batch_tokens)) { ctx_group[group_n++] = &ctx_reqs[k]; group_docs += ctx_reqs[k].n_docs; group_tokens += ctx_reqs[k].total_tokens; @@ -2601,26 +2642,19 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { free(std_reqs); } -static int configure_loaded_model(loaded_model *m, - const embed_config_t *config, - int token_dim) -{ - if (!m || !m->info || !config || - config->hidden_size != m->info->dim || - token_dim != m->info->token_dim || - config->attention_mode != m->info->attention_mode || +static int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) { + if (!m || !m->info || !config || config->hidden_size != m->info->dim || + token_dim != m->info->token_dim || config->attention_mode != m->info->attention_mode || config->pooling_mode != m->info->pooling_mode || config->normalize_embeddings != m->info->normalize_embeddings) { if (m && m->info && config) - server_log("embed-server: model %s has incompatible config", - m->info->id); + server_log("embed-server: model %s has incompatible config", m->info->id); return -1; } m->append_terminal_token = config->append_terminal_token; /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ m->renormalize_truncated = - config->normalize_embeddings && - config->pooling_mode == EMBED_POOL_LAST_TOKEN; + config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN; return 0; } @@ -2631,28 +2665,27 @@ static void *worker_main(void *arg) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; - if (!m->info) continue; + if (!m->info) + continue; embed_mlx_options_t mlx_opts = { .quantize_bits = s->mlx_quantize_bits, .quantize_group_size = s->mlx_quantize_group_size, }; if (m->info->kind == MODEL_KIND_LATE) - m->mlx_late_ctx = - embed_mlx_late_load_with_options(m->path, &mlx_opts); + m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts); else m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts); if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) || (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load MLX model on worker: %s", - m->path); + server_log("embed-server: failed to load MLX model on worker: %s", m->path); rc = 1; break; } const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_config(m->mlx_late_ctx) - : embed_mlx_config(m->mlx_ctx); - int token_dim = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; + ? embed_mlx_late_config(m->mlx_late_ctx) + : embed_mlx_config(m->mlx_ctx); + int token_dim = + m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; if (configure_loaded_model(m, config, token_dim) != 0 || (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || @@ -2665,7 +2698,8 @@ static void *worker_main(void *arg) { pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; - if (rc) s->stopping = 1; + if (rc) + s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { @@ -2685,20 +2719,19 @@ static void *worker_main(void *arg) { } else #endif #ifdef USE_CUDA - if (s->use_cuda) { + if (s->use_cuda) { int rc = 0; for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; - if (!m->info) continue; + if (!m->info) + continue; m->cuda_ctx = embed_cuda_load(m->path); if (!m->cuda_ctx) { - server_log("embed-server: failed to load CUDA model on worker: %s", - m->path); + server_log("embed-server: failed to load CUDA model on worker: %s", m->path); rc = 1; break; } - if (configure_loaded_model( - m, embed_cuda_config(m->cuda_ctx), 0) != 0) { + if (configure_loaded_model(m, embed_cuda_config(m->cuda_ctx), 0) != 0) { rc = 1; break; } @@ -2706,7 +2739,8 @@ static void *worker_main(void *arg) { pthread_mutex_lock(&s->mu); s->worker_init_rc = rc; s->worker_ready = 1; - if (rc) s->stopping = 1; + if (rc) + s->stopping = 1; pthread_cond_broadcast(&s->cv); pthread_mutex_unlock(&s->mu); if (rc) { @@ -2730,9 +2764,9 @@ static void *worker_main(void *arg) { for (;;) { job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = collect_job_batch( - s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); - if (n_jobs == 0) break; + int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); + if (n_jobs == 0) + break; process_job_group(s, jobs, n_jobs); } #ifdef USE_MLX @@ -2792,8 +2826,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } m->tok_ws = qwen_tokenizer_workspace_new(); if (!m->tok_ws) { - server_log("embed-server: failed to allocate tokenizer workspace: %s", - path); + server_log("embed-server: failed to allocate tokenizer workspace: %s", path); return -1; } /* The contextual chunk separator is the tokenizer's <|endoftext|>. The @@ -2802,8 +2835,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { * token (e.g. small test models) takes precedence so the id stays * inside that model's embedding table. */ int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>"); - m->context_separator_id = sep_id >= 0 - ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> * suffix - the same token as the separator, not the model's chat * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ @@ -2812,16 +2844,13 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { int id = qwen_tokenizer_token_id(m->tok, "[MASK]"); m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; id = qwen_tokenizer_token_id(m->tok, "[Q]"); - m->late_query_prefix_id = - id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; + m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; id = qwen_tokenizer_token_id(m->tok, "[D]"); - m->late_document_prefix_id = - id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; + m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; - for (const char *p = punct; - *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / - sizeof(m->late_skip_ids[0])); + for (const char *p = punct; *p && m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / + sizeof(m->late_skip_ids[0])); p++) { char text[2] = {*p, '\0'}; int n_ids = 0; @@ -2854,8 +2883,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model); if (!m->cpu_late_ws) { - server_log("embed-server: failed to allocate late workspace: %s", - path); + server_log("embed-server: failed to allocate late workspace: %s", path); return -1; } config = embed_late_model_config(m->cpu_late_model); @@ -2877,8 +2905,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { return -1; } if (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || + (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size)) { server_log("embed-server: late special-token ids exceed model vocab"); return -1; @@ -2890,18 +2917,27 @@ static void free_models(http_server *s) { for (int i = 0; i < MODEL_COUNT; i++) { loaded_model *m = &s->models[i]; free(m->path); - if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); - if (m->tok) qwen_tokenizer_free(m->tok); - if (m->cpu_ws) embed_workspace_free(m->cpu_ws); - if (m->cpu_model) embed_model_free(m->cpu_model); - if (m->cpu_late_ws) embed_late_workspace_free(m->cpu_late_ws); - if (m->cpu_late_model) embed_late_model_free(m->cpu_late_model); + if (m->tok_ws) + qwen_tokenizer_workspace_free(m->tok_ws); + if (m->tok) + qwen_tokenizer_free(m->tok); + if (m->cpu_ws) + embed_workspace_free(m->cpu_ws); + if (m->cpu_model) + embed_model_free(m->cpu_model); + if (m->cpu_late_ws) + embed_late_workspace_free(m->cpu_late_ws); + if (m->cpu_late_model) + embed_late_model_free(m->cpu_late_model); #ifdef USE_MLX - if (m->mlx_ctx) embed_mlx_free(m->mlx_ctx); - if (m->mlx_late_ctx) embed_mlx_late_free(m->mlx_late_ctx); + if (m->mlx_ctx) + embed_mlx_free(m->mlx_ctx); + if (m->mlx_late_ctx) + embed_mlx_late_free(m->mlx_late_ctx); #endif #ifdef USE_CUDA - if (m->cuda_ctx) embed_cuda_free(m->cuda_ctx); + if (m->cuda_ctx) + embed_cuda_free(m->cuda_ctx); #endif } } @@ -2910,14 +2946,12 @@ static uint64_t physical_memory_nbytes(void) { #ifdef __APPLE__ uint64_t bytes = 0; size_t len = sizeof(bytes); - if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && - len == sizeof(bytes) && bytes > 0) + if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) return bytes; #endif long pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); - if (pages <= 0 || page_size <= 0 || - (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) + if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) return 0; return (uint64_t)pages * (uint64_t)page_size; } @@ -2928,13 +2962,11 @@ static int validate_model_specs(const embed_server_config_t *cfg) { const embed_server_model_spec_t *spec = &cfg->models[i]; model_slot slot = model_slot_for_id(spec->id); if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "embed-server: unknown model id: %s\n", - spec->id ? spec->id : ""); + fprintf(stderr, "embed-server: unknown model id: %s\n", spec->id ? spec->id : ""); return -1; } if (!spec->path || !spec->path[0]) { - fprintf(stderr, "embed-server: model %s has an empty path\n", - spec->id); + fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); return -1; } if (seen[slot]) { @@ -2942,8 +2974,7 @@ static int validate_model_specs(const embed_server_config_t *cfg) { return -1; } if (cfg->use_cuda && k_models[slot].kind == MODEL_KIND_LATE) { - fprintf(stderr, - "embed-server: model %s has no CUDA late-interaction backend\n", + fprintf(stderr, "embed-server: model %s has no CUDA late-interaction backend\n", spec->id); return -1; } @@ -2953,17 +2984,16 @@ static int validate_model_specs(const embed_server_config_t *cfg) { } static int mlx_memory_preflight(const embed_server_config_t *cfg) { - if (!cfg->use_mlx) return 0; + if (!cfg->use_mlx) + return 0; uint64_t payload = 0; for (int i = 0; i < cfg->n_models; i++) { - multi_safetensors_t *ms = - multi_safetensors_open(cfg->models[i].path); + multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path); size_t model_payload = 0; if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { multi_safetensors_close(ms); - fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", - cfg->models[i].path); + fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path); return -1; } multi_safetensors_close(ms); @@ -2985,22 +3015,21 @@ static int mlx_memory_preflight(const embed_server_config_t *cfg) { "skipping MLX memory preflight"); return 0; } - double utilization = cfg->memory_utilization > 0.0 - ? cfg->memory_utilization - : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; + double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization + : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; uint64_t budget = (uint64_t)((double)physical * utilization); const double gib = 1024.0 * 1024.0 * 1024.0; server_log("embed-server: MLX memory preflight: %.1f GiB tensors, " "%.1f GiB estimated resident, %.1f GiB budget " "(%.2f of physical memory)", - (double)payload / gib, (double)estimated / gib, - (double)budget / gib, utilization); + (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); if (cfg->mlx_quantize_bits) { server_log("embed-server: MLX %d-bit quantization enabled; preflight " "uses source payload as a conservative peak estimate", cfg->mlx_quantize_bits); } - if (estimated <= budget) return 0; + if (estimated <= budget) + return 0; server_log("embed-server: refusing MLX model set above the host-memory " "budget"); server_log("embed-server: use BF16 artifacts, load fewer models, or raise " @@ -3020,24 +3049,23 @@ int embed_run_server(const embed_server_config_t *cfg) { memset(&s, 0, sizeof(s)); s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; s.port = cfg->port > 0 ? cfg->port : 8000; - s.batch_size = cfg->batch_size > 0 - ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; - s.max_batch_tokens = cfg->max_batch_tokens > 0 - ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; - s.batch_wait_us = cfg->batch_wait_us >= 0 - ? cfg->batch_wait_us - : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US - : EMBED_SERVER_BATCH_WAIT_US); + s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = + cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us + : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US + : EMBED_SERVER_BATCH_WAIT_US); s.use_mlx = cfg->use_mlx; s.use_cuda = cfg->use_cuda; s.mlx_quantize_bits = cfg->mlx_quantize_bits; - s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 - ? cfg->mlx_quantize_group_size : 64; + s.mlx_quantize_group_size = + cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); else { const char *env_key = getenv("EMBED_API_KEY"); - if (env_key && env_key[0]) s.api_key = xstrdup(env_key); + if (env_key && env_key[0]) + s.api_key = xstrdup(env_key); } pthread_mutex_init(&s.mu, NULL); @@ -3124,7 +3152,8 @@ int embed_run_server(const embed_server_config_t *cfg) { aeMain(s.loop); server_log("embed-server: shutdown requested"); - if (s.listen_fd >= 0) close(s.listen_fd); + if (s.listen_fd >= 0) + close(s.listen_fd); client *c = s.clients; while (c) { client *next = c->next; @@ -3155,59 +3184,58 @@ int embed_run_server(const embed_server_config_t *cfg) { * ==================================================================== */ #ifndef EMBED_SERVER_TEST -static void print_usage(const char *prog) -{ +static void print_usage(const char *prog) { fprintf(stderr, - "Usage: %s --model ID=DIR [options]\n" - "\n" - "Options:\n" - " --model ID=DIR Model to serve (repeatable)\n" + "Usage: %s --model ID=DIR [options]\n" + "\n" + "Options:\n" + " --model ID=DIR Model to serve (repeatable)\n" #ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" + " --mlx Use Apple MLX GPU backend\n" #endif #ifdef USE_CUDA - " --cuda Use NVIDIA CUDA GPU backend\n" + " --cuda Use NVIDIA CUDA GPU backend\n" #endif - " --host HOST Bind host (default: 127.0.0.1)\n" - " --port N Bind port (default: 8000)\n" - " --api-key K Require Authorization: Bearer K\n" - " -b, --batch-size N Max texts or documents per inference batch\n" - " (default: 32)\n" - " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N First-arrival micro-batch deadline in us\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 32)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N First-arrival micro-batch deadline in us\n" #ifdef USE_CUDA - " (default: CUDA 1000; CPU 0)\n" + " (default: CUDA 1000; CPU 0)\n" #else - " (default: 0)\n" + " (default: 0)\n" #endif #ifdef USE_MLX - " --memory-utilization F Fraction of physical memory the MLX model-set\n" - " preflight may plan for (default: 0.90; values\n" - " above 1.0 overcommit)\n" - " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" - " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" + " --memory-utilization F Fraction of physical memory the MLX model-set\n" + " preflight may plan for (default: 0.90; values\n" + " above 1.0 overcommit)\n" + " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" + " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" #endif #ifdef USE_CUDA - " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" - " (default: f32, exact)\n" - " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" - " f32); bf16 halves weight memory\n" + " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n" #endif - " -t, --threads N CPU threads (default: all cores)\n" - " -v, --verbose Verbose (-vv for debug)\n" - " -V, --version Print version and exit\n" - " --build-info Print build details and exit\n" - " -h, --help Show this help\n" - "\n" - "Examples:\n" - " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" + " -t, --threads N CPU threads (default: all cores)\n" + " -v, --verbose Verbose (-vv for debug)\n" + " -V, --version Print version and exit\n" + " --build-info Print build details and exit\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" #ifdef USE_MLX - " %s --mlx --mlx-quant-bits 8 \\\n" - " --model pplx-embed-v1-4b=./model-4b-bf16\n", - prog, prog, prog); + " %s --mlx --mlx-quant-bits 8 \\\n" + " --model pplx-embed-v1-4b=./model-4b-bf16\n", + prog, prog, prog); #else - , - prog, prog); + , + prog, prog); #endif } @@ -3217,8 +3245,7 @@ typedef struct { int cap; } model_specs_t; -static int append_model_spec(model_specs_t *specs, const char *arg) -{ +static int append_model_spec(model_specs_t *specs, const char *arg) { const char *eq = strchr(arg, '='); if (!eq || eq == arg || !eq[1]) { fprintf(stderr, "--model expects MODEL_ID=DIR\n"); @@ -3227,7 +3254,8 @@ static int append_model_spec(model_specs_t *specs, const char *arg) if (specs->n == specs->cap) { int new_cap = specs->cap ? specs->cap * 2 : 4; void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) return -1; + if (!p) + return -1; specs->v = p; specs->cap = new_cap; } @@ -3246,9 +3274,9 @@ static int append_model_spec(model_specs_t *specs, const char *arg) return 0; } -static void free_model_specs(model_specs_t *specs) -{ - if (!specs) return; +static void free_model_specs(model_specs_t *specs) { + if (!specs) + return; for (int i = 0; i < specs->n; i++) { free((char *)specs->v[i].id); free((char *)specs->v[i].path); @@ -3256,8 +3284,7 @@ static void free_model_specs(model_specs_t *specs) free(specs->v); } -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]) { int n_threads = 0; int verbose = 0; int batch_size = 0; @@ -3287,37 +3314,34 @@ int main(int argc, char *argv[]) embed_print_version(prog); free_model_specs(&model_specs); return 0; - } - else if (!strcmp(f, "--build-info")) { + } else if (!strcmp(f, "--build-info")) { embed_print_build_info(prog); free_model_specs(&model_specs); return 0; - } - else if (!strcmp(f, "--model")) { + } else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg]) != 0) { free_model_specs(&model_specs); return 1; } } #ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { use_mlx = 1; } - else if (!strcmp(f, "--mlx-quant-bits")) { + else if (!strcmp(f, "--mlx")) { + use_mlx = 1; + } else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { + } else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg]); if (mlx_quantize_group_size <= 0) { fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--memory-utilization")) { + } else if (!strcmp(f, "--memory-utilization")) { memory_utilization = atof(argv[++arg]); if (memory_utilization <= 0.0) { fprintf(stderr, "--memory-utilization must be > 0\n"); @@ -3327,47 +3351,47 @@ int main(int argc, char *argv[]) } #endif #ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { use_cuda = 1; } - else if (!strcmp(f, "--cuda-gemm-mode")) { + else if (!strcmp(f, "--cuda")) { + use_cuda = 1; + } else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { + } else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg]; } #endif - else if (!strcmp(f, "--host")) { host = argv[++arg]; } - else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } - else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } - else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + else if (!strcmp(f, "--host")) { + host = argv[++arg]; + } else if (!strcmp(f, "--port")) { + port = atoi(argv[++arg]); + } else if (!strcmp(f, "--api-key")) { + api_key = argv[++arg]; + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg]); - } - else if (!strcmp(f, "--max-batch-tokens")) { + } else if (!strcmp(f, "--max-batch-tokens")) { max_batch_tokens = atoi(argv[++arg]); if (max_batch_tokens <= 0) { fprintf(stderr, "--max-batch-tokens must be > 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "--batch-wait-us")) { + } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { fprintf(stderr, "--batch-wait-us must be >= 0\n"); free_model_specs(&model_specs); return 1; } - } - else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { n_threads = atoi(argv[++arg]); - } - else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } - else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(prog); free_model_specs(&model_specs); return 0; - } - else { + } else { fprintf(stderr, "unknown option: %s\n", f); print_usage(prog); free_model_specs(&model_specs); @@ -3431,18 +3455,22 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (use_mlx) { - if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + if (verbose >= 1) + fprintf(stderr, "Using MLX GPU backend\n"); } else #endif #ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); + if (use_cuda) { + if (verbose >= 1) + fprintf(stderr, "Using CUDA GPU backend\n"); } else #endif { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + if (n_threads <= 0) + n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + if (verbose >= 1) + fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } embed_server_config_t scfg = { diff --git a/embed_server.h b/embed_server.h index c10dddb..2a39b94 100644 --- a/embed_server.h +++ b/embed_server.h @@ -31,7 +31,7 @@ typedef struct { int use_cuda; int mlx_quantize_bits; int mlx_quantize_group_size; - double memory_utilization; /* fraction of physical memory the MLX + double memory_utilization; /* fraction of physical memory the MLX preflight may plan for; 0 = default 0.90, values above 1.0 allow overcommit */ const char *api_key; diff --git a/qwen_kernels.c b/qwen_kernels.c index dcd724c..c7f5762 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -5,7 +5,8 @@ #include #include #include -#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) #include #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) @@ -92,8 +93,10 @@ static void *worker_loop(void *arg) { } void qwen_set_threads(int n) { - if (n < 1) n = 1; - if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS; + if (n < 1) + n = 1; + if (n > QWEN_MAX_THREADS) + n = QWEN_MAX_THREADS; #if defined(USE_OPENBLAS) /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the @@ -119,8 +122,7 @@ void qwen_set_threads(int n) { if (n <= 1) { if (qwen_verbose >= 2) { #if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", - n, blas_threads); + fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); #else fprintf(stderr, "Thread pool: %d threads\n", n); #endif @@ -135,8 +137,7 @@ void qwen_set_threads(int n) { if (qwen_verbose >= 2) { #if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", - n, blas_threads); + fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); #else fprintf(stderr, "Thread pool: %d threads\n", n); #endif @@ -183,7 +184,8 @@ static void parallel_for(parallel_fn_t fn, void *arg) { * ======================================================================== */ void qwen_add_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) a[i] += b[i]; + for (int i = 0; i < n; i++) + a[i] += b[i]; } /* ======================================================================== @@ -192,8 +194,7 @@ void qwen_add_inplace(float *a, const float *b, int n) { void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { #ifdef USE_BLAS - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); #else for (int m = 0; m < M; m++) { for (int n = 0; n < N; n++) { @@ -207,18 +208,20 @@ void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N #endif } -void qwen_linear(float *y, const float *x, const float *W, const float *b, - int seq_len, int in_dim, int out_dim) { +void qwen_linear(float *y, + const float *x, + const float *W, + const float *b, + int seq_len, + int in_dim, + int out_dim) { #ifdef USE_BLAS if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, - out_dim, in_dim, 1.0f, W, in_dim, x, 1, - 0.0f, y, 1); + cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, + 1); } else { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - seq_len, out_dim, in_dim, - 1.0f, x, in_dim, W, in_dim, - 0.0f, y, out_dim); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, + in_dim, W, in_dim, 0.0f, y, out_dim); } if (b != NULL) { for (int s = 0; s < seq_len; s++) { @@ -243,8 +246,8 @@ void qwen_linear(float *y, const float *x, const float *W, const float *b, #endif } -void qwen_linear_nobias(float *y, const float *x, const float *W, - int seq_len, int in_dim, int out_dim) { +void qwen_linear_nobias( + float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim); } @@ -273,8 +276,8 @@ void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias * Processes 2 output rows at a time to amortize x vector loads. */ -static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +static void bf16_matvec_fused( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); } @@ -293,22 +296,22 @@ static void matvec_worker(int tid, int n_threads, void *arg) { int chunk = (t->out_dim + n_threads - 1) / n_threads; int start = tid * chunk; int end = start + chunk; - if (end > t->out_dim) end = t->out_dim; - if (start >= end) return; + if (end > t->out_dim) + end = t->out_dim; + if (start >= end) + return; - bf16_matvec_fused(t->y + start, t->x, - t->W_bf16 + (size_t)start * t->in_dim, - t->bias ? t->bias + start : NULL, - t->in_dim, end - start); + bf16_matvec_fused(t->y + start, t->x, t->W_bf16 + (size_t)start * t->in_dim, + t->bias ? t->bias + start : NULL, t->in_dim, end - start); } -static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +static void bf16_matvec_threaded( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { if (tp.n_threads <= 1) { bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); return; } - matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim }; + matvec_task_t task = {y, x, W_bf16, bias, in_dim, out_dim}; parallel_for(matvec_worker, &task); } @@ -327,18 +330,23 @@ static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { int chunk = (t->seq_len + n_threads - 1) / n_threads; int start = tid * chunk; int end = start + chunk; - if (end > t->seq_len) end = t->seq_len; - if (start >= end) return; + if (end > t->seq_len) + end = t->seq_len; + if (start >= end) + return; for (int s = start; s < end; s++) { - bf16_matvec_fused(t->y + (size_t)s * t->out_dim, - t->x + (size_t)s * t->in_dim, - t->W_bf16, t->bias, t->in_dim, t->out_dim); + bf16_matvec_fused(t->y + (size_t)s * t->out_dim, t->x + (size_t)s * t->in_dim, t->W_bf16, + t->bias, t->in_dim, t->out_dim); } } -static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int seq_len, int in_dim, +static void bf16_linear_rows(float *y, + const float *x, + const uint16_t *W_bf16, + const float *bias, + int seq_len, + int in_dim, int out_dim) { bf16_linear_rows_task_t task = { .y = y, @@ -369,8 +377,7 @@ typedef struct { int total_dim; } pair_matvec_task_t; -static void pair_matvec_range(const pair_matvec_task_t *t, int row, - int start, int end) { +static void pair_matvec_range(const pair_matvec_task_t *t, int row, int start, int end) { const float *x_row = t->x + (size_t)row * t->in_dim; int a_end = t->a_dim; int b_end = a_end + t->b_dim; @@ -380,8 +387,7 @@ static void pair_matvec_range(const pair_matvec_task_t *t, int row, int e = end < a_end ? end : a_end; if (s < e) { bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, - t->Wa_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); + t->Wa_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } @@ -391,8 +397,7 @@ static void pair_matvec_range(const pair_matvec_task_t *t, int row, int e = e_abs - a_end; if (s < e) { bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, - t->Wb_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); + t->Wb_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } } @@ -403,8 +408,10 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) { int chunk = (total_work + n_threads - 1) / n_threads; int start = tid * chunk; int end = start + chunk; - if (end > total_work) end = total_work; - if (start >= end) return; + if (end > total_work) + end = total_work; + if (start >= end) + return; while (start < end) { int row = start / t->total_dim; @@ -416,20 +423,23 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) { } } -void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, +void qwen_linear_nobias_bf16_pair(float *a, + float *b, + const float *x, const uint16_t *Wa_bf16, const uint16_t *Wb_bf16, - int seq_len, int in_dim, int a_dim, + int seq_len, + int in_dim, + int a_dim, int b_dim) { - if (seq_len <= 0) return; + if (seq_len <= 0) + return; if (tp.n_threads <= 1) { for (int s = 0; s < seq_len; s++) { const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(a + (size_t)s * a_dim, x_row, - Wa_bf16, NULL, in_dim, a_dim); - bf16_matvec_fused(b + (size_t)s * b_dim, x_row, - Wb_bf16, NULL, in_dim, b_dim); + bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim); + bf16_matvec_fused(b + (size_t)s * b_dim, x_row, Wb_bf16, NULL, in_dim, b_dim); } return; } @@ -464,8 +474,7 @@ typedef struct { int total_dim; } qkv_matvec_task_t; -static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, - int start, int end) { +static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int start, int end) { const float *x_row = t->x + (size_t)row * t->in_dim; int q_end = t->q_dim; int k_end = q_end + t->kv_dim; @@ -476,8 +485,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int e = end < q_end ? end : q_end; if (s < e) { bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, - t->Wq_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); + t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } @@ -487,8 +495,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int e = e_abs - q_end; if (s < e) { bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, - t->Wk_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); + t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } @@ -498,8 +505,7 @@ static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int e = e_abs - k_end; if (s < e) { bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, - t->Wv_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); + t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); } } } @@ -510,8 +516,10 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { int chunk = (total_work + n_threads - 1) / n_threads; int start = tid * chunk; int end = start + chunk; - if (end > total_work) end = total_work; - if (start >= end) return; + if (end > total_work) + end = total_work; + if (start >= end) + return; while (start < end) { int row = start / t->total_dim; @@ -523,23 +531,26 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { } } -void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, +void qwen_linear_nobias_bf16_qkv(float *q, + float *k, + float *v, + const float *x, const uint16_t *Wq_bf16, const uint16_t *Wk_bf16, const uint16_t *Wv_bf16, - int seq_len, int in_dim, int q_dim, + int seq_len, + int in_dim, + int q_dim, int kv_dim) { - if (seq_len <= 0) return; + if (seq_len <= 0) + return; if (tp.n_threads <= 1) { for (int s = 0; s < seq_len; s++) { const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(q + (size_t)s * q_dim, x_row, - Wq_bf16, NULL, in_dim, q_dim); - bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, - Wk_bf16, NULL, in_dim, kv_dim); - bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, - Wv_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim); + bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, Wk_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, Wv_bf16, NULL, in_dim, kv_dim); } return; } @@ -561,8 +572,8 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, parallel_for(qkv_matvec_worker, &task); } -void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, - int seq_len, int in_dim, int out_dim) { +void qwen_linear_nobias_bf16( + float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) { if (seq_len == 1) { bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; @@ -573,7 +584,6 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } - /* ======================================================================== * Normalization * ======================================================================== */ @@ -582,9 +592,8 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, * for the dispatch. */ #define QWEN_RMS_NORM_PARALLEL_ELEMS 262144 -static void qwen_rms_norm_range(float *out, const float *x, - const float *weight, int start, int end, - int hidden, float eps) { +static void qwen_rms_norm_range( + float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { for (int s = start; s < end; s++) { const float *x_row = x + (size_t)s * hidden; float *out_row = out + (size_t)s * hidden; @@ -597,7 +606,8 @@ static void qwen_rms_norm_range(float *out, const float *x, accv = _mm512_fmadd_ps(v, v, accv); } float sum_sq = _mm512_reduce_add_ps(accv); - for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; + for (; i < hidden; i++) + sum_sq += x_row[i] * x_row[i]; #elif defined(__AVX2__) && defined(__FMA__) __m256 accv = _mm256_setzero_ps(); int i = 0; @@ -609,7 +619,8 @@ static void qwen_rms_norm_range(float *out, const float *x, acc128 = _mm_hadd_ps(acc128, acc128); acc128 = _mm_hadd_ps(acc128, acc128); float sum_sq = _mm_cvtss_f32(acc128); - for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; + for (; i < hidden; i++) + sum_sq += x_row[i] * x_row[i]; #else float sum_sq = 0.0f; for (int i = 0; i < hidden; i++) { @@ -626,7 +637,8 @@ static void qwen_rms_norm_range(float *out, const float *x, __m512 vw = _mm512_loadu_ps(weight + j); _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); } - for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; + for (; j < hidden; j++) + out_row[j] = x_row[j] * rms_inv * weight[j]; #elif defined(__AVX2__) __m256 scale = _mm256_set1_ps(rms_inv); int j = 0; @@ -635,7 +647,8 @@ static void qwen_rms_norm_range(float *out, const float *x, __m256 vw = _mm256_loadu_ps(weight + j); _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); } - for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; + for (; j < hidden; j++) + out_row[j] = x_row[j] * rms_inv * weight[j]; #else for (int i = 0; i < hidden; i++) { out_row[i] = x_row[i] * rms_inv * weight[i]; @@ -658,15 +671,17 @@ static void rms_norm_worker(int tid, int n_threads, void *arg) { int chunk = (t->seq_len + n_threads - 1) / n_threads; int start = tid * chunk; int end = start + chunk; - if (end > t->seq_len) end = t->seq_len; - if (start >= end) return; - qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, - t->eps); + if (end > t->seq_len) + end = t->seq_len; + if (start >= end) + return; + qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); } -void qwen_rms_norm(float *out, const float *x, const float *weight, - int seq_len, int hidden, float eps) { - if (seq_len <= 0 || hidden <= 0) return; +void qwen_rms_norm( + float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { + if (seq_len <= 0 || hidden <= 0) + return; long long elems = (long long)seq_len * hidden; if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) { @@ -684,8 +699,8 @@ void qwen_rms_norm(float *out, const float *x, const float *weight, } } -void qwen_rms_norm_per_head(float *x, const float *weight, - int seq_len, int n_heads, int head_dim, float eps) { +void qwen_rms_norm_per_head( + float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ int hidden = n_heads * head_dim; for (int s = 0; s < seq_len; s++) { @@ -700,7 +715,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight, accv = _mm512_fmadd_ps(v, v, accv); } float sum_sq = _mm512_reduce_add_ps(accv); - for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; + for (; d < head_dim; d++) + sum_sq += vec[d] * vec[d]; #elif defined(__AVX2__) && defined(__FMA__) __m256 accv = _mm256_setzero_ps(); int d = 0; @@ -708,11 +724,13 @@ void qwen_rms_norm_per_head(float *x, const float *weight, __m256 v = _mm256_loadu_ps(vec + d); accv = _mm256_fmadd_ps(v, v, accv); } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + __m128 acc128 = + _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); acc128 = _mm_hadd_ps(acc128, acc128); acc128 = _mm_hadd_ps(acc128, acc128); float sum_sq = _mm_cvtss_f32(acc128); - for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; + for (; d < head_dim; d++) + sum_sq += vec[d] * vec[d]; #else float sum_sq = 0.0f; for (int d = 0; d < head_dim; d++) { @@ -729,7 +747,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight, __m512 w = _mm512_loadu_ps(weight + j); _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); } - for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; + for (; j < head_dim; j++) + vec[j] = vec[j] * rms_inv * weight[j]; #elif defined(__AVX2__) __m256 scale = _mm256_set1_ps(rms_inv); int j = 0; @@ -738,7 +757,8 @@ void qwen_rms_norm_per_head(float *x, const float *weight, __m256 w = _mm256_loadu_ps(weight + j); _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); } - for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; + for (; j < head_dim; j++) + vec[j] = vec[j] * rms_inv * weight[j]; #else for (int d = 0; d < head_dim; d++) { vec[d] = vec[d] * rms_inv * weight[d]; @@ -759,7 +779,8 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n) { for (int base = 0; base < n; base += CHUNK) { int len = n - base; - if (len > CHUNK) len = CHUNK; + if (len > CHUNK) + len = CHUNK; for (int i = 0; i < len; i++) exp_neg[i] = -gate[base + i]; @@ -783,7 +804,8 @@ void qwen_softmax(float *x, int rows, int cols) { float *row = x + r * cols; float max_val = row[0]; for (int c = 1; c < cols; c++) { - if (row[c] > max_val) max_val = row[c]; + if (row[c] > max_val) + max_val = row[c]; } float sum = 0.0f; for (int c = 0; c < cols; c++) { @@ -801,9 +823,7 @@ void qwen_softmax(float *x, int rows, int cols) { * Attention Operations * ======================================================================== */ -float qwen_dot_f32(const float *a, const float *b, int n) { - return qwen_dot_f32_impl(a, b, n); -} +float qwen_dot_f32(const float *a, const float *b, int n) { return qwen_dot_f32_impl(a, b, n); } static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) { return qwen_dot_f32_impl(a, b, n); @@ -824,13 +844,22 @@ static inline void qwen_vec_scale_add(float *dst, const float *src, float correc qwen_vec_scale_add_impl(dst, src, correction, n); } -#define QWEN_PACKED_ATTN_QUERY_TILE 32 +#define QWEN_PACKED_ATTN_QUERY_TILE 32 #define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128 -static void qwen_bidirectional_gqa_attention_packed_online_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int end, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end) { +static void qwen_bidirectional_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { int heads_per_kv = n_heads / n_kv_heads; int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; @@ -842,7 +871,8 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows( float max_score = -1e30f; float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; for (int j = start; j < end; j++) { const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; @@ -870,10 +900,20 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows( } #ifdef USE_BLAS -static void qwen_bidirectional_gqa_attention_packed_blas_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int end, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end, float *scores) { +static void qwen_bidirectional_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { int heads_per_kv = n_heads / n_kv_heads; int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; @@ -886,13 +926,11 @@ static void qwen_bidirectional_gqa_attention_packed_blas_rows( const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; float *o = out + (size_t)query_start * q_hidden + h * head_dim; - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, - 0.0f, scores, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, + k, kv_hidden, 0.0f, scores, keys); qwen_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, - 0.0f, o, q_hidden); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); } #endif @@ -914,9 +952,8 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; int task_id = 0; #ifdef USE_BLAS - float *scores = t->scores - ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq - : NULL; + float *scores = + t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { @@ -925,41 +962,43 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { int end = t->offsets[b + 1]; for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; - if (q1 > end) q1 = end; - if (task_id++ % n_threads != tid) continue; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; #ifdef USE_BLAS if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { qwen_bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, - scores); + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, + t->head_dim, t->scale, h, q0, q1, scores); continue; } #endif qwen_bidirectional_gqa_attention_packed_online_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); } } } } -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( - const int *offsets, int batch) { +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { #ifdef USE_BLAS - if (!offsets || batch <= 0) return 0; + if (!offsets || batch <= 0) + return 0; int max_seq = 0; for (int b = 0; b < batch; b++) { int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) max_seq = len; + if (len > max_seq) + max_seq = len; } - if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0; - if ((size_t)max_seq > SIZE_MAX / (sizeof(float) * - QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) + if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) + return 0; + if ((size_t)max_seq > + SIZE_MAX / (sizeof(float) * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) return 0; - return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * - (size_t)max_seq * sizeof(float); + return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); #else (void)offsets; (void)batch; @@ -967,33 +1006,48 @@ size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( #endif } -void qwen_bidirectional_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes) { +void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { long long qk_work = 0; int max_seq = 0; for (int b = 0; b < batch; b++) { int len = offsets[b + 1] - offsets[b]; qk_work += (long long)len * len * n_heads; - if (len > max_seq) max_seq = len; + if (len > max_seq) + max_seq = len; } float *scores = NULL; #ifdef USE_BLAS - size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( - offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; #else (void)scratch; (void)scratch_bytes; #endif - packed_gqa_attn_task_t task = { - .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, - .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, - .head_dim = head_dim, .scale = scale, .scores = scores, - .max_seq = max_seq - }; + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) { parallel_for(packed_gqa_attn_worker, &task); } else { @@ -1001,24 +1055,35 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch( } } -void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale) { - size_t scratch_bytes = - qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); +void qwen_bidirectional_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; qwen_bidirectional_gqa_attention_packed_with_scratch( - out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, - scores, scratch_bytes); + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); free(scores); } -static void qwen_causal_gqa_attention_packed_online_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end) { +static void qwen_causal_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { int heads_per_kv = n_heads / n_kv_heads; int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; @@ -1030,7 +1095,8 @@ static void qwen_causal_gqa_attention_packed_online_rows( float max_score = -1e30f; float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; for (int j = start; j <= i; j++) { const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; @@ -1055,10 +1121,19 @@ static void qwen_causal_gqa_attention_packed_online_rows( } #ifdef USE_BLAS -static void qwen_causal_gqa_attention_packed_blas_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end, float *scores) { +static void qwen_causal_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { int heads_per_kv = n_heads / n_kv_heads; int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; @@ -1071,18 +1146,17 @@ static void qwen_causal_gqa_attention_packed_blas_rows( const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; float *o = out + (size_t)query_start * q_hidden + h * head_dim; - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, - 0.0f, scores, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, + k, kv_hidden, 0.0f, scores, keys); for (int r = 0; r < rows; r++) { int allowed = query_start + r - start + 1; float *row = scores + (size_t)r * keys; - for (int j = allowed; j < keys; j++) row[j] = -INFINITY; + for (int j = allowed; j < keys; j++) + row[j] = -INFINITY; } qwen_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, - 0.0f, o, q_hidden); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); } #endif @@ -1090,9 +1164,8 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; int task_id = 0; #ifdef USE_BLAS - float *scores = t->scores - ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq - : NULL; + float *scores = + t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { @@ -1101,69 +1174,88 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { int end = t->offsets[b + 1]; for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; - if (q1 > end) q1 = end; - if (task_id++ % n_threads != tid) continue; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; #ifdef USE_BLAS if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { qwen_causal_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, - scores); + t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1, scores); continue; } #endif - qwen_causal_gqa_attention_packed_online_rows( - t->out, t->Q, t->K, t->V, start, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); + qwen_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, + t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); } } } } -void qwen_causal_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes) { +void qwen_causal_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { long long qk_work = 0; int max_seq = 0; for (int b = 0; b < batch; b++) { int len = offsets[b + 1] - offsets[b]; qk_work += (long long)len * (len + 1) / 2 * n_heads; - if (len > max_seq) max_seq = len; + if (len > max_seq) + max_seq = len; } float *scores = NULL; #ifdef USE_BLAS - size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( - offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; #else (void)scratch; (void)scratch_bytes; #endif - packed_gqa_attn_task_t task = { - .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, - .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, - .head_dim = head_dim, .scale = scale, .scores = scores, - .max_seq = max_seq - }; + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) parallel_for(packed_causal_gqa_attn_worker, &task); else packed_causal_gqa_attn_worker(0, 1, &task); } -void qwen_causal_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale) { - size_t scratch_bytes = - qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); +void qwen_causal_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - qwen_causal_gqa_attention_packed_with_scratch( - out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, - scores, scratch_bytes); + qwen_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, scores, scratch_bytes); free(scores); } @@ -1171,8 +1263,8 @@ void qwen_causal_gqa_attention_packed(float *out, const float *Q, * Position Embeddings * ======================================================================== */ -void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, - int seq, int head_dim, float theta) { +void qwen_compute_rope_neox( + float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { int half = head_dim / 2; for (int s = 0; s < seq; s++) { @@ -1191,8 +1283,8 @@ void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions } } -void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, - int seq, int n_heads, int head_dim) { +void qwen_apply_rope_neox( + float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { /* * NeoX split-half style: * x1 = x[..., :half], x2 = x[..., half:] @@ -1225,7 +1317,7 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #elif defined(__AVX2__) && defined(__FMA__) @@ -1243,7 +1335,7 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #elif defined(__ARM_NEON) || defined(__ARM_NEON__) @@ -1261,14 +1353,14 @@ void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #else for (int d = 0; d < half; d++) { - float x1 = vec[d]; /* first half */ - float x2 = vec[half + d]; /* second half */ - vec[d] = x1 * c[d] + (-x2) * sn[d]; + float x1 = vec[d]; /* first half */ + float x2 = vec[half + d]; /* second half */ + vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #endif diff --git a/qwen_kernels.h b/qwen_kernels.h index c611194..3a6a4d6 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -35,34 +35,42 @@ void qwen_add_inplace(float *a, const float *b, int n); void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); /* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ -void qwen_linear(float *y, const float *x, const float *W, const float *b, - int seq_len, int in_dim, int out_dim); +void qwen_linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void qwen_linear_nobias(float *y, const float *x, const float *W, - int seq_len, int in_dim, int out_dim); +void qwen_linear_nobias( + float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); /* bf16 weight variants */ void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); -void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, - int seq_len, int in_dim, int out_dim); +void qwen_linear_nobias_bf16( + float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); /* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ -void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, +void qwen_linear_nobias_bf16_pair(float *a, + float *b, + const float *x, const uint16_t *Wa_bf16, const uint16_t *Wb_bf16, - int seq_len, int in_dim, int a_dim, + int seq_len, + int in_dim, + int a_dim, int b_dim); /* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ -void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, +void qwen_linear_nobias_bf16_qkv(float *q, + float *k, + float *v, + const float *x, const uint16_t *Wq_bf16, const uint16_t *Wk_bf16, const uint16_t *Wv_bf16, - int seq_len, int in_dim, int q_dim, + int seq_len, + int in_dim, + int q_dim, int kv_dim); - /* Dot product using the best available local SIMD implementation. */ float qwen_dot_f32(const float *a, const float *b, int n); @@ -71,14 +79,14 @@ float qwen_dot_f32(const float *a, const float *b, int n); * ======================================================================== */ /* RMS Normalization: out = x / rms(x) * weight */ -void qwen_rms_norm(float *out, const float *x, const float *weight, - int seq_len, int hidden, float eps); +void qwen_rms_norm( + float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); /* Per-head RMS Normalization for Q/K norms in decoder * x: [seq, n_heads, head_dim], weight: [head_dim] * Normalizes each head independently */ -void qwen_rms_norm_per_head(float *x, const float *weight, - int seq_len, int n_heads, int head_dim, float eps); +void qwen_rms_norm_per_head( + float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); /* ======================================================================== * Activation Functions @@ -102,33 +110,58 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n); * Attention is block-diagonal: tokens only attend within their own * offsets[b]..offsets[b+1] span. */ -void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale); +void qwen_bidirectional_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale); /* Optional reusable scratch for the BLAS-tiled packed attention path. */ -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( - const int *offsets, int batch); -void qwen_bidirectional_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes); +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); +void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); /* * Causal GQA attention over a packed/ragged batch. Query position i attends * only to positions at or before i within its sequence. */ -void qwen_causal_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale); -void qwen_causal_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes); +void qwen_causal_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale); +void qwen_causal_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); /* ======================================================================== * Position Embeddings @@ -139,16 +172,16 @@ void qwen_causal_gqa_attention_packed_with_scratch( * cos_out, sin_out: [seq, head_dim] * cos[d] and cos[half+d] are the same (duplicated for full head_dim). */ -void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, - int seq, int head_dim, float theta); +void qwen_compute_rope_neox( + float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta); /* * Apply NeoX-style RoPE to Q or K. * x: [seq, n_heads * head_dim] (in-place) * cos_vals, sin_vals: [seq, head_dim] */ -void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, - int seq, int n_heads, int head_dim); +void qwen_apply_rope_neox( + float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim); /* ======================================================================== * Threading diff --git a/qwen_kernels_avx.c b/qwen_kernels_avx.c index 0fc1df8..42c358b 100644 --- a/qwen_kernels_avx.c +++ b/qwen_kernels_avx.c @@ -22,8 +22,8 @@ #if defined(__AVX512F__) && defined(__AVX512BW__) -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +void qwen_bf16_matvec_fused_avx( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; /* Process 4 output rows at a time */ @@ -46,26 +46,34 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 /* Row 0 */ __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); - a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + a0 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); /* Row 1 */ __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); - a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + a2 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); /* Row 2 */ __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k)); __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16)); - a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4); - a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5); + a4 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4); + a5 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5); /* Row 3 */ __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k)); __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16)); - a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6); - a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7); + a6 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6); + a7 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7); } for (; k + 16 <= in_dim; k += 16) { @@ -74,26 +82,39 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k)); __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); - a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4); - a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6); + a0 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + a4 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4); + a6 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6); } - float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); - float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); - float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f); - float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f); + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f); + float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o + 2] : 0.0f); + float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o + 3] : 0.0f); for (; k < in_dim; k++) { float xk = x[k]; - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; - float v0,v1,v2,v3; - memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); - s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; + uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16; + uint32_t b2 = ((uint32_t)w2[k]) << 16, b3 = ((uint32_t)w3[k]) << 16; + float v0, v1, v2, v3; + memcpy(&v0, &b0, 4); + memcpy(&v1, &b1, 4); + memcpy(&v2, &b2, 4); + memcpy(&v3, &b3, 4); + s0 += v0 * xk; + s1 += v1 * xk; + s2 += v2 * xk; + s3 += v3 * xk; } - y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3; + y[o] = s0; + y[o + 1] = s1; + y[o + 2] = s2; + y[o + 3] = s3; } /* Handle 2-row remainder */ @@ -111,28 +132,38 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); - a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); - a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + a0 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + a2 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); } for (; k + 16 <= in_dim; k += 16) { __m512 xv = _mm512_loadu_ps(x + k); __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + a0 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); } - float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); - float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f); for (; k < in_dim; k++) { - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); - s0+=v0*x[k]; s1+=v1*x[k]; + uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16; + float v0, v1; + memcpy(&v0, &b0, 4); + memcpy(&v1, &b1, 4); + s0 += v0 * x[k]; + s1 += v1 * x[k]; } - y[o]=s0; y[o+1]=s1; + y[o] = s0; + y[o + 1] = s1; } /* Single remaining row */ @@ -142,13 +173,16 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 int k = 0; for (; k + 16 <= in_dim; k += 16) { __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); - acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), - _mm512_loadu_ps(x + k), acc); + acc = _mm512_fmadd_ps( + _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), + _mm512_loadu_ps(x + k), acc); } float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f); for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum += wv*x[k]; + uint32_t bits = ((uint32_t)w[k]) << 16; + float wv; + memcpy(&wv, &bits, 4); + sum += wv * x[k]; } y[o] = sum; } @@ -167,15 +201,15 @@ static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); } -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +void qwen_bf16_matvec_fused_avx( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; for (; o + 1 < out_dim; o += 2) { const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; const uint16_t *w1 = w0 + in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); - __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps(); + __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(); + __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps(); int k = 0; for (; k + 16 <= in_dim; k += 16) { __m256 xlo = _mm256_loadu_ps(x + k); @@ -188,36 +222,45 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 a2 = _mm256_fmadd_ps(wlo, xlo, a2); a3 = _mm256_fmadd_ps(whi, xhi, a3); } - a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3); + a0 = _mm256_add_ps(a0, a1); + a2 = _mm256_add_ps(a2, a3); __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); - r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0); + r0 = _mm_hadd_ps(r0, r1); + r0 = _mm_hadd_ps(r0, r0); float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f); - float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f); + float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0, r0, 1)) + (bias ? bias[o + 1] : 0.0f); for (; k < in_dim; k++) { - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); - s0+=v0*x[k]; s1+=v1*x[k]; + uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16; + float v0, v1; + memcpy(&v0, &b0, 4); + memcpy(&v1, &b1, 4); + s0 += v0 * x[k]; + s1 += v1 * x[k]; } - y[o]=s0; y[o+1]=s1; + y[o] = s0; + y[o + 1] = s1; } for (; o < out_dim; o++) { const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(); int k = 0; for (; k + 16 <= in_dim; k += 16) { __m256 wlo, whi; bf16x16_to_f32(w + k, &wlo, &whi); - a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); - a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); + a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x + k), a0); + a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x + k + 8), a1); } a0 = _mm256_add_ps(a0, a1); __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); - r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f); for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum+=wv*x[k]; + uint32_t bits = ((uint32_t)w[k]) << 16; + float wv; + memcpy(&wv, &bits, 4); + sum += wv * x[k]; } y[o] = sum; } @@ -238,7 +281,7 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) { __m512 acc2 = _mm512_setzero_ps(); __m512 acc3 = _mm512_setzero_ps(); for (; i + 64 <= n; i += 64) { - acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); @@ -248,7 +291,8 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) { acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); } float sum = _mm512_reduce_add_ps(acc); - for (; i < n; i++) sum += a[i] * b[i]; + for (; i < n; i++) + sum += a[i] * b[i]; return sum; #else int i = 0; @@ -257,20 +301,21 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) { __m256 acc2 = _mm256_setzero_ps(); __m256 acc3 = _mm256_setzero_ps(); for (; i + 32 <= n; i += 32) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); - acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1); - acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2); - acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3); + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); } acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); for (; i + 8 <= n; i += 8) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); } __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); float sum = _mm_cvtss_f32(r); - for (; i < n; i++) sum += a[i] * b[i]; + for (; i < n; i++) + sum += a[i] * b[i]; return sum; #endif } @@ -280,7 +325,7 @@ void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { int i = 0; __m512 s = _mm512_set1_ps(scale); for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); @@ -288,20 +333,22 @@ void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { for (; i + 16 <= n; i += 16) { _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); } - for (; i < n; i++) dst[i] *= scale; + for (; i < n; i++) + dst[i] *= scale; #else int i = 0; __m256 s = _mm256_set1_ps(scale); for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); - _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s)); - _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s)); - _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s)); + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); + _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); + _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); } for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); } - for (; i < n; i++) dst[i] *= scale; + for (; i < n; i++) + dst[i] *= scale; #endif } @@ -312,31 +359,38 @@ void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) for (; i + 64 <= n; i += 64) { _mm512_storeu_ps(dst + i, _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - _mm512_storeu_ps(dst + i + 16, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16))); - _mm512_storeu_ps(dst + i + 32, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32))); - _mm512_storeu_ps(dst + i + 48, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, + _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, + _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, + _mm512_loadu_ps(dst + i + 48))); } for (; i + 16 <= n; i += 16) { _mm512_storeu_ps(dst + i, _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); } - for (; i < n; i++) dst[i] += alpha * src[i]; + for (; i < n; i++) + dst[i] += alpha * src[i]; #else int i = 0; __m256 a = _mm256_set1_ps(alpha); for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); - _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8))); - _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16))); - _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24))); + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, + _mm256_loadu_ps(dst + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, + _mm256_loadu_ps(dst + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, + _mm256_loadu_ps(dst + i + 24))); } for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); } - for (; i < n; i++) dst[i] += alpha * src[i]; + for (; i < n; i++) + dst[i] += alpha * src[i]; #endif } @@ -347,31 +401,38 @@ void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int for (; i + 64 <= n; i += 64) { _mm512_storeu_ps(dst + i, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - _mm512_storeu_ps(dst + i + 16, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16))); - _mm512_storeu_ps(dst + i + 32, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32))); - _mm512_storeu_ps(dst + i + 48, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, + _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, + _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, + _mm512_loadu_ps(src + i + 48))); } for (; i + 16 <= n; i += 16) { _mm512_storeu_ps(dst + i, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; #else int i = 0; __m256 c = _mm256_set1_ps(correction); for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); - _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8))); - _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16))); - _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24))); + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, + _mm256_loadu_ps(src + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, + _mm256_loadu_ps(src + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, + _mm256_loadu_ps(src + i + 24))); } for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; #endif } diff --git a/qwen_kernels_generic.c b/qwen_kernels_generic.c index fd07455..149dfe0 100644 --- a/qwen_kernels_generic.c +++ b/qwen_kernels_generic.c @@ -6,8 +6,8 @@ #include -void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +void qwen_bf16_matvec_fused_generic( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { for (int o = 0; o < out_dim; o++) { const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; float sum = bias ? bias[o] : 0.0f; @@ -23,18 +23,22 @@ void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_ float qwen_dot_f32_generic(const float *a, const float *b, int n) { float sum = 0.0f; - for (int i = 0; i < n; i++) sum += a[i] * b[i]; + for (int i = 0; i < n; i++) + sum += a[i] * b[i]; return sum; } void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) { - for (int i = 0; i < n; i++) dst[i] *= scale; + for (int i = 0; i < n; i++) + dst[i] *= scale; } void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { - for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; + for (int i = 0; i < n; i++) + dst[i] += alpha * src[i]; } void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { - for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; + for (int i = 0; i < n; i++) + dst[i] = dst[i] * correction + src[i]; } diff --git a/qwen_kernels_impl.h b/qwen_kernels_impl.h index 43e4fbe..217f1e8 100644 --- a/qwen_kernels_impl.h +++ b/qwen_kernels_impl.h @@ -7,47 +7,47 @@ #include -void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); +void qwen_bf16_matvec_fused_generic( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); float qwen_dot_f32_generic(const float *a, const float *b, int n); void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n); #ifdef __ARM_NEON -void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); +void qwen_bf16_matvec_fused_neon( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); float qwen_dot_f32_neon(const float *a, const float *b, int n); void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon -#define qwen_dot_f32_impl qwen_dot_f32_neon +#define qwen_dot_f32_impl qwen_dot_f32_neon #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon -#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon +#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon #elif defined(__AVX2__) && defined(__FMA__) -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); +void qwen_bf16_matvec_fused_avx( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); float qwen_dot_f32_avx(const float *a, const float *b, int n); void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx -#define qwen_dot_f32_impl qwen_dot_f32_avx +#define qwen_dot_f32_impl qwen_dot_f32_avx #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx -#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx +#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx #else #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic -#define qwen_dot_f32_impl qwen_dot_f32_generic +#define qwen_dot_f32_impl qwen_dot_f32_generic #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic -#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic +#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic #endif #endif /* QWEN_KERNELS_IMPL_H */ diff --git a/qwen_kernels_neon.c b/qwen_kernels_neon.c index d656f1d..5d51fc6 100644 --- a/qwen_kernels_neon.c +++ b/qwen_kernels_neon.c @@ -9,8 +9,8 @@ #include #include -void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { +void qwen_bf16_matvec_fused_neon( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */ @@ -128,7 +128,8 @@ float qwen_dot_f32_neon(const float *a, const float *b, int n) { acc1 = vfmaq_f32(acc1, a1, b1); } float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); - for (; i < n; i++) sum += a[i] * b[i]; + for (; i < n; i++) + sum += a[i] * b[i]; return sum; } @@ -141,7 +142,8 @@ void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); } - for (; i < n; i++) dst[i] *= scale; + for (; i < n; i++) + dst[i] *= scale; } void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { @@ -155,7 +157,8 @@ void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); } - for (; i < n; i++) dst[i] += alpha * src[i]; + for (; i < n; i++) + dst[i] += alpha * src[i]; } void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { @@ -169,7 +172,8 @@ void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; } #endif /* __ARM_NEON */ diff --git a/qwen_safetensors.c b/qwen_safetensors.c index 030798e..e1c0e87 100644 --- a/qwen_safetensors.c +++ b/qwen_safetensors.c @@ -13,29 +13,37 @@ * ======================================================================== */ static void skip_whitespace(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') + (*p)++; } static int parse_string(const char **p, char *out, size_t max_len) { skip_whitespace(p); - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; size_t i = 0; while (**p && **p != '"' && i < max_len - 1) { if (**p == '\\') { (*p)++; - if (**p == 'n') out[i++] = '\n'; - else if (**p == 't') out[i++] = '\t'; - else if (**p == '"') out[i++] = '"'; - else if (**p == '\\') out[i++] = '\\'; - else out[i++] = **p; + if (**p == 'n') + out[i++] = '\n'; + else if (**p == 't') + out[i++] = '\t'; + else if (**p == '"') + out[i++] = '"'; + else if (**p == '\\') + out[i++] = '\\'; + else + out[i++] = **p; } else { out[i++] = **p; } (*p)++; } out[i] = '\0'; - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; return 0; } @@ -44,8 +52,12 @@ static int parse_int(const char **p, int64_t *out) { skip_whitespace(p); int64_t val = 0; int neg = 0; - if (**p == '-') { neg = 1; (*p)++; } - if (**p < '0' || **p > '9') return -1; + if (**p == '-') { + neg = 1; + (*p)++; + } + if (**p < '0' || **p > '9') + return -1; while (**p >= '0' && **p <= '9') { val = val * 10 + (**p - '0'); (*p)++; @@ -55,18 +67,25 @@ static int parse_int(const char **p, int64_t *out) { } static safetensor_dtype_t parse_dtype(const char *s) { - if (strcmp(s, "F32") == 0) return DTYPE_F32; - if (strcmp(s, "F16") == 0) return DTYPE_F16; - if (strcmp(s, "BF16") == 0) return DTYPE_BF16; - if (strcmp(s, "I32") == 0) return DTYPE_I32; - if (strcmp(s, "I64") == 0) return DTYPE_I64; - if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL; + if (strcmp(s, "F32") == 0) + return DTYPE_F32; + if (strcmp(s, "F16") == 0) + return DTYPE_F16; + if (strcmp(s, "BF16") == 0) + return DTYPE_BF16; + if (strcmp(s, "I32") == 0) + return DTYPE_I32; + if (strcmp(s, "I64") == 0) + return DTYPE_I64; + if (strcmp(s, "BOOL") == 0) + return DTYPE_BOOL; return DTYPE_UNKNOWN; } static int parse_tensor_entry(const char **p, safetensor_t *t) { skip_whitespace(p); - if (**p != '{') return -1; + if (**p != '{') + return -1; (*p)++; t->dtype = DTYPE_UNKNOWN; @@ -76,79 +95,109 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) { while (**p && **p != '}') { skip_whitespace(p); - if (**p == ',') { (*p)++; continue; } + if (**p == ',') { + (*p)++; + continue; + } char key[64]; - if (parse_string(p, key, sizeof(key)) != 0) return -1; + if (parse_string(p, key, sizeof(key)) != 0) + return -1; skip_whitespace(p); - if (**p != ':') return -1; + if (**p != ':') + return -1; (*p)++; skip_whitespace(p); if (strcmp(key, "dtype") == 0) { char dtype_str[32]; - if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1; + if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) + return -1; t->dtype = parse_dtype(dtype_str); } else if (strcmp(key, "shape") == 0) { - if (**p != '[') return -1; + if (**p != '[') + return -1; (*p)++; t->ndim = 0; while (**p && **p != ']') { skip_whitespace(p); - if (**p == ',') { (*p)++; continue; } - if (t->ndim >= 8) return -1; - if (parse_int(p, &t->shape[t->ndim]) != 0) return -1; + if (**p == ',') { + (*p)++; + continue; + } + if (t->ndim >= 8) + return -1; + if (parse_int(p, &t->shape[t->ndim]) != 0) + return -1; t->ndim++; } - if (**p != ']') return -1; + if (**p != ']') + return -1; (*p)++; } else if (strcmp(key, "data_offsets") == 0) { - if (**p != '[') return -1; + if (**p != '[') + return -1; (*p)++; skip_whitespace(p); int64_t start_i; - if (parse_int(p, &start_i) != 0) return -1; + if (parse_int(p, &start_i) != 0) + return -1; skip_whitespace(p); - if (**p != ',') return -1; + if (**p != ',') + return -1; (*p)++; skip_whitespace(p); int64_t end_i; - if (parse_int(p, &end_i) != 0) return -1; - if (start_i < 0 || end_i < start_i) return -1; + if (parse_int(p, &end_i) != 0) + return -1; + if (start_i < 0 || end_i < start_i) + return -1; t->data_offset = (size_t)start_i; t->data_size = (size_t)(end_i - start_i); skip_whitespace(p); - if (**p != ']') return -1; + if (**p != ']') + return -1; (*p)++; } else { /* Skip unknown value */ if (**p == '"') { (*p)++; while (**p && **p != '"') { - if (**p == '\\') (*p)++; - if (**p) (*p)++; + if (**p == '\\') + (*p)++; + if (**p) + (*p)++; } - if (**p == '"') (*p)++; + if (**p == '"') + (*p)++; } else if (**p == '[') { - int depth = 1; (*p)++; + int depth = 1; + (*p)++; while (**p && depth > 0) { - if (**p == '[') depth++; - else if (**p == ']') depth--; + if (**p == '[') + depth++; + else if (**p == ']') + depth--; (*p)++; } } else if (**p == '{') { - int depth = 1; (*p)++; + int depth = 1; + (*p)++; while (**p && depth > 0) { - if (**p == '{') depth++; - else if (**p == '}') depth--; + if (**p == '{') + depth++; + else if (**p == '}') + depth--; (*p)++; } } else { - while (**p && **p != ',' && **p != '}') (*p)++; + while (**p && **p != ',' && **p != '}') + (*p)++; } } } - if (**p != '}') return -1; + if (**p != '}') + return -1; (*p)++; return 0; } @@ -156,29 +205,39 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) { static int parse_header(safetensors_file_t *sf) { const char *p = sf->header_json; skip_whitespace(&p); - if (*p != '{') return -1; + if (*p != '{') + return -1; p++; sf->num_tensors = 0; while (*p && *p != '}') { skip_whitespace(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; char name[256]; - if (parse_string(&p, name, sizeof(name)) != 0) return -1; + if (parse_string(&p, name, sizeof(name)) != 0) + return -1; skip_whitespace(&p); - if (*p != ':') return -1; + if (*p != ':') + return -1; p++; if (strcmp(name, "__metadata__") == 0) { skip_whitespace(&p); if (*p == '{') { - int depth = 1; p++; + int depth = 1; + p++; while (*p && depth > 0) { - if (*p == '{') depth++; - else if (*p == '}') depth--; + if (*p == '{') + depth++; + else if (*p == '}') + depth--; p++; } } @@ -193,10 +252,12 @@ static int parse_header(safetensors_file_t *sf) { safetensor_t *t = &sf->tensors[sf->num_tensors]; snprintf(t->name, sizeof(t->name), "%s", name); - if (parse_tensor_entry(&p, t) != 0) return -1; + if (parse_tensor_entry(&p, t) != 0) + return -1; sf->num_tensors++; } - if (*p != '}') return -1; + if (*p != '}') + return -1; return 0; } @@ -206,24 +267,38 @@ static int parse_header(safetensors_file_t *sf) { safetensors_file_t *safetensors_open(const char *path) { int fd = open(path, O_RDONLY); - if (fd < 0) return NULL; + if (fd < 0) + return NULL; struct stat st; - if (fstat(fd, &st) < 0) { close(fd); return NULL; } + if (fstat(fd, &st) < 0) { + close(fd); + return NULL; + } size_t file_size = (size_t)st.st_size; - if (file_size < 8) { close(fd); return NULL; } + if (file_size < 8) { + close(fd); + return NULL; + } void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (data == MAP_FAILED) return NULL; + if (data == MAP_FAILED) + return NULL; uint64_t header_size = 0; memcpy(&header_size, data, 8); - if (header_size > file_size - 8) { munmap(data, file_size); return NULL; } + if (header_size > file_size - 8) { + munmap(data, file_size); + return NULL; + } safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); - if (!sf) { munmap(data, file_size); return NULL; } + if (!sf) { + munmap(data, file_size); + return NULL; + } sf->path = strdup(path); sf->data = data; @@ -231,18 +306,26 @@ safetensors_file_t *safetensors_open(const char *path) { sf->header_size = (size_t)header_size; sf->header_json = malloc(header_size + 1); - if (!sf->header_json) { safetensors_close(sf); return NULL; } + if (!sf->header_json) { + safetensors_close(sf); + return NULL; + } memcpy(sf->header_json, (char *)data + 8, header_size); sf->header_json[header_size] = '\0'; - if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } + if (parse_header(sf) != 0) { + safetensors_close(sf); + return NULL; + } return sf; } void safetensors_close(safetensors_file_t *sf) { - if (!sf) return; - if (sf->data) munmap(sf->data, sf->file_size); + if (!sf) + return; + if (sf->data) + munmap(sf->data, sf->file_size); free(sf->path); free(sf->header_json); free(sf); @@ -254,7 +337,8 @@ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t int64_t safetensor_numel(const safetensor_t *t) { int64_t n = 1; - for (int i = 0; i < t->ndim; i++) n *= t->shape[i]; + for (int i = 0; i < t->ndim; i++) + n *= t->shape[i]; return n; } @@ -267,25 +351,28 @@ static float bf16_to_f32(uint16_t bf16) { float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { int64_t n = safetensor_numel(t); - if (n <= 0) return NULL; + if (n <= 0) + return NULL; float *out = malloc(n * sizeof(float)); - if (!out) return NULL; + if (!out) + return NULL; const void *data = safetensors_data(sf, t); switch (t->dtype) { - case DTYPE_F32: - memcpy(out, data, n * sizeof(float)); - break; - case DTYPE_BF16: { - const uint16_t *src = (const uint16_t *)data; - for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]); - break; - } - default: - free(out); - return NULL; + case DTYPE_F32: + memcpy(out, data, n * sizeof(float)); + break; + case DTYPE_BF16: { + const uint16_t *src = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) + out[i] = bf16_to_f32(src[i]); + break; + } + default: + free(out); + return NULL; } return out; } @@ -313,7 +400,8 @@ static int parse_shard_name(const char *name, int *idx, int *total) { multi_safetensors_t *multi_safetensors_open(const char *model_dir) { multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); - if (!ms) return NULL; + if (!ms) + return NULL; char path[4096]; @@ -328,7 +416,10 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) { /* Scan directory for shard files */ DIR *dir = opendir(model_dir); - if (!dir) { free(ms); return NULL; } + if (!dir) { + free(ms); + return NULL; + } struct dirent *entry; char shard_names[SAFETENSORS_MAX_SHARDS][256]; @@ -341,15 +432,14 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) { strstr(entry->d_name, ".safetensors") != NULL) { int shard_idx = 0, shard_total = 0; if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { - fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", - entry->d_name); + fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); closedir(dir); free(ms); return NULL; } if (shard_total > SAFETENSORS_MAX_SHARDS) { - fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", - shard_total, SAFETENSORS_MAX_SHARDS); + fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, + SAFETENSORS_MAX_SHARDS); closedir(dir); free(ms); return NULL; @@ -364,16 +454,14 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) { return NULL; } if (seen[shard_idx - 1]) { - fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", - shard_idx); + fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); closedir(dir); free(ms); return NULL; } seen[shard_idx - 1] = 1; - snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), - "%s", entry->d_name); + snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); n_shards++; } } @@ -392,8 +480,8 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) { } for (int i = 0; i < expected_total; i++) { if (!seen[i]) { - fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", - i + 1, expected_total, model_dir); + fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, + expected_total, model_dir); free(ms); return NULL; } @@ -417,22 +505,24 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) { } void multi_safetensors_close(multi_safetensors_t *ms) { - if (!ms) return; + if (!ms) + return; for (int i = 0; i < ms->num_shards; i++) { safetensors_close(ms->shards[i]); } free(ms); } -int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, - size_t *out_nbytes) { - if (!ms || !out_nbytes) return -1; +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes) { + if (!ms || !out_nbytes) + return -1; size_t total = 0; for (int s = 0; s < ms->num_shards; s++) { const safetensors_file_t *sf = ms->shards[s]; for (int i = 0; i < sf->num_tensors; i++) { size_t n = sf->tensors[i].data_size; - if (n > SIZE_MAX - total) return -1; + if (n > SIZE_MAX - total) + return -1; total += n; } } @@ -441,13 +531,14 @@ int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, } const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf) { + const char *name, + safetensors_file_t **out_sf) { for (int s = 0; s < ms->num_shards; s++) { safetensors_file_t *sf = ms->shards[s]; for (int i = 0; i < sf->num_tensors; i++) { if (strcmp(sf->tensors[i].name, name) == 0) { - if (out_sf) *out_sf = sf; + if (out_sf) + *out_sf = sf; return &sf->tensors[i]; } } diff --git a/qwen_safetensors.h b/qwen_safetensors.h index ba61217..5e839a9 100644 --- a/qwen_safetensors.h +++ b/qwen_safetensors.h @@ -10,7 +10,7 @@ #include #define SAFETENSORS_MAX_TENSORS 1024 -#define SAFETENSORS_MAX_SHARDS 8 +#define SAFETENSORS_MAX_SHARDS 8 typedef enum { DTYPE_F32 = 0, @@ -54,13 +54,12 @@ void safetensors_close(safetensors_file_t *sf); /* Open model from directory (auto-detects single file or multi-shard) */ multi_safetensors_t *multi_safetensors_open(const char *model_dir); void multi_safetensors_close(multi_safetensors_t *ms); -int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, - size_t *out_nbytes); +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes); /* Find a tensor by name across all shards */ const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf); + const char *name, + safetensors_file_t **out_sf); /* Get raw pointer to tensor data (within mmap'd region) */ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c index 59f8784..e813071 100644 --- a/qwen_tokenizer.c +++ b/qwen_tokenizer.c @@ -15,24 +15,31 @@ static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ static int gpt2_mapping_initialized = 0; static void init_gpt2_mapping(void) { - if (gpt2_mapping_initialized) return; + if (gpt2_mapping_initialized) + return; memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); int n = 0; for (int b = 0; b < 256; b++) { int is_normal = 0; - if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */ - if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */ - if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */ + if (b >= 33 && b <= 126) + is_normal = 1; /* '!'..'~' */ + if (b >= 161 && b <= 172) + is_normal = 1; /* '¡'..'¬' */ + if (b >= 174 && b <= 255) + is_normal = 1; /* '®'..'ÿ' */ - if (is_normal) gpt2_byte_to_unicode[b] = b; - else gpt2_byte_to_unicode[b] = 256 + n++; + if (is_normal) + gpt2_byte_to_unicode[b] = b; + else + gpt2_byte_to_unicode[b] = 256 + n++; } for (int b = 0; b < 256; b++) { int cp = gpt2_byte_to_unicode[b]; - if (cp < 512) gpt2_unicode_to_byte[cp] = b; + if (cp < 512) + gpt2_unicode_to_byte[cp] = b; } gpt2_mapping_initialized = 1; @@ -64,7 +71,8 @@ static char *decode_gpt2_token(const char *token_str) { size_t len = strlen(token_str); unsigned char *bytes = (unsigned char *)malloc(len + 1); - if (!bytes) return NULL; + if (!bytes) + return NULL; int byte_count = 0; const unsigned char *p = (const unsigned char *)token_str; @@ -79,18 +87,24 @@ static char *decode_gpt2_token(const char *token_str) { nbytes = 1; } else if ((*p & 0xE0) == 0xC0) { cp = (*p & 0x1F) << 6; - if (p + 1 < end) cp |= (p[1] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F); nbytes = 2; } else if ((*p & 0xF0) == 0xE0) { cp = (*p & 0x0F) << 12; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 6; - if (p + 2 < end) cp |= (p[2] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F) << 6; + if (p + 2 < end) + cp |= (p[2] & 0x3F); nbytes = 3; } else if ((*p & 0xF8) == 0xF0) { cp = (*p & 0x07) << 18; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 12; - if (p + 2 < end) cp |= (p[2] & 0x3F) << 6; - if (p + 3 < end) cp |= (p[3] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F) << 12; + if (p + 2 < end) + cp |= (p[2] & 0x3F) << 6; + if (p + 3 < end) + cp |= (p[3] & 0x3F); nbytes = 4; } else { cp = *p; @@ -113,31 +127,41 @@ static char *decode_gpt2_token(const char *token_str) { * ======================================================================== */ static void skip_ws(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') + (*p)++; } static int parse_json_string(const char **p, char *out, size_t max_len) { skip_ws(p); - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; size_t i = 0; while (**p && **p != '"' && i < max_len - 1) { if (**p == '\\') { (*p)++; - if (**p == 'n') out[i++] = '\n'; - else if (**p == 't') out[i++] = '\t'; - else if (**p == '"') out[i++] = '"'; - else if (**p == '\\') out[i++] = '\\'; - else if (**p == '/') out[i++] = '/'; + if (**p == 'n') + out[i++] = '\n'; + else if (**p == 't') + out[i++] = '\t'; + else if (**p == '"') + out[i++] = '"'; + else if (**p == '\\') + out[i++] = '\\'; + else if (**p == '/') + out[i++] = '/'; else if (**p == 'u') { (*p)++; unsigned int cp = 0; for (int j = 0; j < 4 && **p; j++, (*p)++) { cp <<= 4; char c = **p; - if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0'); - else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10); - else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10); + if (c >= '0' && c <= '9') + cp |= (unsigned)(c - '0'); + else if (c >= 'a' && c <= 'f') + cp |= (unsigned)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') + cp |= (unsigned)(c - 'A' + 10); } if (cp < 0x80 && i + 1 < max_len) { out[i++] = (char)cp; @@ -159,7 +183,8 @@ static int parse_json_string(const char **p, char *out, size_t max_len) { (*p)++; } out[i] = '\0'; - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; return 0; } @@ -167,7 +192,10 @@ static int parse_json_string(const char **p, char *out, size_t max_len) { static int64_t parse_json_int(const char **p) { skip_ws(p); int neg = 0; - if (**p == '-') { neg = 1; (*p)++; } + if (**p == '-') { + neg = 1; + (*p)++; + } int64_t val = 0; while (**p >= '0' && **p <= '9') { val = val * 10 + (**p - '0'); @@ -211,23 +239,28 @@ static uint64_t fnv1a_hash_pair(const char *a, const char *b) { static int pair_key_eq(const char *key, const char *a, const char *b) { while (*a) { - if (*key++ != *a++) return 0; + if (*key++ != *a++) + return 0; } - if (*key++ != ' ') return 0; + if (*key++ != ' ') + return 0; while (*b) { - if (*key++ != *b++) return 0; + if (*key++ != *b++) + return 0; } return *key == '\0'; } static int next_pow2(int x) { int p = 1; - while (p < x) p <<= 1; + while (p < x) + p <<= 1; return p; } static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { @@ -246,9 +279,11 @@ static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { } static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) { - if (!keyp) return -1; + if (!keyp) + return -1; char *key = *keyp; - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { @@ -268,26 +303,31 @@ static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int valu } static int map_get(const str_int_entry_t *map, int cap, const char *key) { - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (strcmp(map[idx].key, key) == 0) return map[idx].value; + if (!map[idx].key) + return -1; + if (strcmp(map[idx].key, key) == 0) + return map[idx].value; } return -1; } -static int map_get_pair(const str_int_entry_t *map, int cap, - const char *a, const char *b) { - if (!map || cap <= 0 || !a || !b) return -1; +static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, const char *b) { + if (!map || cap <= 0 || !a || !b) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask); for (int i = 0; i < cap; i++) { int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (pair_key_eq(map[idx].key, a, b)) return map[idx].value; + if (!map[idx].key) + return -1; + if (pair_key_eq(map[idx].key, a, b)) + return map[idx].value; } return -1; } @@ -297,35 +337,43 @@ static int map_get_pair(const str_int_entry_t *map, int cap, * ======================================================================== */ static int utf8_char_len(unsigned char c) { - if ((c & 0x80) == 0) return 1; - if ((c & 0xE0) == 0xC0) return 2; - if ((c & 0xF0) == 0xE0) return 3; - if ((c & 0xF8) == 0xF0) return 4; + if ((c & 0x80) == 0) + return 1; + if ((c & 0xE0) == 0xC0) + return 2; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xF8) == 0xF0) + return 4; return 1; } static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { *out_syms = NULL; *out_n = 0; - if (!s || !*s) return 0; + if (!s || !*s) + return 0; int n = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { + for (const unsigned char *p = (const unsigned char *)s; *p;) { int l = utf8_char_len(*p); p += l; n++; } - if (n <= 0) return 0; + if (n <= 0) + return 0; char **syms = (char **)calloc((size_t)n, sizeof(char *)); - if (!syms) return -1; + if (!syms) + return -1; int i = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { + for (const unsigned char *p = (const unsigned char *)s; *p;) { int l = utf8_char_len(*p); syms[i] = (char *)malloc((size_t)l + 1); if (!syms[i]) { - for (int j = 0; j < i; j++) free(syms[j]); + for (int j = 0; j < i; j++) + free(syms[j]); free(syms); return -1; } @@ -343,7 +391,8 @@ static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { static char *str_concat2(const char *a, const char *b) { size_t la = strlen(a), lb = strlen(b); char *r = (char *)malloc(la + lb + 1); - if (!r) return NULL; + if (!r) + return NULL; memcpy(r, a, la); memcpy(r + la, b, lb); r[la + lb] = '\0'; @@ -355,21 +404,23 @@ static char *str_concat2(const char *a, const char *b) { * ======================================================================== */ typedef struct { - uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ + uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ int rank; int merged_id; } pair_merge_entry_t; static uint64_t pair_hash(uint64_t k) { /* splitmix64 finalizer */ - k ^= k >> 30; k *= 0xbf58476d1ce4e5b9ULL; - k ^= k >> 27; k *= 0x94d049bb133111ebULL; + k ^= k >> 30; + k *= 0xbf58476d1ce4e5b9ULL; + k ^= k >> 27; + k *= 0x94d049bb133111ebULL; k ^= k >> 31; return k; } -static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, - int rank, int merged_id) { +static void +pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) { int mask = cap - 1; int idx = (int)(pair_hash(key) & (uint64_t)mask); for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) { @@ -379,22 +430,21 @@ static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, map[idx].merged_id = merged_id; return; } - if (map[idx].key == key) return; /* first (lowest) rank wins */ + if (map[idx].key == key) + return; /* first (lowest) rank wins */ } } /* Returns the rank and sets *merged_id, or INT_MAX when the pair never * merges. */ -static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, - int *merged_id) { - const pair_merge_entry_t *map = - (const pair_merge_entry_t *)tok->int_merges; +static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, int *merged_id) { + const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges; uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; int mask = tok->int_merges_cap - 1; int idx = (int)(pair_hash(key) & (uint64_t)mask); - for (int probe = 0; probe < tok->int_merges_cap; - probe++, idx = (idx + 1) & mask) { - if (map[idx].key == UINT64_MAX) return INT_MAX; + for (int probe = 0; probe < tok->int_merges_cap; probe++, idx = (idx + 1) & mask) { + if (map[idx].key == UINT64_MAX) + return INT_MAX; if (map[idx].key == key) { *merged_id = map[idx].merged_id; return map[idx].rank; @@ -404,10 +454,10 @@ static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, } static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { - if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; + if (!tok->merge_map || tok->merge_map_cap <= 0) + return INT_MAX; - int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, a, b); + int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, a, b); return rank >= 0 ? rank : INT_MAX; } @@ -415,7 +465,8 @@ static int append_id(int **arr, int *n, int *cap, int id) { if (*n >= *cap) { int new_cap = (*cap == 0) ? 16 : (*cap * 2); int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); - if (!tmp) return -1; + if (!tmp) + return -1; *arr = tmp; *cap = new_cap; } @@ -429,25 +480,28 @@ static char *text_to_bpe_unicode(const char *text) { size_t len = strlen(text); /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ char *out = (char *)malloc(len * 3 + 1); - if (!out) return NULL; + if (!out) + return NULL; size_t w = 0; for (size_t i = 0; i < len; i++) { int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; char tmp[4]; int n = utf8_encode_cp(cp, tmp); - for (int j = 0; j < n; j++) out[w++] = tmp[j]; + for (int j = 0; j < n; j++) + out[w++] = tmp[j]; } out[w] = '\0'; return out; } -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, - int **out_ids, int *out_n); +static int +encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); -static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, - int *cp_out, int *len_out) { - if (p >= end || *p == '\0') return 0; +static int +utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) { + if (p >= end || *p == '\0') + return 0; unsigned char c = *p; if ((c & 0x80) == 0) { @@ -455,27 +509,19 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, *len_out = 1; return 1; } - if ((c & 0xE0) == 0xC0 && p + 1 < end && - (p[1] & 0xC0) == 0x80) { + if ((c & 0xE0) == 0xC0 && p + 1 < end && (p[1] & 0xC0) == 0x80) { *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F); *len_out = 2; return 1; } - if ((c & 0xF0) == 0xE0 && p + 2 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { - *cp_out = ((c & 0x0F) << 12) | - ((p[1] & 0x3F) << 6) | - (p[2] & 0x3F); + if ((c & 0xF0) == 0xE0 && p + 2 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { + *cp_out = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F); *len_out = 3; return 1; } - if ((c & 0xF8) == 0xF0 && p + 3 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && + if ((c & 0xF8) == 0xF0 && p + 3 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && (p[3] & 0xC0) == 0x80) { - *cp_out = ((c & 0x07) << 18) | - ((p[1] & 0x3F) << 12) | - ((p[2] & 0x3F) << 6) | - (p[3] & 0x3F); + *cp_out = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F); *len_out = 4; return 1; } @@ -494,47 +540,56 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, #define CPC_LETTER 4 static const unsigned char ascii_cp_class[128] = { - ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, - ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, - ['0' ... '9'] = CPC_NUMBER, - ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, + ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, + ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, + ['0' ... '9'] = CPC_NUMBER, ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, }; static int is_number_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_NUMBER; - if (cp >= 0x0660 && cp <= 0x0669) return 1; - if (cp >= 0x06F0 && cp <= 0x06F9) return 1; - if (cp >= 0x0966 && cp <= 0x096F) return 1; - if (cp >= 0xFF10 && cp <= 0xFF19) return 1; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_NUMBER; + if (cp >= 0x0660 && cp <= 0x0669) + return 1; + if (cp >= 0x06F0 && cp <= 0x06F9) + return 1; + if (cp >= 0x0966 && cp <= 0x096F) + return 1; + if (cp >= 0xFF10 && cp <= 0xFF19) + return 1; return 0; } static int is_space_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_SPACE; - return cp == 0x85 || cp == 0xA0 || - cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || - cp == 0x2028 || cp == 0x2029 || cp == 0x202F || - cp == 0x205F || cp == 0x3000; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_SPACE; + return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || + cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; } static int is_symbolish_cp(int cp) { - if (cp < 0x80) return 0; - if (cp >= 0x0300 && cp <= 0x036F) return 1; - if (cp >= 0x00A1 && cp <= 0x00BF && - cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) + if (cp < 0x80) + return 0; + if (cp >= 0x0300 && cp <= 0x036F) + return 1; + if (cp >= 0x00A1 && cp <= 0x00BF && cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) + return 1; + if (cp == 0x00D7 || cp == 0x00F7) + return 1; + if (cp >= 0x064B && cp <= 0x065F) + return 1; + if ((cp >= 0x0900 && cp <= 0x0903) || (cp >= 0x093A && cp <= 0x094F) || + (cp >= 0x0951 && cp <= 0x0957) || (cp >= 0x0962 && cp <= 0x0963)) return 1; - if (cp == 0x00D7 || cp == 0x00F7) return 1; - if (cp >= 0x064B && cp <= 0x065F) return 1; - if ((cp >= 0x0900 && cp <= 0x0903) || - (cp >= 0x093A && cp <= 0x094F) || - (cp >= 0x0951 && cp <= 0x0957) || - (cp >= 0x0962 && cp <= 0x0963)) + if (cp >= 0x2000 && cp <= 0x206F) + return 1; + if (cp >= 0x2190 && cp <= 0x27BF) + return 1; + if (cp >= 0x2E00 && cp <= 0x2E7F) + return 1; + if (cp >= 0x3000 && cp <= 0x303F) + return 1; + if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; - if (cp >= 0x2000 && cp <= 0x206F) return 1; - if (cp >= 0x2190 && cp <= 0x27BF) return 1; - if (cp >= 0x2E00 && cp <= 0x2E7F) return 1; - if (cp >= 0x3000 && cp <= 0x303F) return 1; - if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; return 0; } @@ -542,63 +597,162 @@ static int compose_latin_mark(int base, int mark) { switch (mark) { case 0x0300: switch (base) { - case 'A': return 0x00C0; case 'E': return 0x00C8; - case 'I': return 0x00CC; case 'O': return 0x00D2; - case 'U': return 0x00D9; case 'a': return 0x00E0; - case 'e': return 0x00E8; case 'i': return 0x00EC; - case 'o': return 0x00F2; case 'u': return 0x00F9; + case 'A': + return 0x00C0; + case 'E': + return 0x00C8; + case 'I': + return 0x00CC; + case 'O': + return 0x00D2; + case 'U': + return 0x00D9; + case 'a': + return 0x00E0; + case 'e': + return 0x00E8; + case 'i': + return 0x00EC; + case 'o': + return 0x00F2; + case 'u': + return 0x00F9; } break; case 0x0301: switch (base) { - case 'A': return 0x00C1; case 'C': return 0x0106; - case 'E': return 0x00C9; case 'I': return 0x00CD; - case 'L': return 0x0139; case 'N': return 0x0143; - case 'O': return 0x00D3; case 'R': return 0x0154; - case 'S': return 0x015A; case 'U': return 0x00DA; - case 'Y': return 0x00DD; case 'Z': return 0x0179; - case 'a': return 0x00E1; case 'c': return 0x0107; - case 'e': return 0x00E9; case 'i': return 0x00ED; - case 'l': return 0x013A; case 'n': return 0x0144; - case 'o': return 0x00F3; case 'r': return 0x0155; - case 's': return 0x015B; case 'u': return 0x00FA; - case 'y': return 0x00FD; case 'z': return 0x017A; + case 'A': + return 0x00C1; + case 'C': + return 0x0106; + case 'E': + return 0x00C9; + case 'I': + return 0x00CD; + case 'L': + return 0x0139; + case 'N': + return 0x0143; + case 'O': + return 0x00D3; + case 'R': + return 0x0154; + case 'S': + return 0x015A; + case 'U': + return 0x00DA; + case 'Y': + return 0x00DD; + case 'Z': + return 0x0179; + case 'a': + return 0x00E1; + case 'c': + return 0x0107; + case 'e': + return 0x00E9; + case 'i': + return 0x00ED; + case 'l': + return 0x013A; + case 'n': + return 0x0144; + case 'o': + return 0x00F3; + case 'r': + return 0x0155; + case 's': + return 0x015B; + case 'u': + return 0x00FA; + case 'y': + return 0x00FD; + case 'z': + return 0x017A; } break; case 0x0302: switch (base) { - case 'A': return 0x00C2; case 'E': return 0x00CA; - case 'I': return 0x00CE; case 'O': return 0x00D4; - case 'U': return 0x00DB; case 'a': return 0x00E2; - case 'e': return 0x00EA; case 'i': return 0x00EE; - case 'o': return 0x00F4; case 'u': return 0x00FB; + case 'A': + return 0x00C2; + case 'E': + return 0x00CA; + case 'I': + return 0x00CE; + case 'O': + return 0x00D4; + case 'U': + return 0x00DB; + case 'a': + return 0x00E2; + case 'e': + return 0x00EA; + case 'i': + return 0x00EE; + case 'o': + return 0x00F4; + case 'u': + return 0x00FB; } break; case 0x0303: switch (base) { - case 'A': return 0x00C3; case 'N': return 0x00D1; - case 'O': return 0x00D5; case 'a': return 0x00E3; - case 'n': return 0x00F1; case 'o': return 0x00F5; + case 'A': + return 0x00C3; + case 'N': + return 0x00D1; + case 'O': + return 0x00D5; + case 'a': + return 0x00E3; + case 'n': + return 0x00F1; + case 'o': + return 0x00F5; } break; case 0x0308: switch (base) { - case 'A': return 0x00C4; case 'E': return 0x00CB; - case 'I': return 0x00CF; case 'O': return 0x00D6; - case 'U': return 0x00DC; case 'Y': return 0x0178; - case 'a': return 0x00E4; case 'e': return 0x00EB; - case 'i': return 0x00EF; case 'o': return 0x00F6; - case 'u': return 0x00FC; case 'y': return 0x00FF; + case 'A': + return 0x00C4; + case 'E': + return 0x00CB; + case 'I': + return 0x00CF; + case 'O': + return 0x00D6; + case 'U': + return 0x00DC; + case 'Y': + return 0x0178; + case 'a': + return 0x00E4; + case 'e': + return 0x00EB; + case 'i': + return 0x00EF; + case 'o': + return 0x00F6; + case 'u': + return 0x00FC; + case 'y': + return 0x00FF; } break; case 0x030A: switch (base) { - case 'A': return 0x00C5; case 'a': return 0x00E5; + case 'A': + return 0x00C5; + case 'a': + return 0x00E5; } break; case 0x0327: switch (base) { - case 'C': return 0x00C7; case 'c': return 0x00E7; + case 'C': + return 0x00C7; + case 'c': + return 0x00E7; } break; } @@ -609,13 +763,15 @@ static char *normalize_nfc_latin(const char *text) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); char *out = (char *)malloc((size_t)(end - p) * 2 + 1); - if (!out) return NULL; + if (!out) + return NULL; size_t w = 0; while (p < end) { const unsigned char *cur = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; p += len; if (p < end) { @@ -625,7 +781,8 @@ static char *normalize_nfc_latin(const char *text) { if (composed) { char tmp[4]; int n = utf8_encode_cp(composed, tmp); - for (int i = 0; i < n; i++) out[w++] = tmp[i]; + for (int i = 0; i < n; i++) + out[w++] = tmp[i]; p += mark_len; continue; } @@ -640,26 +797,22 @@ static char *normalize_nfc_latin(const char *text) { } static int is_letter_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_LETTER; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_LETTER; if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp)) return 0; return 1; } -static int is_word_cp(int cp) { - return is_letter_cp(cp) || is_number_cp(cp); -} +static int is_word_cp(int cp) { return is_letter_cp(cp) || is_number_cp(cp); } -static int is_newline_cp(int cp) { - return cp == '\n' || cp == '\r'; -} +static int is_newline_cp(int cp) { return cp == '\n' || cp == '\r'; } -static int ascii_lower(int c) { - return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; -} +static int ascii_lower(int c) { return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; } static int match_contraction(const unsigned char *p, const unsigned char *end) { - if (p >= end || *p != '\'') return 0; + if (p >= end || *p != '\'') + return 0; int remain = (int)(end - p); if (remain >= 2) { int c1 = ascii_lower(p[1]); @@ -669,27 +822,31 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) { if (remain >= 3) { int c1 = ascii_lower(p[1]); int c2 = ascii_lower(p[2]); - if ((c1 == 'r' && c2 == 'e') || - (c1 == 'v' && c2 == 'e') || - (c1 == 'l' && c2 == 'l')) + if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l')) return 3; } return 0; } static int append_encoded_piece(const qwen_tokenizer_t *tok, - const char *piece, size_t piece_len, - int **ids, int *n_ids, int *cap) { - if (piece_len == 0) return 0; + const char *piece, + size_t piece_len, + int **ids, + int *n_ids, + int *cap) { + if (piece_len == 0) + return 0; char *tmp = (char *)malloc(piece_len + 1); - if (!tmp) return -1; + if (!tmp) + return -1; memcpy(tmp, piece, piece_len); tmp[piece_len] = '\0'; char *mapped = text_to_bpe_unicode(tmp); free(tmp); - if (!mapped) return -1; + if (!mapped) + return -1; int *piece_ids = NULL; int n_piece = 0; @@ -710,8 +867,8 @@ static int append_encoded_piece(const qwen_tokenizer_t *tok, return 0; } -static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, - int **out_ids, int *out_n) { +static int +encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; const unsigned char *end = start + strlen(text); @@ -721,7 +878,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { const unsigned char *piece = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; int clen = match_contraction(p, end); if (clen > 0) { @@ -731,13 +889,13 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int next_cp = 0, next_len = 0; int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { + if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_letter_cp(cp)) { @@ -745,7 +903,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_number_cp(cp)) { @@ -756,14 +915,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (q < end) { int c = 0, l = 0; utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; + if (!is_space_cp(c)) + break; + if (is_newline_cp(c)) + saw_newline = 1; q += l; if (saw_newline) { while (q < end) { int c2 = 0, l2 = 0; utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; + if (!is_newline_cp(c2)) + break; q += l2; } break; @@ -772,19 +934,20 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, if (saw_newline) { p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } else { @@ -794,7 +957,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (q2 < end) { int c = 0, l = 0; utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; + if (!is_space_cp(c)) + break; last = q2; q2 += l; count++; @@ -809,20 +973,22 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } } - if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), - &ids, &n_ids, &cap) != 0) { + if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, + &cap) != 0) { free(ids); return -1; } @@ -834,14 +1000,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, } /* Encode one mapped BPE unicode string to token IDs. */ -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { +static int +encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { *out_ids = NULL; *out_n = 0; - if (!mapped || !*mapped) return 0; + if (!mapped || !*mapped) + return 0; char **syms = NULL; int n_syms = 0; - if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1; + if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) + return -1; if (n_syms <= 0) { free(syms); return 0; @@ -857,18 +1026,21 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int best_i = i; } } - if (best_i < 0 || best_rank == INT_MAX) break; + if (best_i < 0 || best_rank == INT_MAX) + break; char *merged = str_concat2(syms[best_i], syms[best_i + 1]); if (!merged) { - for (int i = 0; i < n_syms; i++) free(syms[i]); + for (int i = 0; i < n_syms; i++) + free(syms[i]); free(syms); return -1; } free(syms[best_i]); free(syms[best_i + 1]); syms[best_i] = merged; - for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1]; + for (int j = best_i + 1; j < n_syms - 1; j++) + syms[j] = syms[j + 1]; n_syms--; } @@ -878,20 +1050,23 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); if (id < 0) { /* Should not happen with valid vocab + merges + byte-level mapping. */ - for (int k = 0; k < n_syms; k++) free(syms[k]); + for (int k = 0; k < n_syms; k++) + free(syms[k]); free(syms); free(ids); return -1; } if (append_id(&ids, &n_ids, &cap, id) != 0) { - for (int k = 0; k < n_syms; k++) free(syms[k]); + for (int k = 0; k < n_syms; k++) + free(syms[k]); free(syms); free(ids); return -1; } } - for (int i = 0; i < n_syms; i++) free(syms[i]); + for (int i = 0; i < n_syms; i++) + free(syms[i]); free(syms); *out_ids = ids; @@ -927,72 +1102,87 @@ struct qwen_tokenizer_workspace { }; static int reserve_bpe(qwen_tokenizer_workspace_t *ws, int need) { - if (ws->bpe_cap >= need) return 0; + if (ws->bpe_cap >= need) + return 0; int new_cap = ws->bpe_cap ? ws->bpe_cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } int *p; -#define R(field) \ +#define R(field) \ p = (int *)realloc(ws->field, (size_t)new_cap * sizeof(int)); \ - if (!p) return -1; \ + if (!p) \ + return -1; \ ws->field = p; - R(bpe_ids) R(bpe_prv) R(bpe_nxt) R(bpe_rnk) R(bpe_mrg) + R(bpe_ids) + R(bpe_prv) + R(bpe_nxt) + R(bpe_rnk) + R(bpe_mrg) #undef R - ws->bpe_cap = new_cap; + ws->bpe_cap = new_cap; return 0; } static int reserve_bytes(char **buf, size_t *cap, size_t need) { - if (*cap >= need) return 0; + if (*cap >= need) + return 0; size_t new_cap = *cap ? *cap : 256; while (new_cap < need) { - if (new_cap > SIZE_MAX / 2) return -1; + if (new_cap > SIZE_MAX / 2) + return -1; new_cap *= 2; } char *p = (char *)realloc(*buf, new_cap); - if (!p) return -1; + if (!p) + return -1; *buf = p; *cap = new_cap; return 0; } static int reserve_ints(int **buf, int *cap, int need) { - if (*cap >= need) return 0; + if (*cap >= need) + return 0; int new_cap = *cap ? *cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); - if (!p) return -1; + if (!p) + return -1; *buf = p; *cap = new_cap; return 0; } static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { - if (ws->syms_cap >= need) return 0; + if (ws->syms_cap >= need) + return 0; int new_cap = ws->syms_cap ? ws->syms_cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } - bpe_sym_ref_t *p = - (bpe_sym_ref_t *)realloc(ws->syms, - (size_t)new_cap * sizeof(*ws->syms)); - if (!p) return -1; + bpe_sym_ref_t *p = (bpe_sym_ref_t *)realloc(ws->syms, (size_t)new_cap * sizeof(*ws->syms)); + if (!p) + return -1; ws->syms = p; ws->syms_cap = new_cap; return 0; } -static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, - size_t len, size_t *off) { - if (ws->arena_len > SIZE_MAX - len - 1) return -1; +static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { + if (ws->arena_len > SIZE_MAX - len - 1) + return -1; size_t need = ws->arena_len + len + 1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) + return -1; *off = ws->arena_len; memcpy(ws->arena + ws->arena_len, s, len); ws->arena_len += len; @@ -1000,26 +1190,26 @@ static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, return 0; } -static const char *arena_str(const qwen_tokenizer_workspace_t *ws, - bpe_sym_ref_t sym) { +static const char *arena_str(const qwen_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { return ws->arena + sym.off; } -static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, - const char *text, const char **out) { +static int +normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, const char *text, const char **out) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); size_t in_len = (size_t)(end - p); - if (in_len > (SIZE_MAX - 1) / 2) return -1; - if (reserve_bytes(&ws->normalized, &ws->normalized_cap, - in_len * 2 + 1) != 0) + if (in_len > (SIZE_MAX - 1) / 2) + return -1; + if (reserve_bytes(&ws->normalized, &ws->normalized_cap, in_len * 2 + 1) != 0) return -1; size_t w = 0; while (p < end) { const unsigned char *cur = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; p += len; if (p < end) { @@ -1046,10 +1236,12 @@ static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, } static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, - const char *text, size_t len, + const char *text, + size_t len, const char **out) { init_gpt2_mapping(); - if (len > (SIZE_MAX - 1) / 3) return -1; + if (len > (SIZE_MAX - 1) / 3) + return -1; if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) return -1; @@ -1066,8 +1258,7 @@ static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, return 0; } -static int output_append_id(int *out_ids, int out_cap, int *n_ids, - int *overflow, int id) { +static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow, int id) { if (*n_ids < out_cap && out_ids) out_ids[*n_ids] = id; else @@ -1079,22 +1270,27 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids, static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *mapped, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (!mapped || !*mapped) return 0; + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (!mapped || !*mapped) + return 0; int n_syms = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int l = utf8_char_len(*p); p += l; n_syms++; } - if (n_syms <= 0) return 0; - if (reserve_syms(ws, n_syms) != 0) return -1; + if (n_syms <= 0) + return 0; + if (reserve_syms(ws, n_syms) != 0) + return -1; ws->arena_len = 0; int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int l = utf8_char_len(*p); size_t off = 0; if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) @@ -1107,21 +1303,21 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, int best_rank = INT_MAX; int best_i = -1; for (int j = 0; j < n_syms - 1; j++) { - int r = merge_rank(tok, arena_str(ws, ws->syms[j]), - arena_str(ws, ws->syms[j + 1])); + int r = merge_rank(tok, arena_str(ws, ws->syms[j]), arena_str(ws, ws->syms[j + 1])); if (r < best_rank) { best_rank = r; best_i = j; } } - if (best_i < 0 || best_rank == INT_MAX) break; + if (best_i < 0 || best_rank == INT_MAX) + break; const char *a = arena_str(ws, ws->syms[best_i]); const char *b = arena_str(ws, ws->syms[best_i + 1]); size_t la = strlen(a), lb = strlen(b); - if (la > SIZE_MAX - lb) return -1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, - ws->arena_len + la + lb + 1) != 0) + if (la > SIZE_MAX - lb) + return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, ws->arena_len + la + lb + 1) != 0) return -1; a = arena_str(ws, ws->syms[best_i]); @@ -1139,9 +1335,10 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, } for (int j = 0; j < n_syms; j++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, arena_str(ws, ws->syms[j])); - if (id < 0) return -1; + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, + arena_str(ws, ws->syms[j])); + if (id < 0) + return -1; output_append_id(out_ids, out_cap, n_ids, overflow, id); } return 0; @@ -1155,35 +1352,41 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, static int encode_bpe_word_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *mapped, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (!mapped || !*mapped) return 0; + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (!mapped || !*mapped) + return 0; if (!tok->int_merges || !tok->cp_to_id) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, - n_ids, overflow); + return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); int n = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { p += utf8_char_len(*p); n++; } - if (n <= 0) return 0; - if (reserve_bpe(ws, n) != 0) return -1; + if (n <= 0) + return 0; + if (reserve_bpe(ws, n) != 0) + return -1; int *ids = ws->bpe_ids, *prv = ws->bpe_prv, *nxt = ws->bpe_nxt; int *rnk = ws->bpe_rnk, *mrg = ws->bpe_mrg; int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int len = utf8_char_len(*p); int cp; - if (len == 1) cp = *p; - else if (len == 2) cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F); - else cp = 512; /* byte-level alphabet never exceeds two bytes */ + if (len == 1) + cp = *p; + else if (len == 2) + cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F); + else + cp = 512; /* byte-level alphabet never exceeds two bytes */ int id = cp < 512 ? tok->cp_to_id[cp] : -1; if (id < 0) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, - out_cap, n_ids, overflow); + return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); ids[i] = id; prv[i] = i - 1; nxt[i] = i + 1 < n ? i + 1 : -1; @@ -1193,7 +1396,8 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, for (int j = 0; j < n - 1; j++) rnk[j] = pair_merge_lookup(tok, ids[j], ids[j + 1], &mrg[j]); - if (n > 0) rnk[n - 1] = INT_MAX; + if (n > 0) + rnk[n - 1] = INT_MAX; for (;;) { int best = -1, best_rank = INT_MAX; @@ -1203,19 +1407,19 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, best = j; } } - if (best < 0) break; + if (best < 0) + break; ids[best] = mrg[best]; int dead = nxt[best]; nxt[best] = nxt[dead]; - if (nxt[dead] != -1) prv[nxt[dead]] = best; + if (nxt[dead] != -1) + prv[nxt[dead]] = best; if (prv[best] != -1) - rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], - &mrg[prv[best]]); - rnk[best] = nxt[best] != -1 - ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) - : INT_MAX; + rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]); + rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) + : INT_MAX; } for (int j = 0; j != -1; j = nxt[j]) @@ -1225,22 +1429,26 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, static int append_encoded_piece_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, - const char *piece, size_t piece_len, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (piece_len == 0) return 0; + const char *piece, + size_t piece_len, + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (piece_len == 0) + return 0; const char *mapped = NULL; if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) return -1; - return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, - overflow); + return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); } static int encode_pretokenized_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, - int *out_ids, int out_cap, + int *out_ids, + int out_cap, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; @@ -1251,7 +1459,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { const unsigned char *piece = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; int clen = match_contraction(p, end); if (clen > 0) { @@ -1261,13 +1470,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, int next_cp = 0, next_len = 0; int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { + if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_letter_cp(cp)) { @@ -1275,7 +1484,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_number_cp(cp)) { @@ -1286,14 +1496,17 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (q < end) { int c = 0, l = 0; utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; + if (!is_space_cp(c)) + break; + if (is_newline_cp(c)) + saw_newline = 1; q += l; if (saw_newline) { while (q < end) { int c2 = 0, l2 = 0; utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; + if (!is_newline_cp(c2)) + break; q += l2; } break; @@ -1302,19 +1515,20 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, if (saw_newline) { p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } else { @@ -1324,7 +1538,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (q2 < end) { int c = 0, l = 0; utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; + if (!is_space_cp(c)) + break; last = q2; q2 += l; count++; @@ -1339,21 +1554,22 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } } - if (append_encoded_piece_into(tok, ws, (const char *)piece, - (size_t)(p - piece), out_ids, out_cap, - &n_ids, &overflow) != 0) + if (append_encoded_piece_into(tok, ws, (const char *)piece, (size_t)(p - piece), out_ids, + out_cap, &n_ids, &overflow) != 0) return -1; } @@ -1364,11 +1580,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { const char *slash = strrchr(vocab_path, '/'); if (!slash) { - if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1; + if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) + return -1; return 0; } size_t dir_len = (size_t)(slash - vocab_path); - if (dir_len + 12 >= out_cap) return -1; + if (dir_len + 12 >= out_cap) + return -1; memcpy(out_path, vocab_path, dir_len); out_path[dir_len] = '\0'; snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); @@ -1377,22 +1595,28 @@ static int derive_merges_path(const char *vocab_path, char *out_path, size_t out static void trim_newline(char *line) { size_t n = strlen(line); - while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0'; + while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) + line[--n] = '\0'; } static int parse_merge_pair(char *line, char **a, char **b) { *a = NULL; *b = NULL; trim_newline(line); - if (line[0] == '\0') return -1; - if (line[0] == '#') return -1; + if (line[0] == '\0') + return -1; + if (line[0] == '#') + return -1; char *sp = strchr(line, ' '); - if (!sp) return -1; + if (!sp) + return -1; *sp = '\0'; char *p2 = sp + 1; - while (*p2 == ' ') p2++; - if (*line == '\0' || *p2 == '\0') return -1; + while (*p2 == ' ') + p2++; + if (*line == '\0' || *p2 == '\0') + return -1; *a = line; *b = p2; return 0; @@ -1400,13 +1624,15 @@ static int parse_merge_pair(char *line, char **a, char **b) { static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { FILE *f = fopen(merges_path, "rb"); - if (!f) return -1; + if (!f) + return -1; int n_pairs = 0; char line[8192]; while (fgets(line, sizeof(line), f)) { char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) == 0) n_pairs++; + if (parse_merge_pair(line, &a, &b) == 0) + n_pairs++; } if (n_pairs <= 0) { fclose(f); @@ -1426,8 +1652,8 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { return -1; } tok->int_merges_cap = next_pow2(n_pairs * 2); - pair_merge_entry_t *pm = (pair_merge_entry_t *) - calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t)); + pair_merge_entry_t *pm = + (pair_merge_entry_t *)calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t)); if (pm) { for (int i = 0; i < tok->int_merges_cap; i++) { pm[i].key = UINT64_MAX; @@ -1440,11 +1666,13 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { int rank = 0; while (fgets(line, sizeof(line), f)) { char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) != 0) continue; + if (parse_merge_pair(line, &a, &b) != 0) + continue; size_t la = strlen(a), lb = strlen(b); char *key = (char *)malloc(la + 1 + lb + 1); - if (!key) continue; + if (!key) + continue; memcpy(key, a, la); key[la] = ' '; memcpy(key + la + 1, b, lb); @@ -1470,15 +1698,11 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { tok->int_merges = NULL; tok->int_merges_cap = 0; } else { - pair_map_insert((pair_merge_entry_t *)tok->int_merges, - tok->int_merges_cap, - ((uint64_t)(uint32_t)left << 32) | - (uint32_t)right, - rank, merged); + pair_map_insert((pair_merge_entry_t *)tok->int_merges, tok->int_merges_cap, + ((uint64_t)(uint32_t)left << 32) | (uint32_t)right, rank, merged); } } - (void)map_insert_owned((str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, &key, rank); + (void)map_insert_owned((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, &key, rank); free(key); rank++; } @@ -1489,17 +1713,22 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { init_gpt2_mapping(); tok->cp_to_id = (int *)malloc(512 * sizeof(int)); if (tok->cp_to_id) { - for (int i = 0; i < 512; i++) tok->cp_to_id[i] = -1; + for (int i = 0; i < 512; i++) + tok->cp_to_id[i] = -1; for (int byte = 0; byte < 256 && tok->cp_to_id; byte++) { int cp = gpt2_byte_to_unicode[byte]; char buf[4]; int n; - if (cp < 0x80) { buf[0] = (char)cp; n = 1; } - else { buf[0] = (char)(0xC0 | (cp >> 6)); - buf[1] = (char)(0x80 | (cp & 0x3F)); n = 2; } + if (cp < 0x80) { + buf[0] = (char)cp; + n = 1; + } else { + buf[0] = (char)(0xC0 | (cp >> 6)); + buf[1] = (char)(0x80 | (cp & 0x3F)); + n = 2; + } buf[n] = '\0'; - int id = map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, buf); + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, buf); if (id < 0) { free(tok->cp_to_id); tok->cp_to_id = NULL; @@ -1554,16 +1783,23 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { const char *saved = p; while (*p && *p != '}') { skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; + if (parse_json_string(&p, key, sizeof(key)) != 0) + break; skip_ws(&p); - if (*p != ':') break; + if (*p != ':') + break; p++; int64_t id = parse_json_int(&p); - if (id > max_id) max_id = (int)id; + if (id > max_id) + max_id = (int)id; } int vocab_size = max_id + 1; @@ -1584,13 +1820,19 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { p = saved; while (*p && *p != '}') { skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; + if (parse_json_string(&p, key, sizeof(key)) != 0) + break; skip_ws(&p); - if (*p != ':') break; + if (*p != ':') + break; p++; int64_t id = parse_json_int(&p); @@ -1604,7 +1846,9 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { free(json); int n_vocab_entries = 0; - for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++; + for (int i = 0; i < vocab_size; i++) + if (tok->id_to_bpe[i]) + n_vocab_entries++; tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); if (!tok->vocab_map) { @@ -1620,7 +1864,8 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { char merges_path[1024]; if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { - fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", + fprintf(stderr, + "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", merges_path); } } @@ -1629,22 +1874,26 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { } const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { - if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; + if (!tok || token_id < 0 || token_id >= tok->vocab_size) + return ""; return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; } int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) { - if (!tok || !token || !tok->vocab_map) return -1; - return map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, token); + if (!tok || !token || !tok->vocab_map) + return -1; + return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); } int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !text || text[0] == '\0') return NULL; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !text || text[0] == '\0') + return NULL; char *normalized = normalize_nfc_latin(text); - if (!normalized) return NULL; + if (!normalized) + return NULL; int *ids = NULL; int n_ids = 0; @@ -1655,17 +1904,18 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o } free(normalized); - if (out_n_tokens) *out_n_tokens = n_ids; + if (out_n_tokens) + *out_n_tokens = n_ids; return ids; } qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { - return (qwen_tokenizer_workspace_t *)calloc(1, - sizeof(qwen_tokenizer_workspace_t)); + return (qwen_tokenizer_workspace_t *)calloc(1, sizeof(qwen_tokenizer_workspace_t)); } void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { - if (!ws) return; + if (!ws) + return; free(ws->normalized); free(ws->mapped); free(ws->arena); @@ -1682,20 +1932,24 @@ void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, - int *out_ids, int out_cap, + int *out_ids, + int out_cap, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || out_cap < 0) return -1; - if (text[0] == '\0') return 0; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text || out_cap < 0) + return -1; + if (text[0] == '\0') + return 0; const char *normalized = NULL; if (normalize_nfc_latin_into(ws, text, &normalized) != 0) return -1; int n_ids = 0; - int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, - &n_ids); - if (out_n_tokens) *out_n_tokens = n_ids; + int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, &n_ids); + if (out_n_tokens) + *out_n_tokens = n_ids; return rc; } @@ -1703,50 +1957,58 @@ int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || text[0] == '\0') return NULL; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text || text[0] == '\0') + return NULL; size_t len = strlen(text); size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; - if (cap_sz > (size_t)INT_MAX) return NULL; + if (cap_sz > (size_t)INT_MAX) + return NULL; int cap = (int)cap_sz; if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) return NULL; int n_ids = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); + int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); if (rc == -2) { if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) return NULL; - rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); + rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); } - if (rc != 0 || n_ids <= 0) return NULL; + if (rc != 0 || n_ids <= 0) + return NULL; int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); - if (!ids) return NULL; + if (!ids) + return NULL; memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); - if (out_n_tokens) *out_n_tokens = n_ids; + if (out_n_tokens) + *out_n_tokens = n_ids; return ids; } void qwen_tokenizer_free(qwen_tokenizer_t *tok) { - if (!tok) return; + if (!tok) + return; if (tok->id_to_text) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]); + for (int i = 0; i < tok->vocab_size; i++) + free(tok->id_to_text[i]); free(tok->id_to_text); } if (tok->id_to_bpe) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]); + for (int i = 0; i < tok->vocab_size; i++) + free(tok->id_to_bpe[i]); free(tok->id_to_bpe); } if (tok->merge_map) { str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; - for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key); + for (int i = 0; i < tok->merge_map_cap; i++) + free(m[i].key); free(tok->merge_map); } free(tok->vocab_map); diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index d7670b7..f7a3941 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -21,8 +21,8 @@ #endif typedef struct { - char **id_to_text; /* [vocab_size] decoded text strings */ - char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ int vocab_size; /* Internal hash maps (opaque to callers) */ @@ -55,7 +55,8 @@ EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *t /* Encode UTF-8 text into token IDs using BPE. * Returns malloc'd array of token IDs and sets *out_n_tokens. * Returns NULL on error (and sets *out_n_tokens to 0). */ -EMBED_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); +EMBED_API int * +qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); /* Reusable scratch for allocation-light repeated encoding. */ EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); @@ -66,17 +67,18 @@ EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); * In all cases *out_n_tokens is set to the number of tokens needed/emitted * when out_n_tokens is non-NULL. */ EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, int out_cap, - int *out_n_tokens); + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); /* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and * performs only the final returned int[] allocation. */ EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens); + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens); /* Free tokenizer */ EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok); diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c index cc9b2fb..9a5d842 100644 --- a/tests/test_bf16_model.c +++ b/tests/test_bf16_model.c @@ -15,12 +15,14 @@ int main(void) { char root[1024], f32_dir[1088], bf16_dir[1088]; snprintf(root, sizeof(root), "%s/embed-bf16-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(root)) { perror("mkdtemp"); return 2; } + if (!mkdtemp(root)) { + perror("mkdtemp"); + return 2; + } snprintf(f32_dir, sizeof(f32_dir), "%s/f32", root); snprintf(bf16_dir, sizeof(bf16_dir), "%s/bf16", root); if (mkdir(f32_dir, 0755) != 0 || mkdir(bf16_dir, 0755) != 0 || - tm_write_model(f32_dir, "F32") != 0 || - tm_write_model(bf16_dir, "BF16") != 0) { + tm_write_model(f32_dir, "F32") != 0 || tm_write_model(bf16_dir, "BF16") != 0) { fprintf(stderr, "fixture creation failed\n"); return 2; } @@ -58,7 +60,8 @@ int main(void) { return 1; } float d = fabsf(a[i] - b[i]); - if (d > max_diff) max_diff = d; + if (d > max_diff) + max_diff = d; norm += b[i] * b[i]; } norm = sqrtf(norm); @@ -73,7 +76,8 @@ int main(void) { * (linear_nobias_weight); parity must hold there too. */ enum { LONG_N = 24 }; int long_ids[LONG_N]; - for (int i = 0; i < LONG_N; i++) long_ids[i] = (i % 14) + 1; + for (int i = 0; i < LONG_N; i++) + long_ids[i] = (i % 14) + 1; if (embed_model_encode_into(mf32, wf32, long_ids, LONG_N, a) != 0 || embed_model_encode_into(mbf16, wbf16, long_ids, LONG_N, b) != 0) { fprintf(stderr, "long-sequence embedding failed\n"); @@ -86,15 +90,15 @@ int main(void) { return 1; } float d = fabsf(a[i] - b[i]); - if (d > long_diff) long_diff = d; + if (d > long_diff) + long_diff = d; } if (long_diff > 2e-5f) { fprintf(stderr, "bad widen-path parity: max_diff=%.9g\n", long_diff); return 1; } - printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n", - dim, max_diff, norm); + printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n", dim, max_diff, norm); embed_workspace_free(wf32); embed_workspace_free(wbf16); diff --git a/tests/test_cli.c b/tests/test_cli.c index 58cbad3..8d56f09 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -13,68 +13,69 @@ #include #include -static char g_dir[1024]; /* fixture model dir */ -static char g_out[1100]; /* captured stdout */ -static char g_err[1100]; /* captured stderr */ -static char g_in[1100]; /* stdin source */ +static char g_dir[1024]; /* fixture model dir */ +static char g_out[1100]; /* captured stdout */ +static char g_err[1100]; /* captured stderr */ +static char g_in[1100]; /* stdin source */ static const char *g_cli; static int g_failures; /* Run the CLI with sh-formatted args; returns its exit status (-1 on a * harness error). stdin_text == NULL feeds /dev/null. */ -static int run_cli(const char *args, const char *stdin_text) -{ +static int run_cli(const char *args, const char *stdin_text) { if (stdin_text) { FILE *f = fopen(g_in, "w"); - if (!f) return -1; + if (!f) + return -1; fputs(stdin_text, f); fclose(f); } char cmd[4096]; - int n = snprintf(cmd, sizeof(cmd), "'%s' %s > '%s' 2> '%s' < '%s'", - g_cli, args, g_out, g_err, + int n = snprintf(cmd, sizeof(cmd), "'%s' %s > '%s' 2> '%s' < '%s'", g_cli, args, g_out, g_err, stdin_text ? g_in : "/dev/null"); - if (n < 0 || (size_t)n >= sizeof(cmd)) return -1; + if (n < 0 || (size_t)n >= sizeof(cmd)) + return -1; int st = system(cmd); - if (st == -1 || !WIFEXITED(st)) return -1; + if (st == -1 || !WIFEXITED(st)) + return -1; return WEXITSTATUS(st); } -static void expect(int cond, const char *what) -{ +static void expect(int cond, const char *what) { if (!cond) { fprintf(stderr, "FAIL: %s\n", what); g_failures++; } } -static int file_contains(const char *path, const char *needle) -{ +static int file_contains(const char *path, const char *needle) { static char buf[1 << 16]; FILE *f = fopen(path, "rb"); - if (!f) return 0; + if (!f) + return 0; size_t n = fread(buf, 1, sizeof(buf) - 1, f); fclose(f); buf[n] = '\0'; return strstr(buf, needle) != NULL; } -static int count_lines(const char *path) -{ +static int count_lines(const char *path) { FILE *f = fopen(path, "rb"); - if (!f) return -1; + if (!f) + return -1; int lines = 0, c; while ((c = fgetc(f)) != EOF) - if (c == '\n') lines++; + if (c == '\n') + lines++; fclose(f); return lines; } /* First stdout line must hold exactly `dim` finite floats. */ -static int stdout_is_embedding(int dim) -{ +static int stdout_is_embedding(int dim) { FILE *f = fopen(g_out, "r"); - if (!f) return 0; + if (!f) + return 0; int count = 0; float v; while (fscanf(f, "%f", &v) == 1) { @@ -88,8 +89,7 @@ static int stdout_is_embedding(int dim) return count == dim; } -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { g_cli = argc > 1 ? argv[1] : "./embed"; tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE}; @@ -109,13 +109,11 @@ int main(int argc, char **argv) /* Help and missing-argument handling. */ expect(run_cli("-h", NULL) == 0, "-h exits 0"); expect(run_cli("", NULL) == 1, "no arguments exits 1"); - expect(run_cli("-d /nonexistent-embed-dir hi", NULL) == 1, - "missing model dir exits 1"); + expect(run_cli("-d /nonexistent-embed-dir hi", NULL) == 1, "missing model dir exits 1"); /* -V reports the program's base name, not the path it was invoked by. */ expect(run_cli("-V", NULL) == 0, "-V exits 0"); - expect(!file_contains(g_out, "tests/"), - "-V strips the directory from the program name"); + expect(!file_contains(g_out, "tests/"), "-V strips the directory from the program name"); /* Mode 1: one text prints one embedding line. */ snprintf(args, sizeof(args), "-d '%s' 'hello world'", g_dir); @@ -123,11 +121,9 @@ int main(int argc, char **argv) expect(stdout_is_embedding(dims.hidden), "single text prints dim floats"); /* Thread count. */ - snprintf(args, sizeof(args), - "-d '%s' -t 2 -v 'hello world'", g_dir); + snprintf(args, sizeof(args), "-d '%s' -t 2 -v 'hello world'", g_dir); expect(run_cli(args, NULL) == 0, "-t 2 -v exits 0"); - expect(file_contains(g_err, "Using 2 CPU thread(s)"), - "-v reports the thread count"); + expect(file_contains(g_err, "Using 2 CPU thread(s)"), "-v reports the thread count"); /* Mode 2+: several texts print a bare numeric matrix, one row per line, * with no titles or labels. Two texts give a 2x2 with a 1.0 diagonal. */ @@ -152,8 +148,7 @@ int main(int argc, char **argv) /* -e appends the raw embeddings (label-free) after the matrix: three * matrix rows, a blank separator, then three embedding rows. */ - snprintf(args, sizeof(args), - "-d '%s' -e -b 1 'hello' 'world' 'held'", g_dir); + snprintf(args, sizeof(args), "-d '%s' -e -b 1 'hello' 'world' 'held'", g_dir); expect(run_cli(args, NULL) == 0, "-e -b 1 with three texts exits 0"); expect(count_lines(g_out) == 7, "-e prints matrix then raw embeddings"); @@ -167,16 +162,17 @@ int main(int argc, char **argv) snprintf(args, sizeof(args), "-d '%s' --stream", g_dir); expect(run_cli(args, "hello\nworld\n") == 0, "--stream exits 0"); expect(count_lines(g_out) == 2, "--stream writes one line per input"); - expect(file_contains(g_out, "{\"embedding\":["), - "--stream writes embedding JSON"); + expect(file_contains(g_out, "{\"embedding\":["), "--stream writes embedding JSON"); snprintf(args, sizeof(args), "-d '%s' --stream -b 2 -v", g_dir); - expect(run_cli(args, "hello\nworld\nheld\n") == 0, - "--stream -b 2 exits 0"); + expect(run_cli(args, "hello\nworld\nheld\n") == 0, "--stream -b 2 exits 0"); expect(count_lines(g_out) == 3, "--stream -b 2 covers the tail batch"); /* Flag validation: missing model dir must exit 1. */ - static const struct { const char *args; const char *what; } bad[] = { + static const struct { + const char *args; + const char *what; + } bad[] = { {"-d '' t", "empty model dir"}, }; for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 4a2a57d..fb3d2df 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -18,8 +18,7 @@ static void expect_close(const char *name, float got, float want, float tol) { float diff = fabsf(got - want); float scale = fmaxf(1.0f, fmaxf(fabsf(got), fabsf(want))); if (diff > tol * scale) { - fprintf(stderr, "%s: got %.9g want %.9g diff %.9g\n", - name, got, want, diff); + fprintf(stderr, "%s: got %.9g want %.9g diff %.9g\n", name, got, want, diff); failures++; } } @@ -39,10 +38,7 @@ static float bf16_to_f32(uint16_t x) { static void test_rms_norm(void) { const int seq = 2, hidden = 4; - const float x[8] = { - 1.0f, 2.0f, -3.0f, 4.0f, - 0.5f, -1.0f, 2.0f, -0.5f - }; + const float x[8] = {1.0f, 2.0f, -3.0f, 4.0f, 0.5f, -1.0f, 2.0f, -0.5f}; const float w[4] = {1.0f, 0.5f, -1.0f, 2.0f}; float out[8]; @@ -66,10 +62,7 @@ static void test_rope_neox(void) { const int seq = 2, heads = 1, head_dim = 4; const int positions[2] = {0, 3}; float cosv[8], sinv[8]; - float x[8] = { - 1.0f, 2.0f, 3.0f, 4.0f, - -1.0f, 0.5f, 2.0f, -0.25f - }; + float x[8] = {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, 0.5f, 2.0f, -0.25f}; float orig[8]; memcpy(orig, x, sizeof(x)); @@ -92,10 +85,17 @@ static void test_rope_neox(void) { } } -static void reference_packed_gqa(float *out, const float *Q, const float *K, - const float *V, const int *offsets, int batch, - int n_heads, int n_kv_heads, int head_dim, - float scale, int causal) { +static void reference_packed_gqa(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int causal) { int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; int heads_per_kv = n_heads / n_kv_heads; @@ -112,22 +112,28 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K, for (int j = start; j < key_end; j++) { const float *k = K + j * kv_hidden + kv_h * head_dim; float score = 0.0f; - for (int d = 0; d < head_dim; d++) score += q[d] * k[d]; + for (int d = 0; d < head_dim; d++) + score += q[d] * k[d]; score *= scale; - if (score > max_score) max_score = score; + if (score > max_score) + max_score = score; } float denom = 0.0f; - for (int d = 0; d < head_dim; d++) o[d] = 0.0f; + for (int d = 0; d < head_dim; d++) + o[d] = 0.0f; for (int j = start; j < key_end; j++) { const float *k = K + j * kv_hidden + kv_h * head_dim; const float *v = V + j * kv_hidden + kv_h * head_dim; float score = 0.0f; - for (int d = 0; d < head_dim; d++) score += q[d] * k[d]; + for (int d = 0; d < head_dim; d++) + score += q[d] * k[d]; float wt = expf(score * scale - max_score); denom += wt; - for (int d = 0; d < head_dim; d++) o[d] += wt * v[d]; + for (int d = 0; d < head_dim; d++) + o[d] += wt * v[d]; } - for (int d = 0; d < head_dim; d++) o[d] /= denom; + for (int d = 0; d < head_dim; d++) + o[d] /= denom; } } } @@ -151,11 +157,9 @@ static void test_packed_gqa_attention(void) { V[i] = sinf((float)i * 0.07f) - cosf((float)i * 0.13f); } - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); - reference_packed_gqa(want, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); for (int i = 0; i < total * q_hidden; i++) @@ -172,7 +176,8 @@ static void test_packed_gqa_attention_long(void) { float *V = malloc((size_t)total * kv_hidden * sizeof(*V)); float *got = malloc((size_t)total * q_hidden * sizeof(*got)); float *want = malloc((size_t)total * q_hidden * sizeof(*want)); - if (!Q || !K || !V || !got || !want) exit(2); + if (!Q || !K || !V || !got || !want) + exit(2); for (int i = 0; i < total * q_hidden; i++) Q[i] = sinf((float)i * 0.017f) - 0.25f; @@ -182,11 +187,9 @@ static void test_packed_gqa_attention_long(void) { } qwen_set_threads(4); - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); - reference_packed_gqa(want, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); qwen_set_threads(1); @@ -209,7 +212,8 @@ static void test_packed_causal_gqa_attention(void) { float *V = malloc((size_t)total * kv_hidden * sizeof(*V)); float *got = malloc((size_t)total * q_hidden * sizeof(*got)); float *want = malloc((size_t)total * q_hidden * sizeof(*want)); - if (!Q || !K || !V || !got || !want) exit(2); + if (!Q || !K || !V || !got || !want) + exit(2); for (int i = 0; i < total * q_hidden; i++) Q[i] = sinf((float)i * 0.019f) - 0.2f; @@ -219,11 +223,9 @@ static void test_packed_causal_gqa_attention(void) { } qwen_set_threads(4); - qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); - reference_packed_gqa(want, Q, K, V, offsets, 2, - n_heads, n_kv_heads, head_dim, + reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 1); qwen_set_threads(1); @@ -238,18 +240,13 @@ static void test_packed_causal_gqa_attention(void) { static void test_bf16_linear(void) { enum { seq = 2, in_dim = 3, out_dim = 2 }; - const float x[6] = { - 1.0f, -2.0f, 0.5f, - -1.0f, 0.25f, 2.0f - }; - const float wf32[6] = { - 0.5f, -1.0f, 2.0f, - -0.25f, 0.75f, 1.5f - }; + const float x[6] = {1.0f, -2.0f, 0.5f, -1.0f, 0.25f, 2.0f}; + const float wf32[6] = {0.5f, -1.0f, 2.0f, -0.25f, 0.75f, 1.5f}; uint16_t wbf16[6]; float got[seq * out_dim]; - for (int i = 0; i < 6; i++) wbf16[i] = f32_to_bf16(wf32[i]); + for (int i = 0; i < 6; i++) + wbf16[i] = f32_to_bf16(wf32[i]); qwen_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim); for (int s = 0; s < seq; s++) { @@ -287,8 +284,7 @@ static void check_bf16_qkv(int n_threads) { } qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, - seq, in_dim, q_dim, kv_dim); + qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < q_dim; o++) { @@ -349,7 +345,7 @@ static void test_bf16_matvec(void) { * rms_norm_worker; row-wise math is identical, so results must match the * single-thread run exactly. */ static void test_rms_norm_threaded(void) { - enum { seq = 96, hidden = 3072 }; /* 294912 elements */ + enum { seq = 96, hidden = 3072 }; /* 294912 elements */ size_t elems = (size_t)seq * hidden; float *x = (float *)malloc(elems * sizeof(float)); float *w = (float *)malloc((size_t)hidden * sizeof(float)); @@ -358,7 +354,10 @@ static void test_rms_norm_threaded(void) { if (!x || !w || !one || !many) { fprintf(stderr, "rms_norm_threaded: allocation failure\n"); failures++; - free(many); free(one); free(w); free(x); + free(many); + free(one); + free(w); + free(x); return; } unsigned s = 7u; @@ -377,13 +376,15 @@ static void test_rms_norm_threaded(void) { for (size_t i = 0; i < elems; i++) { if (one[i] != many[i]) { - fprintf(stderr, "rms_norm_threaded: mismatch at %zu: %.9g %.9g\n", - i, one[i], many[i]); + fprintf(stderr, "rms_norm_threaded: mismatch at %zu: %.9g %.9g\n", i, one[i], many[i]); failures++; break; } } - free(many); free(one); free(w); free(x); + free(many); + free(one); + free(w); + free(x); } /* bf16->f32 widening is an exact bit shift; check the SIMD body and the @@ -407,8 +408,7 @@ static void test_bf16_widen_buf(void) { memcpy(&got_bits, &got[i], sizeof(got_bits)); memcpy(&want_bits, &want, sizeof(want_bits)); if (got_bits != want_bits) { - fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n", - n, i, got[i], want); + fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n", n, i, got[i], want); failures++; } } @@ -436,8 +436,7 @@ static void check_bf16_pair(int n_threads) { } qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, - seq, in_dim, a_dim, b_dim); + qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < a_dim; o++) { @@ -477,8 +476,10 @@ static void test_generic_vs_impl(void) { s = s * 1664525u + 1013904223u; b[i] = (float)(int32_t)s * 1e-9f; } - for (int i = 0; i < IN; i++) x[i] = a[i]; - for (int o = 0; o < OUT; o++) bias[o] = b[o]; + for (int i = 0; i < IN; i++) + x[i] = a[i]; + for (int o = 0; o < OUT; o++) + bias[o] = b[o]; for (size_t i = 0; i < sizeof(W) / sizeof(W[0]); i++) { s = s * 1664525u + 1013904223u; W[i] = f32_to_bf16((float)(int32_t)s * 1e-9f); @@ -497,8 +498,7 @@ static void test_generic_vs_impl(void) { const int sizes[] = {1, 7, 64, NMAX}; for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; - expect_close("dot generic-vs-impl", - qwen_dot_f32_generic(a, b, n), + expect_close("dot generic-vs-impl", qwen_dot_f32_generic(a, b, n), qwen_dot_f32_impl(a, b, n), 1e-5f); memcpy(d0, a, (size_t)n * sizeof(float)); @@ -538,7 +538,8 @@ int main(void) { test_bf16_widen_buf(); test_bf16_pair(); test_generic_vs_impl(); - if (failures != 0) return 1; + if (failures != 0) + return 1; puts("ok: kernel golden tests passed"); return 0; } diff --git a/tests/test_late.c b/tests/test_late.c index 94bdfe2..e491596 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -15,50 +15,50 @@ enum { TOKEN_DIM = 6 }; -static float max_abs_diff(const float *a, const float *b, size_t n) -{ +static float max_abs_diff(const float *a, const float *b, size_t n) { float m = 0.0f; for (size_t i = 0; i < n; i++) { float d = fabsf(a[i] - b[i]); - if (d > m) m = d; + if (d > m) + m = d; } return m; } -static float row_norm(const float *v, int n) -{ +static float row_norm(const float *v, int n) { float s = 0.0f; - for (int i = 0; i < n; i++) s += v[i] * v[i]; + for (int i = 0; i < n; i++) + s += v[i] * v[i]; return sqrtf(s); } /* Parity between the grouped batch scorer and per-document MaxSim. */ -static int check_maxsim_batch_parity(const float *query, int query_tokens, - const float *docs, const int *offsets, - int n_docs, int dim, const char *what) -{ +static int check_maxsim_batch_parity(const float *query, + int query_tokens, + const float *docs, + const int *offsets, + int n_docs, + int dim, + const char *what) { float scores[8]; - if (n_docs > (int)(sizeof(scores) / sizeof(scores[0]))) return 1; - if (embed_late_maxsim_batch(query, query_tokens, docs, offsets, n_docs, - dim, scores) != 0) { + if (n_docs > (int)(sizeof(scores) / sizeof(scores[0]))) + return 1; + if (embed_late_maxsim_batch(query, query_tokens, docs, offsets, n_docs, dim, scores) != 0) { fprintf(stderr, "maxsim_batch (%s) failed\n", what); return 1; } for (int i = 0; i < n_docs; i++) { - float one = embed_late_maxsim(query, query_tokens, - docs + (size_t)offsets[i] * dim, - offsets[i + 1] - offsets[i], dim); + float one = embed_late_maxsim(query, query_tokens, docs + (size_t)offsets[i] * dim, + offsets[i + 1] - offsets[i], dim); if (fabsf(scores[i] - one) > 0.0001f) { - fprintf(stderr, "maxsim_batch (%s) doc %d: %g vs %g\n", - what, i, scores[i], one); + fprintf(stderr, "maxsim_batch (%s) doc %d: %g vs %g\n", what, i, scores[i], one); return 1; } } return 0; } -int main(void) -{ +int main(void) { int rc = 1; tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE}; int hidden = dims.hidden; @@ -77,8 +77,7 @@ int main(void) char dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-late-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(dir) || tf_write_vocab(dir) != 0 || - tm_write_model_dims(dir, "F32", &dims) != 0) { + if (!mkdtemp(dir) || tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0) { fprintf(stderr, "fixture creation failed\n"); return 2; } @@ -96,8 +95,7 @@ int main(void) fprintf(stderr, "late load succeeded without 1_Dense\n"); goto fail; } - if (embed_late_model_config(NULL) != NULL || - embed_late_model_token_dim(NULL) != 0 || + if (embed_late_model_config(NULL) != NULL || embed_late_model_token_dim(NULL) != 0 || embed_late_workspace_new(NULL) != NULL) { fprintf(stderr, "NULL-argument handling failed\n"); goto fail; @@ -106,8 +104,7 @@ int main(void) embed_late_model_free(NULL); /* A head whose tensor is not named linear.weight must be rejected. */ - if (tm_write_late_projection_named(dir, "F32", TOKEN_DIM, hidden, - "dense.weight") != 0) { + if (tm_write_late_projection_named(dir, "F32", TOKEN_DIM, hidden, "dense.weight") != 0) { fprintf(stderr, "failed to write malformed projection\n"); goto fail; } @@ -133,8 +130,7 @@ int main(void) } const embed_config_t *cfg = embed_late_model_config(late); - if (!cfg || cfg->hidden_size != hidden || - embed_late_model_token_dim(late) != TOKEN_DIM) { + if (!cfg || cfg->hidden_size != hidden || embed_late_model_token_dim(late) != TOKEN_DIM) { fprintf(stderr, "late config/token_dim mismatch\n"); goto fail; } @@ -253,33 +249,30 @@ int main(void) } offsets[i + 1] = offsets[i] + lens[i]; } - doc_vecs = (float *)calloc((size_t)offsets[3] * TOKEN_DIM, - sizeof(float)); + doc_vecs = (float *)calloc((size_t)offsets[3] * TOKEN_DIM, sizeof(float)); if (!doc_vecs) { fprintf(stderr, "allocation failure\n"); goto fail; } for (int i = 0; i < 3; i++) { - if (embed_late_model_encode_tokens( - late, ws, doc_ids[i], lens[i], 1, - doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) { + if (embed_late_model_encode_tokens(late, ws, doc_ids[i], lens[i], 1, + doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) { fprintf(stderr, "doc encode failed\n"); goto fail; } } - if (check_maxsim_batch_parity(vecs, n_tokens, doc_vecs, offsets, 3, - TOKEN_DIM, "encoded docs") != 0) + if (check_maxsim_batch_parity(vecs, n_tokens, doc_vecs, offsets, 3, TOKEN_DIM, + "encoded docs") != 0) goto fail; float scores[3]; int bad_first[4] = {1, 2, 3, 4}; int empty_doc[3] = {0, 2, 2}; - if (embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3, - TOKEN_DIM, scores) == 0 || - embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2, - TOKEN_DIM, scores) == 0 || - embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, offsets, 3, - TOKEN_DIM, NULL) == 0) { + if (embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3, TOKEN_DIM, scores) == + 0 || + embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2, TOKEN_DIM, scores) == + 0 || + embed_late_maxsim_batch(vecs, n_tokens, doc_vecs, offsets, 3, TOKEN_DIM, NULL) == 0) { fprintf(stderr, "maxsim_batch accepted invalid arguments\n"); goto fail; } @@ -305,20 +298,20 @@ int main(void) s = s * 1664525u + 1013904223u; big_d[i] = (float)((s >> 8) & 0xFFFF) / 32768.0f - 1.0f; } - if (check_maxsim_batch_parity(big_q, BIG_Q, big_d, offsets, 2, - BIG_DIM, "oversized doc") != 0) + if (check_maxsim_batch_parity(big_q, BIG_Q, big_d, offsets, 2, BIG_DIM, "oversized doc") != + 0) goto fail; } - printf("ok: late API token_dim=%d tokens=%d hidden=%d\n", - TOKEN_DIM, n_tokens, hidden); + printf("ok: late API token_dim=%d tokens=%d hidden=%d\n", TOKEN_DIM, n_tokens, hidden); rc = 0; fail: free(big_d); free(big_q); free(doc_vecs); - for (int i = 0; i < 3; i++) free(doc_ids[i]); + for (int i = 0; i < 3; i++) + free(doc_ids[i]); free(expected); free(vecs); free(raw); diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c index 4d63a35..d66e0f7 100644 --- a/tests/test_qwen3.c +++ b/tests/test_qwen3.c @@ -9,24 +9,22 @@ #include #include -static float max_abs_diff(const float *a, const float *b, int n) -{ +static float max_abs_diff(const float *a, const float *b, int n) { float out = 0.0f; for (int i = 0; i < n; i++) { float d = fabsf(a[i] - b[i]); - if (d > out) out = d; + if (d > out) + out = d; } return out; } -int main(void) -{ +int main(void) { char dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-qwen3-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); tm_dims_t dims = {4, 2, 1, 2, 8, 16}; - if (!mkdtemp(dir) || - tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) { + if (!mkdtemp(dir) || tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) { fprintf(stderr, "fixture creation failed\n"); return 2; } @@ -39,8 +37,8 @@ int main(void) return 1; } if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL || - cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || - !cfg->normalize_embeddings || !cfg->append_terminal_token) { + cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || !cfg->normalize_embeddings || + !cfg->append_terminal_token) { fprintf(stderr, "Qwen3 config semantics were not detected\n"); return 1; } @@ -58,15 +56,15 @@ int main(void) float expected[4]; memcpy(expected, states + 3 * 4, sizeof(expected)); - if (embed_l2_normalize(expected, 4) != 0 || - max_abs_diff(embedding, expected, 4) > 1e-5f || + if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f || max_abs_diff(pooled, expected, 4) > 1e-5f) { fprintf(stderr, "last-token pooling mismatch\n"); return 1; } float norm_sq = 0.0f; - for (int d = 0; d < 4; d++) norm_sq += embedding[d] * embedding[d]; + for (int d = 0; d < 4; d++) + norm_sq += embedding[d] * embedding[d]; if (fabsf(norm_sq - 1.0f) > 1e-5f) { fprintf(stderr, "Qwen3 embedding is not normalized: %.9g\n", norm_sq); return 1; @@ -78,8 +76,7 @@ int main(void) float single[4]; if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 || embed_model_encode_into(model, ws, short_ids, 2, single) != 0 || - max_abs_diff(batch, embedding, 4) > 1e-5f || - max_abs_diff(batch + 4, single, 4) > 1e-5f) { + max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) { fprintf(stderr, "Qwen3 packed batch parity failed\n"); return 1; } diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c index d527951..36ec8c5 100644 --- a/tests/test_safetensors.c +++ b/tests/test_safetensors.c @@ -15,29 +15,31 @@ static int failures = 0; static char tmproot[1024]; /* Write u64-LE header length + JSON header + payload (safetensors layout). */ -static void write_st(const char *path, const char *header, - const void *payload, size_t payload_len) -{ +static void +write_st(const char *path, const char *header, const void *payload, size_t payload_len) { FILE *f = fopen(path, "wb"); - if (!f) { perror(path); exit(2); } + if (!f) { + perror(path); + exit(2); + } uint64_t n = strlen(header); unsigned char len8[8]; - for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(n >> (8 * i)); + for (int i = 0; i < 8; i++) + len8[i] = (unsigned char)(n >> (8 * i)); fwrite(len8, 1, 8, f); fwrite(header, 1, n, f); - if (payload_len) fwrite(payload, 1, payload_len, f); + if (payload_len) + fwrite(payload, 1, payload_len, f); fclose(f); } -static const char *path_in(const char *rel) -{ +static const char *path_in(const char *rel) { static char buf[2048]; snprintf(buf, sizeof(buf), "%s/%s", tmproot, rel); return buf; } -static void check(const char *mode, const char *path, int want_ok) -{ +static void check(const char *mode, const char *path, int want_ok) { int got_ok; if (strcmp(mode, "file") == 0) { safetensors_file_t *sf = safetensors_open(path); @@ -49,8 +51,8 @@ static void check(const char *mode, const char *path, int want_ok) multi_safetensors_close(ms); } if (got_ok != want_ok) { - fprintf(stderr, "%s %s: got %s want %s\n", mode, path, - got_ok ? "ok" : "fail", want_ok ? "ok" : "fail"); + fprintf(stderr, "%s %s: got %s want %s\n", mode, path, got_ok ? "ok" : "fail", + want_ok ? "ok" : "fail"); failures++; } } @@ -62,11 +64,13 @@ static const char ONE_TENSOR_B[] = static const char ONE_TENSOR_X[] = "{\"x\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; -int main(void) -{ +int main(void) { snprintf(tmproot, sizeof(tmproot), "%s/embed-st-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(tmproot)) { perror("mkdtemp"); return 2; } + if (!mkdtemp(tmproot)) { + perror("mkdtemp"); + return 2; + } static const char zeros[4 * 1025] = {0}; @@ -77,14 +81,15 @@ int main(void) { size_t cap = 1025 * 64 + 16; char *hdr = malloc(cap); - if (!hdr) return 2; + if (!hdr) + return 2; size_t off = 0; off += (size_t)snprintf(hdr + off, cap - off, "{"); for (int i = 0; i < 1025; i++) { off += (size_t)snprintf(hdr + off, cap - off, - "%s\"t%d\":{\"dtype\":\"F32\",\"shape\":[1]," - "\"data_offsets\":[%d,%d]}", - i ? "," : "", i, i * 4, i * 4 + 4); + "%s\"t%d\":{\"dtype\":\"F32\",\"shape\":[1]," + "\"data_offsets\":[%d,%d]}", + i ? "," : "", i, i * 4, i * 4 + 4); } snprintf(hdr + off, cap - off, "}"); write_st(path_in("too_many.safetensors"), hdr, zeros, sizeof(zeros)); @@ -94,27 +99,26 @@ int main(void) write_st(path_in("too_many_dims.safetensors"), "{\"x\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1]," - "\"data_offsets\":[0,4]}}", zeros, 4); + "\"data_offsets\":[0,4]}}", + zeros, 4); check("file", path_in("too_many_dims.safetensors"), 0); write_st(path_in("bad_offsets.safetensors"), "{\"x\":{\"dtype\":\"F32\",\"shape\":[1]," - "\"data_offsets\":[4,0]}}", zeros, 4); + "\"data_offsets\":[4,0]}}", + zeros, 4); check("file", path_in("bad_offsets.safetensors"), 0); mkdir(path_in("valid_multi"), 0755); - write_st(path_in("valid_multi/model-00001-of-00002.safetensors"), - ONE_TENSOR_A, zeros, 4); - write_st(path_in("valid_multi/model-00002-of-00002.safetensors"), - ONE_TENSOR_B, zeros, 4); + write_st(path_in("valid_multi/model-00001-of-00002.safetensors"), ONE_TENSOR_A, zeros, 4); + write_st(path_in("valid_multi/model-00002-of-00002.safetensors"), ONE_TENSOR_B, zeros, 4); check("multi", path_in("valid_multi"), 1); /* Payload accounting across shards: two 4-byte F32 tensors. */ { multi_safetensors_t *ms = multi_safetensors_open(path_in("valid_multi")); size_t nbytes = 0; - if (!ms || multi_safetensors_data_nbytes(ms, &nbytes) != 0 || - nbytes != 8) { + if (!ms || multi_safetensors_data_nbytes(ms, &nbytes) != 0 || nbytes != 8) { fprintf(stderr, "multi data_nbytes: got %zu want 8\n", nbytes); failures++; } @@ -122,18 +126,15 @@ int main(void) } mkdir(path_in("missing"), 0755); - write_st(path_in("missing/model-00001-of-00002.safetensors"), - ONE_TENSOR_X, zeros, 4); + write_st(path_in("missing/model-00001-of-00002.safetensors"), ONE_TENSOR_X, zeros, 4); check("multi", path_in("missing"), 0); mkdir(path_in("too_many_shards"), 0755); - write_st(path_in("too_many_shards/model-00001-of-00009.safetensors"), - ONE_TENSOR_X, zeros, 4); + write_st(path_in("too_many_shards/model-00001-of-00009.safetensors"), ONE_TENSOR_X, zeros, 4); check("multi", path_in("too_many_shards"), 0); mkdir(path_in("bad_name"), 0755); - write_st(path_in("bad_name/model-foo.safetensors"), - ONE_TENSOR_X, zeros, 4); + write_st(path_in("bad_name/model-foo.safetensors"), ONE_TENSOR_X, zeros, 4); check("multi", path_in("bad_name"), 0); if (failures) { diff --git a/tests/test_server.c b/tests/test_server.c index a55462c..d799107 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -16,30 +16,35 @@ static int test_failures = 0; -#define TEST_ASSERT(cond) do { \ - if (!(cond)) { \ - fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - test_failures++; \ - } \ -} while (0) - -static void test_base64_rfc4648(void) -{ +#define TEST_ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + test_failures++; \ + } \ + } while (0) + +static void test_base64_rfc4648(void) { /* RFC 4648 test vectors. */ - static const struct { const char *in, *out; } cases[] = { - {"", ""}, {"f", "Zg=="}, {"fo", "Zm8="}, {"foo", "Zm9v"}, - {"foob", "Zm9vYg=="}, {"fooba", "Zm9vYmE="}, {"foobar", "Zm9vYmFy"}, + static const struct { + const char *in, *out; + } cases[] = { + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { - char *got = base64_encode((const unsigned char *)cases[i].in, - strlen(cases[i].in)); + char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in)); TEST_ASSERT(strcmp(got, cases[i].out) == 0); free(got); } } -static void test_quantize_int8_tanh(void) -{ +static void test_quantize_int8_tanh(void) { TEST_ASSERT(quantize_int8_tanh(0.0f) == 0); /* tanh saturates to +-1, scaled by 127. */ TEST_ASSERT(quantize_int8_tanh(100.0f) == 127); @@ -50,8 +55,7 @@ static void test_quantize_int8_tanh(void) TEST_ASSERT(quantize_int8_tanh(0.5f) == 59); } -static void test_encode_embedding_int8(void) -{ +static void test_encode_embedding_int8(void) { float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; signed char expect[4] = {0, 59, -59, 127}; char *got = encode_embedding(emb, 4, "base64_int8"); @@ -61,8 +65,7 @@ static void test_encode_embedding_int8(void) free(want); } -static void test_encode_embedding_binary(void) -{ +static void test_encode_embedding_binary(void) { /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */ float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f}; unsigned char expect = 0x85; /* bits 0, 2, 7 set */ @@ -82,8 +85,7 @@ static void test_encode_embedding_binary(void) free(want); } -static void test_encode_embedding_base64_float32(void) -{ +static void test_encode_embedding_base64_float32(void) { /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32 * vector - lossless, unlike the int8 formats. */ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; @@ -98,8 +100,7 @@ static void test_encode_embedding_base64_float32(void) free(want); } -static void test_append_embedding_value_float(void) -{ +static void test_append_embedding_value_float(void) { float emb[1] = {0.5f}; /* OpenAI/DashScope (Qwen3): the true float32 value. */ @@ -115,16 +116,14 @@ static void test_append_embedding_value_float(void) sbuf_free(&b2); } -static void test_encoding_from_root_family(void) -{ +static void test_encoding_from_root_family(void) { cJSON *detail = cJSON_CreateArray(); /* Default encoding differs by family when the field is absent. */ cJSON *empty = cJSON_CreateObject(); - TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_OPENAI), - "float") == 0); - TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_PERPLEXITY), - "base64_int8") == 0); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_OPENAI), "float") == 0); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, EMBED_API_PERPLEXITY), "base64_int8") == + 0); cJSON_Delete(empty); TEST_ASSERT(cJSON_GetArraySize(detail) == 0); @@ -144,8 +143,7 @@ static void test_encoding_from_root_family(void) cJSON_Delete(detail); } -static void test_collect_job_batch_deadline(void) -{ +static void test_collect_job_batch_deadline(void) { http_server s; memset(&s, 0, sizeof(s)); s.batch_size = 32; @@ -158,10 +156,8 @@ static void test_collect_job_batch_deadline(void) job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u}; job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u}; snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); - snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", - "/v1/embeddings"); - snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", - "/v1/embeddings"); + snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings"); + snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings"); enqueue_job(&first); enqueue_job(&before_deadline); enqueue_job(&after_deadline); @@ -179,8 +175,7 @@ static void test_collect_job_batch_deadline(void) pthread_mutex_destroy(&s.mu); } -static void test_collect_job_batch_zero_wait(void) -{ +static void test_collect_job_batch_zero_wait(void) { http_server s; memset(&s, 0, sizeof(s)); s.batch_size = 32; @@ -211,10 +206,10 @@ static void test_collect_job_batch_zero_wait(void) #define TEST_API_KEY "tt-test-key" -static int find_free_port(void) -{ +static int find_free_port(void) { int fd = socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) return -1; + if (fd < 0) + return -1; struct sockaddr_in a; memset(&a, 0, sizeof(a)); a.sin_family = AF_INET; @@ -230,10 +225,10 @@ static int find_free_port(void) return port; } -static int tcp_connect(int port) -{ +static int tcp_connect(int port) { int fd = socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) return -1; + if (fd < 0) + return -1; struct sockaddr_in a; memset(&a, 0, sizeof(a)); a.sin_family = AF_INET; @@ -246,13 +241,13 @@ static int tcp_connect(int port) return fd; } -static int send_all(int fd, const char *buf, size_t n) -{ +static int send_all(int fd, const char *buf, size_t n) { size_t off = 0; while (off < n) { ssize_t w = write(fd, buf + off, n - off); if (w < 0) { - if (errno == EINTR) continue; + if (errno == EINTR) + continue; return -1; } off += (size_t)w; @@ -262,10 +257,14 @@ static int send_all(int fd, const char *buf, size_t n) /* Send one request; raw == NULL builds a normal request, otherwise `raw` * bytes go on the wire verbatim (for malformed-HTTP checks). */ -static int http_send(int fd, const char *method, const char *path, - const char *auth, const char *body, const char *raw) -{ - if (raw) return send_all(fd, raw, strlen(raw)); +static int http_send(int fd, + const char *method, + const char *path, + const char *auth, + const char *body, + const char *raw) { + if (raw) + return send_all(fd, raw, strlen(raw)); char head[512]; size_t blen = body ? strlen(body) : 0; int n = snprintf(head, sizeof(head), @@ -275,55 +274,71 @@ static int http_send(int fd, const char *method, const char *path, "Content-Type: application/json\r\n" "Content-Length: %zu\r\n" "\r\n", - method, path, - auth ? "Authorization: Bearer " : "", - auth ? auth : "", auth ? "\r\n" : "", - blen); - if (n < 0 || (size_t)n >= sizeof(head)) return -1; - if (send_all(fd, head, (size_t)n) != 0) return -1; + method, path, auth ? "Authorization: Bearer " : "", auth ? auth : "", + auth ? "\r\n" : "", blen); + if (n < 0 || (size_t)n >= sizeof(head)) + return -1; + if (send_all(fd, head, (size_t)n) != 0) + return -1; return blen ? send_all(fd, body, blen) : 0; } /* Read one response: returns the status code (or -1) and the body, which * the caller frees. Reads exactly Content-Length body bytes so it works * with keep-alive connections. */ -static int http_recv(int fd, char **out_body) -{ +static int http_recv(int fd, char **out_body) { *out_body = NULL; size_t cap = 8192, len = 0; char *buf = malloc(cap); - if (!buf) return -1; + if (!buf) + return -1; char *hdr_end = NULL; while (!hdr_end) { if (len + 1 >= cap) { cap *= 2; char *nb = realloc(buf, cap); - if (!nb) { free(buf); return -1; } + if (!nb) { + free(buf); + return -1; + } buf = nb; } ssize_t r = read(fd, buf + len, cap - len - 1); - if (r <= 0) { free(buf); return -1; } + if (r <= 0) { + free(buf); + return -1; + } len += (size_t)r; buf[len] = '\0'; hdr_end = strstr(buf, "\r\n\r\n"); } int status = -1; - if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) { free(buf); return -1; } + if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) { + free(buf); + return -1; + } size_t content_length = 0; const char *cl = strstr(buf, "Content-Length:"); - if (cl && cl < hdr_end) content_length = strtoul(cl + 15, NULL, 10); + if (cl && cl < hdr_end) + content_length = strtoul(cl + 15, NULL, 10); size_t body_off = (size_t)(hdr_end - buf) + 4; size_t need = body_off + content_length; if (need + 1 > cap) { char *nb = realloc(buf, need + 1); - if (!nb) { free(buf); return -1; } + if (!nb) { + free(buf); + return -1; + } buf = nb; } while (len < need) { ssize_t r = read(fd, buf + len, need - len); - if (r <= 0) { free(buf); return -1; } + if (r <= 0) { + free(buf); + return -1; + } len += (size_t)r; } buf[need] = '\0'; @@ -334,19 +349,25 @@ static int http_recv(int fd, char **out_body) /* One-shot request helper; returns status, body via out_body (caller frees, * may be NULL when only the status matters). */ -static int http_req(int port, const char *method, const char *path, - const char *auth, const char *body, const char *raw, - char **out_body) -{ +static int http_req(int port, + const char *method, + const char *path, + const char *auth, + const char *body, + const char *raw, + char **out_body) { int fd = tcp_connect(port); - if (fd < 0) return -1; + if (fd < 0) + return -1; char *resp = NULL; int status = -1; if (http_send(fd, method, path, auth, body, raw) == 0) status = http_recv(fd, &resp); close(fd); - if (out_body) *out_body = resp; - else free(resp); + if (out_body) + *out_body = resp; + else + free(resp); return status; } @@ -356,18 +377,17 @@ typedef struct { int rc; } srv_ctx; -static void *srv_thread_main(void *arg) -{ +static void *srv_thread_main(void *arg) { srv_ctx *s = (srv_ctx *)arg; s->rc = embed_run_server(&s->cfg); return NULL; } -static cJSON *parse_embeddings_body(const char *body, int want_items) -{ +static cJSON *parse_embeddings_body(const char *body, int want_items) { cJSON *root = cJSON_Parse(body); TEST_ASSERT(root != NULL); - if (!root) return NULL; + if (!root) + return NULL; cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); TEST_ASSERT(cJSON_IsArray(data)); TEST_ASSERT(cJSON_GetArraySize(data) == want_items); @@ -378,71 +398,63 @@ static cJSON *parse_embeddings_body(const char *body, int want_items) return root; } -static void test_http_embeddings(int port) -{ +static void test_http_embeddings(int port) { char *body = NULL; /* Routing. */ - TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL, - NULL) == 204); - TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL, - NULL, NULL) == 404); - TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL, - NULL) == 404); + TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL, NULL) == 204); + TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL, NULL, NULL) == 404); + TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL, NULL) == 404); /* Auth: required, and the exact key. */ - const char *ok_req = - "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\"," - "\"encoding_format\":\"float\"}"; - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL, - NULL) == 401); - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req, - NULL, NULL) == 401); + const char *ok_req = "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\"," + "\"encoding_format\":\"float\"}"; + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL, NULL) == 401); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req, NULL, NULL) == 401); /* Malformed HTTP (400) and malformed JSON (422 detail). */ - TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL, - "GARBAGE\r\n\r\n", NULL) == 400); - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, - "{nope", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL, "GARBAGE\r\n\r\n", NULL) == 400); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{nope", NULL, NULL) == 422); /* Validation: missing/unknown model, wrong endpoint family, valid but * unloaded id, bad encoding enum, dimensions out of range; the * API answers validation errors with 422 detail arrays. */ - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, - "{\"input\":\"hi\"}", NULL, NULL) == 422); - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, - "{\"model\":\"foo\",\"input\":\"hi\"}", NULL, + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"input\":\"hi\"}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"foo\",\"input\":\"hi\"}", NULL, NULL) == 422); TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-0.6b\"," - "\"input\":\"hi\"}", NULL, NULL) == 422); + "\"input\":\"hi\"}", + NULL, NULL) == 422); TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, - "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}", - NULL, NULL) == 503); + "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}", NULL, NULL) == 503); TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\"," - "\"encoding_format\":\"yaml\"}", NULL, NULL) == 422); + "\"encoding_format\":\"yaml\"}", + NULL, NULL) == 422); TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\"," - "\"dimensions\":64}", NULL, NULL) == 422); + "\"dimensions\":64}", + NULL, NULL) == 422); TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\"," - "\"input\":[]}", NULL, NULL) == 422); + "\"input\":[]}", + NULL, NULL) == 422); /* Contextual endpoint: standard id is the wrong enum, the 4B contextual * id is valid but not loaded. */ - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\"," - "\"input\":[[\"a\"]]}", NULL, NULL) == 422); - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + "\"input\":[[\"a\"]]}", + NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-4b\"," - "\"input\":[[\"a\"]]}", NULL, NULL) == 503); + "\"input\":[[\"a\"]]}", + NULL, NULL) == 503); /* Happy path, float encoding, single string input. */ - TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req, - NULL, &body) == 200); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req, NULL, &body) == 200); cJSON *root = body ? parse_embeddings_body(body, 1) : NULL; if (root) { cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); @@ -456,12 +468,12 @@ static void test_http_embeddings(int port) cJSON *v; cJSON_ArrayForEach(v, emb) { TEST_ASSERT(cJSON_IsNumber(v)); - if (v->valuedouble != 0.0) nonzero = 1; + if (v->valuedouble != 0.0) + nonzero = 1; } TEST_ASSERT(nonzero); cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage"); - cJSON *total = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(total && cJSON_IsNumber(total) && total->valueint >= 1); cJSON_Delete(root); } @@ -478,19 +490,15 @@ static void test_http_embeddings(int port) root = body ? parse_embeddings_body(body, 3) : NULL; if (root) { cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); - cJSON *e0 = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(data, 0), "embedding"); - cJSON *e1 = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(data, 1), "embedding"); + cJSON *e0 = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, 0), "embedding"); + cJSON *e1 = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, 1), "embedding"); for (int i = 0; i < 3; i++) { - cJSON *idx = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(data, i), "index"); + cJSON *idx = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(data, i), "index"); TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == i); } int differ = 0; for (int i = 0; i < 1024 && !differ; i++) { - if (cJSON_GetArrayItem(e0, i)->valuedouble != - cJSON_GetArrayItem(e1, i)->valuedouble) + if (cJSON_GetArrayItem(e0, i)->valuedouble != cJSON_GetArrayItem(e1, i)->valuedouble) differ = 1; } TEST_ASSERT(differ); @@ -502,12 +510,12 @@ static void test_http_embeddings(int port) /* Default encoding is base64_int8: a string of 1024 quantized bytes. */ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\"," - "\"input\":\"hello\"}", NULL, &body) == 200); + "\"input\":\"hello\"}", + NULL, &body) == 200); root = body ? parse_embeddings_body(body, 1) : NULL; if (root) { cJSON *emb = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), - 0), "embedding"); + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), 0), "embedding"); TEST_ASSERT(cJSON_IsString(emb)); TEST_ASSERT(strlen(emb->valuestring) == ((1024 + 2) / 3) * 4); cJSON_Delete(root); @@ -524,8 +532,7 @@ static void test_http_embeddings(int port) root = body ? parse_embeddings_body(body, 1) : NULL; if (root) { cJSON *emb = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), - 0), "embedding"); + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), 0), "embedding"); TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 128); cJSON_Delete(root); } @@ -548,28 +555,24 @@ static void test_http_embeddings(int port) NULL, &body) == 200); cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL; if (qwen_full && qwen_mrl) { - cJSON *full_data = - cJSON_GetObjectItemCaseSensitive(qwen_full, "data"); - cJSON *mrl_data = - cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data"); - cJSON *full_emb = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(full_data, 0), "embedding"); - cJSON *mrl_emb = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(mrl_data, 0), "embedding"); - TEST_ASSERT(cJSON_IsArray(full_emb) && - cJSON_GetArraySize(full_emb) == 1024); - TEST_ASSERT(cJSON_IsArray(mrl_emb) && - cJSON_GetArraySize(mrl_emb) == 32); + cJSON *full_data = cJSON_GetObjectItemCaseSensitive(qwen_full, "data"); + cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data"); + cJSON *full_emb = + cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(full_data, 0), "embedding"); + cJSON *mrl_emb = + cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(mrl_data, 0), "embedding"); + TEST_ASSERT(cJSON_IsArray(full_emb) && cJSON_GetArraySize(full_emb) == 1024); + TEST_ASSERT(cJSON_IsArray(mrl_emb) && cJSON_GetArraySize(mrl_emb) == 32); double prefix_diff = 0.0; for (int i = 0; i < 32; i++) { double d = fabs(cJSON_GetArrayItem(full_emb, i)->valuedouble - cJSON_GetArrayItem(mrl_emb, i)->valuedouble); - if (d > prefix_diff) prefix_diff = d; + if (d > prefix_diff) + prefix_diff = d; } TEST_ASSERT(prefix_diff > 0.0); cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage"); - cJSON *total = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(total && total->valueint == 2); } cJSON_Delete(qwen_full); @@ -585,11 +588,12 @@ static void test_http_embeddings(int port) fds[i] = tcp_connect(port); TEST_ASSERT(fds[i] >= 0); if (fds[i] >= 0) - TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings", - TEST_API_KEY, ok_req, NULL) == 0); + TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings", TEST_API_KEY, ok_req, NULL) == + 0); } for (int i = 0; i < N_CONC; i++) { - if (fds[i] < 0) continue; + if (fds[i] < 0) + continue; char *resp = NULL; TEST_ASSERT(http_recv(fds[i], &resp) == 200); free(resp); @@ -598,8 +602,7 @@ static void test_http_embeddings(int port) } /* data[di].data[ci].embedding from a contextual response. */ -static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci) -{ +static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci) { cJSON *docs = cJSON_GetObjectItemCaseSensitive(root, "data"); cJSON *doc = cJSON_GetArrayItem(docs, di); cJSON *chunks = cJSON_GetObjectItemCaseSensitive(doc, "data"); @@ -607,40 +610,40 @@ static cJSON *ctx_chunk_emb(cJSON *root, int di, int ci) return cJSON_GetObjectItemCaseSensitive(item, "embedding"); } -static double emb_max_absdiff(cJSON *a, cJSON *b) -{ - if (!cJSON_IsArray(a) || !cJSON_IsArray(b)) return 1e9; +static double emb_max_absdiff(cJSON *a, cJSON *b) { + if (!cJSON_IsArray(a) || !cJSON_IsArray(b)) + return 1e9; int n = cJSON_GetArraySize(a); - if (n != cJSON_GetArraySize(b)) return 1e9; + if (n != cJSON_GetArraySize(b)) + return 1e9; double m = 0.0; for (int i = 0; i < n; i++) { - double d = fabs(cJSON_GetArrayItem(a, i)->valuedouble - - cJSON_GetArrayItem(b, i)->valuedouble); - if (d > m) m = d; + double d = + fabs(cJSON_GetArrayItem(a, i)->valuedouble - cJSON_GetArrayItem(b, i)->valuedouble); + if (d > m) + m = d; } return m; } -static void test_http_contextual(int port) -{ +static void test_http_contextual(int port) { char *body = NULL; /* Input must be an array of chunk arrays; chunks must be non-empty. */ - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-0.6b\"," - "\"input\":\"hi\"}", NULL, NULL) == 422); - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + "\"input\":\"hi\"}", + NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-0.6b\"," - "\"input\":[[\"\"]]}", NULL, NULL) == 422); + "\"input\":[[\"\"]]}", + NULL, NULL) == 422); /* Happy path: chunks concatenate per document with one separator token * between them, one embedding per chunk in document order. Fixture * token counts: hello=1, world=3, held=2, one separator in doc 0, so * usage must report 7. */ - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-0.6b\"," "\"input\":[[\"hello\",\"world\"],[\"held\"]]," "\"encoding_format\":\"float\"}", @@ -664,21 +667,18 @@ static void test_http_contextual(int port) cJSON *cidx = cJSON_GetObjectItemCaseSensitive(item, "index"); cJSON *emb = ctx_chunk_emb(multi, di, ci); TEST_ASSERT(cidx && cidx->valueint == ci); - TEST_ASSERT(cJSON_IsArray(emb) && - cJSON_GetArraySize(emb) == 1024); + TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 1024); } } cJSON *usage = cJSON_GetObjectItemCaseSensitive(multi, "usage"); - cJSON *total = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(total && total->valueint == 7); } /* A single-chunk document has no separators and its pooling span covers * the whole sequence, so it must match the standard embedding of the * same text from the same weights. */ - TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", - TEST_API_KEY, + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-context-v1-0.6b\"," "\"input\":[[\"hello\"]]," "\"encoding_format\":\"float\"}", @@ -699,8 +699,7 @@ static void test_http_contextual(int port) if (multi && single && std) { cJSON *std_emb = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(std, "data"), - 0), "embedding"); + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(std, "data"), 0), "embedding"); cJSON *alone = ctx_chunk_emb(single, 0, 0); cJSON *in_doc = ctx_chunk_emb(multi, 0, 0); TEST_ASSERT(emb_max_absdiff(std_emb, alone) < 1e-4); @@ -713,8 +712,7 @@ static void test_http_contextual(int port) cJSON_Delete(std); } -static void test_http_rerank(int port) -{ +static void test_http_rerank(int port) { char *body = NULL; TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY, @@ -735,55 +733,45 @@ static void test_http_rerank(int port) "\"top_n\":1,\"top_k\":1}", NULL, NULL) == 422); - const char *request = - "{\"model\":\"pplx-embed-v1-late-0.6b\"," - "\"query\":\"hello\",\"documents\":[" - "\"hello!\",\"world\",\"quoted \\\"document\\\"\"]," - "\"top_n\":2,\"return_documents\":true}"; - TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY, request, - NULL, &body) == 200); + const char *request = "{\"model\":\"pplx-embed-v1-late-0.6b\"," + "\"query\":\"hello\",\"documents\":[" + "\"hello!\",\"world\",\"quoted \\\"document\\\"\"]," + "\"top_n\":2,\"return_documents\":true}"; + TEST_ASSERT(http_req(port, "POST", "/v1/rerank", TEST_API_KEY, request, NULL, &body) == 200); cJSON *root = body ? cJSON_Parse(body) : NULL; TEST_ASSERT(root != NULL); if (root) { cJSON *object = cJSON_GetObjectItemCaseSensitive(root, "object"); cJSON *model = cJSON_GetObjectItemCaseSensitive(root, "model"); cJSON *results = cJSON_GetObjectItemCaseSensitive(root, "results"); - TEST_ASSERT(cJSON_IsString(object) && - strcmp(object->valuestring, "list") == 0); + TEST_ASSERT(cJSON_IsString(object) && strcmp(object->valuestring, "list") == 0); TEST_ASSERT(cJSON_IsString(model) && - strcmp(model->valuestring, - "pplx-embed-v1-late-0.6b") == 0); - TEST_ASSERT(cJSON_IsArray(results) && - cJSON_GetArraySize(results) == 2); + strcmp(model->valuestring, "pplx-embed-v1-late-0.6b") == 0); + TEST_ASSERT(cJSON_IsArray(results) && cJSON_GetArraySize(results) == 2); double previous = INFINITY; int seen[3] = {0}; cJSON *result; cJSON_ArrayForEach(result, results) { - cJSON *index = - cJSON_GetObjectItemCaseSensitive(result, "index"); - cJSON *score = - cJSON_GetObjectItemCaseSensitive(result, "relevance_score"); - cJSON *document = - cJSON_GetObjectItemCaseSensitive(result, "document"); - TEST_ASSERT(cJSON_IsNumber(index) && - index->valueint >= 0 && index->valueint < 3); - TEST_ASSERT(cJSON_IsNumber(score) && - score->valuedouble <= previous); + cJSON *index = cJSON_GetObjectItemCaseSensitive(result, "index"); + cJSON *score = cJSON_GetObjectItemCaseSensitive(result, "relevance_score"); + cJSON *document = cJSON_GetObjectItemCaseSensitive(result, "document"); + TEST_ASSERT(cJSON_IsNumber(index) && index->valueint >= 0 && index->valueint < 3); + TEST_ASSERT(cJSON_IsNumber(score) && score->valuedouble <= previous); TEST_ASSERT(cJSON_IsString(document)); - if (cJSON_IsNumber(index) && index->valueint >= 0 && - index->valueint < 3) + if (cJSON_IsNumber(index) && index->valueint >= 0 && index->valueint < 3) seen[index->valueint]++; - if (cJSON_IsNumber(score)) previous = score->valuedouble; + if (cJSON_IsNumber(score)) + previous = score->valuedouble; } TEST_ASSERT(seen[0] + seen[1] + seen[2] == 2); cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage"); - cJSON *query_tokens = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "query_tokens") : NULL; - cJSON *document_tokens = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL; - cJSON *total_tokens = usage - ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + cJSON *query_tokens = + usage ? cJSON_GetObjectItemCaseSensitive(usage, "query_tokens") : NULL; + cJSON *document_tokens = + usage ? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL; + cJSON *total_tokens = + usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(query_tokens && document_tokens && total_tokens); if (query_tokens && document_tokens && total_tokens) { TEST_ASSERT(cJSON_IsNumber(query_tokens)); @@ -808,18 +796,16 @@ static void test_http_rerank(int port) TEST_ASSERT(root != NULL); if (root) { cJSON *results = cJSON_GetObjectItemCaseSensitive(root, "results"); - TEST_ASSERT(cJSON_IsArray(results) && - cJSON_GetArraySize(results) == 1); - cJSON *document = cJSON_GetObjectItemCaseSensitive( - cJSON_GetArrayItem(results, 0), "document"); + TEST_ASSERT(cJSON_IsArray(results) && cJSON_GetArraySize(results) == 1); + cJSON *document = + cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(results, 0), "document"); TEST_ASSERT(document == NULL); cJSON_Delete(root); } free(body); } -static void test_http_server(void) -{ +static void test_http_server(void) { char dir[1024], late_dir[1024], qwen_dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); @@ -845,14 +831,11 @@ static void test_http_server(void) /* The 0.6b server slot requires hidden_size 1024; everything else stays * tiny. The model vocab covers every tokenizer fixture id. */ tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE}; - if (tf_write_vocab(dir) != 0 || - tm_write_model_dims(dir, "F32", &dims) != 0 || - tf_write_vocab(late_dir) != 0 || - tm_write_model_dims(late_dir, "F32", &dims) != 0 || + if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 || + tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 || tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 || tf_write_vocab(qwen_dir) != 0 || - tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, - TF_EOT_ID) != 0) { + tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0) { fprintf(stderr, "FAIL: fixture write\n"); test_failures++; return; @@ -888,9 +871,13 @@ static void test_http_server(void) } int up = 0; - for (int i = 0; i < 200; i++) { /* up to ~10 s */ + for (int i = 0; i < 200; i++) { /* up to ~10 s */ int fd = tcp_connect(port); - if (fd >= 0) { close(fd); up = 1; break; } + if (fd >= 0) { + close(fd); + up = 1; + break; + } usleep(50 * 1000); } TEST_ASSERT(up); @@ -905,8 +892,7 @@ static void test_http_server(void) TEST_ASSERT(ctx.rc == 0); } -int main(void) -{ +int main(void) { test_base64_rfc4648(); test_quantize_int8_tanh(); test_encode_embedding_int8(); diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index bed5b7d..5920977 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -16,31 +16,33 @@ static int failures = 0; int qwen_verbose = 0; -#define TEST_ASSERT(cond) do { \ - if (!(cond)) { \ - fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - failures++; \ - } \ -} while (0) +#define TEST_ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + failures++; \ + } \ + } while (0) static void check_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *text, - const int *want, int n_want, const char *label) -{ + qwen_tokenizer_workspace_t *ws, + const char *text, + const int *want, + int n_want, + const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL); - if (!ids) return; + if (!ids) + return; if (n != n_want) { - fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", - label, text, n, n_want); + fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", label, text, n, n_want); failures++; } else { for (int i = 0; i < n; i++) { if (ids[i] != want[i]) { - fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", - label, text, i, ids[i], want[i]); + fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", label, text, i, ids[i], + want[i]); failures++; } } @@ -51,25 +53,30 @@ static void check_ids(const qwen_tokenizer_t *tok, /* Concatenated decode of an encoding must reproduce the (NFC-normalized) * input - byte-level BPE is lossless. */ static void check_roundtrip(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *text, - const char *want, const char *label) -{ + qwen_tokenizer_workspace_t *ws, + const char *text, + const char *want, + const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL && n > 0); - if (!ids) return; + if (!ids) + return; size_t cap = 1; for (int i = 0; i < n; i++) cap += strlen(qwen_tokenizer_decode(tok, ids[i])); char *got = (char *)malloc(cap); - if (!got) { free(ids); TEST_ASSERT(0); return; } + if (!got) { + free(ids); + TEST_ASSERT(0); + return; + } got[0] = '\0'; for (int i = 0; i < n; i++) strcat(got, qwen_tokenizer_decode(tok, ids[i])); if (strcmp(got, want) != 0) { - fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", - label, text, got, want); + fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got, + want); failures++; } free(got); @@ -79,15 +86,21 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, /* Two spellings of the same text must encode identically. */ static void check_same_ids(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, - const char *a, const char *b, const char *label) -{ + const char *a, + const char *b, + const char *label) { int na = 0, nb = 0; - int *ia = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, a, &na); - if (!ia) { TEST_ASSERT(0); return; } - int *ib = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, b, &nb); - if (!ib) { free(ia); TEST_ASSERT(0); return; } + int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na); + if (!ia) { + TEST_ASSERT(0); + return; + } + int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb); + if (!ib) { + free(ia); + TEST_ASSERT(0); + return; + } if (na != nb || memcmp(ia, ib, (size_t)na * sizeof(int)) != 0) { fprintf(stderr, "FAIL %s same-ids \"%s\" vs \"%s\"\n", label, a, b); failures++; @@ -96,9 +109,8 @@ static void check_same_ids(const qwen_tokenizer_t *tok, free(ia); } -static void run_cases(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *label) -{ +static void +run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { /* h e l l o -> he -> he,ll -> hell -> hello */ const int hello[] = {259}; check_ids(tok, ws, "hello", hello, 1, label); @@ -147,14 +159,14 @@ static void run_cases(const qwen_tokenizer_t *tok, /* NFC: decomposed base+combining-mark spellings must match the * precomposed codepoint (one pair per mark in the composition table). */ - check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */ - check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */ - check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */ - check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */ - check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */ - check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */ - check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */ - check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */ + check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */ + check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */ + check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */ + check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */ + check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */ + check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */ + check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */ + check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */ check_roundtrip(tok, ws, "cafe\xCC\x81", "caf\xC3\xA9", label); /* 'x' + acute has no composition: bytes pass through untouched. */ check_roundtrip(tok, ws, "x\xCC\x81", "x\xCC\x81", label); @@ -177,30 +189,37 @@ static void run_cases(const qwen_tokenizer_t *tok, check_roundtrip(tok, ws, "\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", label); check_roundtrip(tok, ws, "tab\tand\nnl 42!", "tab\tand\nnl 42!", label); const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'}; - check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label); + check_ids(tok, ws, + "a\xE2\x80\xA6" + "b", + ellipsis, 5, label); } -static void write_file(const char *dir, const char *name, const char *body) -{ +static void write_file(const char *dir, const char *name, const char *body) { char path[2048]; snprintf(path, sizeof(path), "%s/%s", dir, name); FILE *f = fopen(path, "w"); - if (!f) { perror(path); exit(2); } + if (!f) { + perror(path); + exit(2); + } fputs(body, f); fclose(f); } /* Loader rejection paths and vocab.json escape handling, in throwaway * subdirectories of the main fixture dir. */ -static void test_loader_edges(const char *root) -{ +static void test_loader_edges(const char *root) { char dir[1152], path[2048]; TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); /* Not a JSON object. */ snprintf(dir, sizeof(dir), "%s/badjson", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "[1, 2]\n"); write_file(dir, "merges.txt", "a b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); @@ -209,7 +228,10 @@ static void test_loader_edges(const char *root) /* Missing or pair-less merges.txt is tolerated: the tokenizer loads * and encoding falls back to byte/vocab-level (no merges applied). */ snprintf(dir, sizeof(dir), "%s/nomerges", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); qwen_tokenizer_t *nom = qwen_tokenizer_load(path); @@ -225,7 +247,10 @@ static void test_loader_edges(const char *root) } snprintf(dir, sizeof(dir), "%s/emptymerges", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); @@ -235,7 +260,10 @@ static void test_loader_edges(const char *root) /* Escape forms in vocab keys; a malformed merge line is skipped. */ snprintf(dir, sizeof(dir), "%s/escapes", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0, \"b\":1, \"ab\":2,\n" " \"\\u20AC\":3, \"x\\ny\":4, \"q\\\"r\":5,\n" @@ -264,12 +292,14 @@ static void test_loader_edges(const char *root) qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); TEST_ASSERT(rel != NULL); qwen_tokenizer_free(rel); - if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); } + if (chdir(saved_cwd) != 0) { + perror("chdir back"); + exit(2); + } } } -int main(void) -{ +int main(void) { char dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-tok-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); @@ -282,7 +312,8 @@ int main(void) snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); TEST_ASSERT(tok != NULL); - if (!tok) return 1; + if (!tok) + return 1; /* The complete byte alphabet means the integer-pair tables must exist. */ TEST_ASSERT(tok->int_merges != NULL); @@ -312,21 +343,17 @@ int main(void) { int out[8]; int n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, - &n) == 0 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); /* On overflow the prefix is written and n reports the total * required count, so a caller can re-size and retry. */ n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, - &n) == -2 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, - &n) == -2 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && - n == 0); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); } diff --git a/tests/test_workspace.c b/tests/test_workspace.c index 4f83936..a906689 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -14,12 +14,12 @@ #include #include -static float max_abs_diff(const float *a, const float *b, int n) -{ +static float max_abs_diff(const float *a, const float *b, int n) { float m = 0.0f; for (int i = 0; i < n; i++) { float d = fabsf(a[i] - b[i]); - if (d > m) m = d; + if (d > m) + m = d; } return m; } @@ -28,11 +28,11 @@ static float max_abs_diff(const float *a, const float *b, int n) * results validated in main. reference is the batched row for ids0. */ static int check_alloc_and_pooling_variants(const embed_model_t *model, embed_workspace_t *ws, - const int *ids0, int n0, + const int *ids0, + int n0, const float *states, const float *reference, - const embed_config_t *cfg) -{ + const embed_config_t *cfg) { int rc = 1; int dim = cfg->hidden_size; float *emb = NULL, *fwd = NULL, *pooled = NULL, *span_out = NULL; @@ -73,22 +73,19 @@ static int check_alloc_and_pooling_variants(const embed_model_t *model, /* One span covering the whole sequence pools to the embedding, and * two halves recombine into it by token-count weighting. */ embed_span_t whole = {0, n0}; - if (embed_model_encode_spans(model, ws, ids0, n0, &whole, 1, - span_out) != 0 || + if (embed_model_encode_spans(model, ws, ids0, n0, &whole, 1, span_out) != 0 || max_abs_diff(span_out, reference, dim) > 0.00005f) { fprintf(stderr, "whole-sequence span disagrees with embedding\n"); goto done; } int h = n0 / 2; embed_span_t halves[2] = {{0, h}, {h, n0 - h}}; - if (embed_model_encode_spans(model, ws, ids0, n0, halves, 2, - span_out) != 0) { + if (embed_model_encode_spans(model, ws, ids0, n0, halves, 2, span_out) != 0) { fprintf(stderr, "embed_model_encode_spans failed for two spans\n"); goto done; } for (int d = 0; d < dim; d++) { - float combined = ((float)h * span_out[d] + - (float)(n0 - h) * span_out[dim + d]) / (float)n0; + float combined = ((float)h * span_out[d] + (float)(n0 - h) * span_out[dim + d]) / (float)n0; if (fabsf(combined - reference[d]) > 0.00005f) { fprintf(stderr, "half spans do not recombine into embedding\n"); goto done; @@ -108,10 +105,10 @@ done: * doc0 is chunks 0 and 1 joined by a separator token; doc1 is chunk 2. */ static int check_spans_batch_parity(const embed_model_t *model, embed_workspace_t *ws, - int *const ids[], const int *ntok, + int *const ids[], + const int *ntok, int separator_id, - const embed_config_t *cfg) -{ + const embed_config_t *cfg) { int rc = 1; int dim = cfg->hidden_size; int n0 = ntok[0] + 1 + ntok[1]; @@ -128,10 +125,9 @@ static int check_spans_batch_parity(const embed_model_t *model, embed_span_t spans0[2] = {{0, ntok[0]}, {ntok[0] + 1, ntok[1]}}; embed_span_t span1 = {0, ntok[2]}; - if (embed_model_encode_spans(model, ws, doc0, n0, spans0, 2, - expected) != 0 || + if (embed_model_encode_spans(model, ws, doc0, n0, spans0, 2, expected) != 0 || embed_model_encode_spans(model, ws, ids[2], ntok[2], &span1, 1, - expected + (size_t)2 * dim) != 0) { + expected + (size_t)2 * dim) != 0) { fprintf(stderr, "singleton contextual embedding failed\n"); goto done; } @@ -164,8 +160,7 @@ done: return rc; } -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { char fixture_dir[1024]; const char *model_dir; if (argc == 2) { @@ -214,12 +209,10 @@ int main(int argc, char **argv) return 1; } - const char *texts[] = { - "query: what is the capital of France?", - "document: Paris is the capital of France.", - "document: Berlin is the capital of Germany.", - "document: Istanbul is a major city in Turkey." - }; + const char *texts[] = {"query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey."}; const int batch = (int)(sizeof(texts) / sizeof(texts[0])); /* Initialized before the first `goto fail` so the cleanup loop never * sees indeterminate pointers. */ @@ -291,8 +284,10 @@ int main(int argc, char **argv) const float *row = batched + (size_t)i * dim; float diff = max_abs_diff(row, single, dim); float cos = embed_cosine_similarity(row, single, dim); - if (diff > worst_diff) worst_diff = diff; - if (cos < worst_cos) worst_cos = cos; + if (diff > worst_diff) + worst_diff = diff; + if (cos < worst_cos) + worst_cos = cos; } memcpy(normalized, batched, (size_t)dim * sizeof(float)); @@ -308,8 +303,7 @@ int main(int argc, char **argv) for (int i = 0; i < dim; i++) normalized_norm_sq += normalized[i] * normalized[i]; if (fabsf(normalized_norm_sq - 1.0f) > 0.00005f) { - fprintf(stderr, "embed_l2_normalize produced norm_sq=%g\n", - normalized_norm_sq); + fprintf(stderr, "embed_l2_normalize produced norm_sq=%g\n", normalized_norm_sq); free(states); free(normalized); free(single); @@ -326,8 +320,8 @@ int main(int argc, char **argv) goto fail; } - if (check_alloc_and_pooling_variants(model, batch_ws, ids[0], ntok[0], - states, batched, cfg) != 0) { + if (check_alloc_and_pooling_variants(model, batch_ws, ids[0], ntok[0], states, batched, cfg) != + 0) { free(states); free(normalized); free(single); @@ -338,9 +332,9 @@ int main(int argc, char **argv) /* Same separator resolution as the server: the fixture vocab defines * <|endoftext|>, real snapshots use the reserved id. */ int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>"); - if (sep_id < 0) sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID; - if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, - cfg) != 0) { + if (sep_id < 0) + sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) { free(states); free(normalized); free(single); @@ -353,24 +347,26 @@ int main(int argc, char **argv) free(single); free(batched); - for (int i = 0; i < batch; i++) free(ids[i]); + for (int i = 0; i < batch; i++) + free(ids[i]); qwen_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); if (worst_diff > 0.00005f || worst_cos < 0.99999f) { - fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n", - worst_diff, worst_cos); + fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n", worst_diff, + worst_cos); return 1; } - printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n", - dim, batch, batch_ws_bytes, worst_diff, worst_cos); + printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n", dim, + batch, batch_ws_bytes, worst_diff, worst_cos); return 0; fail: - for (int i = 0; i < batch; i++) free(ids[i]); + for (int i = 0; i < batch; i++) + free(ids[i]); qwen_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); diff --git a/tests/tiny_model.h b/tests/tiny_model.h index d4ae431..42542c1 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -14,15 +14,13 @@ #include #include -static uint16_t tm_f32_to_bf16(float x) -{ +static uint16_t tm_f32_to_bf16(float x) { uint32_t u; memcpy(&u, &x, sizeof(u)); return (uint16_t)(u >> 16); } -static float tm_bf16_to_f32(uint16_t b) -{ +static float tm_bf16_to_f32(uint16_t b) { uint32_t u = ((uint32_t)b) << 16; float f; memcpy(&f, &u, sizeof(f)); @@ -30,8 +28,7 @@ static float tm_bf16_to_f32(uint16_t b) } /* Deterministic, BF16-exact value for element i of tensor `name`. */ -static float tm_value(const char *name, size_t i) -{ +static float tm_value(const char *name, size_t i) { float v; size_t len = strlen(name); if ((len >= 11 && strcmp(name + len - 11, "norm.weight") == 0) || @@ -46,7 +43,7 @@ static float tm_value(const char *name, size_t i) typedef struct { const char *name; - int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */ + int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */ } tm_spec_t; /* Always one layer; everything else is free so a test can match a real @@ -57,22 +54,21 @@ typedef struct { enum { TM_N_SPECS = 13 }; -static int tm_write_config(const char *dir, const tm_dims_t *d, - const char *model_type, int eos_token_id) -{ +static int +tm_write_config(const char *dir, const tm_dims_t *d, const char *model_type, int eos_token_id) { char path[2048]; snprintf(path, sizeof(path), "%s/config.json", dir); FILE *f = fopen(path, "w"); - if (!f) return -1; - fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," + if (!f) + return -1; + fprintf(f, + "{\"hidden_size\":%d,\"num_hidden_layers\":1," "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0", - d->hidden, d->heads, d->kv_heads, d->head_dim, - d->intermediate, d->vocab); + d->hidden, d->heads, d->kv_heads, d->head_dim, d->intermediate, d->vocab); if (model_type) - fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d", - model_type, eos_token_id); + fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d", model_type, eos_token_id); fputs("}", f); fclose(f); return 0; @@ -80,9 +76,8 @@ static int tm_write_config(const char *dir, const tm_dims_t *d, /* Write a safetensors file holding `specs` with tm_value() contents. * dtype: "F32" or "BF16". Returns 0 on success. */ -static int tm_write_safetensors(const char *path, const char *dtype, - const tm_spec_t *specs, int n_specs) -{ +static int +tm_write_safetensors(const char *path, const char *dtype, const tm_spec_t *specs, int n_specs) { int bf16 = strcmp(dtype, "BF16") == 0; size_t esize = bf16 ? 2 : 4; @@ -95,23 +90,26 @@ static int tm_write_safetensors(const char *path, const char *dtype, size_t end = doff + n * esize; if (s->cols) hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, - "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d,%d]," - "\"data_offsets\":[%zu,%zu]}", - t ? "," : "", s->name, dtype, s->rows, s->cols, doff, end); + "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d,%d]," + "\"data_offsets\":[%zu,%zu]}", + t ? "," : "", s->name, dtype, s->rows, s->cols, doff, end); else hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, - "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d]," - "\"data_offsets\":[%zu,%zu]}", - t ? "," : "", s->name, dtype, s->rows, doff, end); + "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d]," + "\"data_offsets\":[%zu,%zu]}", + t ? "," : "", s->name, dtype, s->rows, doff, end); doff = end; } hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "}"); - if (hoff >= sizeof(header) - 1) return -1; + if (hoff >= sizeof(header) - 1) + return -1; FILE *f = fopen(path, "wb"); - if (!f) return -1; + if (!f) + return -1; unsigned char len8[8]; - for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(hoff >> (8 * i)); + for (int i = 0; i < 8; i++) + len8[i] = (unsigned char)(hoff >> (8 * i)); fwrite(len8, 1, 8, f); fwrite(header, 1, hoff, f); for (int t = 0; t < n_specs; t++) { @@ -133,9 +131,7 @@ static int tm_write_safetensors(const char *path, const char *dtype, /* Write config.json + model.safetensors with the given dimensions into dir. * dtype: "F32" or "BF16". Returns 0 on success. */ -static int tm_write_model_dims(const char *dir, const char *dtype, - const tm_dims_t *d) -{ +static int tm_write_model_dims(const char *dir, const char *dtype, const tm_dims_t *d) { int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; const tm_spec_t specs[TM_N_SPECS] = { {"embed_tokens.weight", d->vocab, d->hidden}, @@ -154,37 +150,37 @@ static int tm_write_model_dims(const char *dir, const char *dtype, }; char path[2048]; - if (tm_write_config(dir, d, NULL, -1) != 0) return -1; + if (tm_write_config(dir, d, NULL, -1) != 0) + return -1; snprintf(path, sizeof(path), "%s/model.safetensors", dir); return tm_write_safetensors(path, dtype, specs, TM_N_SPECS); } -static inline int tm_write_qwen3_model_dims(const char *dir, const char *dtype, +static inline int tm_write_qwen3_model_dims(const char *dir, + const char *dtype, const tm_dims_t *d, - int eos_token_id) -{ - if (tm_write_model_dims(dir, dtype, d) != 0) return -1; + int eos_token_id) { + if (tm_write_model_dims(dir, dtype, d) != 0) + return -1; return tm_write_config(dir, d, "qwen3", eos_token_id); } /* Add the 1_Dense per-token projection head that turns a base model dir into * a late-interaction snapshot. weight_name is "linear.weight" for a valid * head; pass another name to synthesize a malformed head. */ -static inline int tm_write_late_projection_named(const char *dir, - const char *dtype, - int token_dim, int hidden, - const char *weight_name) -{ +static inline int tm_write_late_projection_named( + const char *dir, const char *dtype, int token_dim, int hidden, const char *weight_name) { char path[2048]; snprintf(path, sizeof(path), "%s/1_Dense", dir); - if (mkdir(path, 0755) != 0 && errno != EEXIST) return -1; + if (mkdir(path, 0755) != 0 && errno != EEXIST) + return -1; snprintf(path, sizeof(path), "%s/1_Dense/config.json", dir); FILE *f = fopen(path, "w"); - if (!f) return -1; - fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}", - hidden, token_dim); + if (!f) + return -1; + fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}", hidden, token_dim); fclose(f); const tm_spec_t spec = {weight_name, token_dim, hidden}; @@ -192,16 +188,13 @@ static inline int tm_write_late_projection_named(const char *dir, return tm_write_safetensors(path, dtype, &spec, 1); } -static inline int tm_write_late_projection(const char *dir, const char *dtype, - int token_dim, int hidden) -{ - return tm_write_late_projection_named(dir, dtype, token_dim, hidden, - "linear.weight"); +static inline int +tm_write_late_projection(const char *dir, const char *dtype, int token_dim, int hidden) { + return tm_write_late_projection_named(dir, dtype, token_dim, hidden, "linear.weight"); } /* The historical default fixture: 4-dim hidden, 16-token vocab. */ -static inline int tm_write_model(const char *dir, const char *dtype) -{ +static inline int tm_write_model(const char *dir, const char *dtype) { static const tm_dims_t d = {4, 2, 1, 2, 8, 16}; return tm_write_model_dims(dir, dtype, &d); } diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h index 7890927..24e1563 100644 --- a/tests/tok_fixture.h +++ b/tests/tok_fixture.h @@ -10,15 +10,13 @@ #include /* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */ -static int tf_byte_to_cp(int b) -{ +static int tf_byte_to_cp(int b) { static int table[256]; static int init = 0; if (!init) { int n = 0; for (int i = 0; i < 256; i++) { - int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || - (i >= 174 && i <= 255); + int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || (i >= 174 && i <= 255); table[i] = normal ? i : 256 + n++; } init = 1; @@ -26,9 +24,12 @@ static int tf_byte_to_cp(int b) return table[b]; } -static int tf_cp_to_utf8(int cp, char *out) -{ - if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; } +static int tf_cp_to_utf8(int cp, char *out) { + if (cp < 0x80) { + out[0] = (char)cp; + out[1] = '\0'; + return 1; + } out[0] = (char)(0xC0 | (cp >> 6)); out[1] = (char)(0x80 | (cp & 0x3F)); out[2] = '\0'; @@ -36,11 +37,11 @@ static int tf_cp_to_utf8(int cp, char *out) } /* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */ -static const char *tf_json_escape(const char *s, char *buf, size_t cap) -{ +static const char *tf_json_escape(const char *s, char *buf, size_t cap) { size_t o = 0; for (; *s && o + 2 < cap; s++) { - if (*s == '"' || *s == '\\') buf[o++] = '\\'; + if (*s == '"' || *s == '\\') + buf[o++] = '\\'; buf[o++] = *s; } buf[o] = '\0'; @@ -52,8 +53,7 @@ static const char *TF_MERGED[] = { "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor", }; static const char *TF_MERGES[] = { - "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", - "\xC4\xA0w or", + "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", "\xC4\xA0w or", }; enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) }; /* Special tokens go last so contextual and late-interaction ids resolve @@ -65,17 +65,16 @@ enum { TF_DOCUMENT_ID = TF_QUERY_ID + 1 }; enum { TF_VOCAB_SIZE = TF_DOCUMENT_ID + 1 }; /* Write vocab.json + merges.txt into dir. Returns 0 on success. */ -static int tf_write_vocab(const char *dir) -{ +static int tf_write_vocab(const char *dir) { char path[2048], tok[4], esc[16]; snprintf(path, sizeof(path), "%s/vocab.json", dir); FILE *f = fopen(path, "w"); - if (!f) return -1; + if (!f) + return -1; fputs("{", f); for (int b = 0; b < 256; b++) { tf_cp_to_utf8(tf_byte_to_cp(b), tok); - fprintf(f, "%s\"%s\": %d", b ? ", " : "", - tf_json_escape(tok, esc, sizeof(esc)), b); + fprintf(f, "%s\"%s\": %d", b ? ", " : "", tf_json_escape(tok, esc, sizeof(esc)), b); } for (int i = 0; i < TF_N_MERGED; i++) fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i); @@ -88,7 +87,8 @@ static int tf_write_vocab(const char *dir) snprintf(path, sizeof(path), "%s/merges.txt", dir); f = fopen(path, "w"); - if (!f) return -1; + if (!f) + return -1; for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++) fprintf(f, "%s\n", TF_MERGES[i]); fclose(f); -- cgit v1.3.1