From ec417bcb52f2aa8e24aeec312e8ddbf3a0798056 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 08:02:03 +0200 Subject: kernels: drop asr_ prefix naming --- Makefile | 38 +- README.md | 2 +- main.c | 4 +- pplx_embed.c | 4 +- pplx_embed_internal.h | 2 +- pplx_embed_mlx.c | 2 +- pplx_server.c | 4 +- qwen_asr_kernels.c | 1745 -------------------------------------------- qwen_asr_kernels.h | 210 ------ qwen_asr_kernels_avx.c | 502 ------------- qwen_asr_kernels_generic.c | 65 -- qwen_asr_kernels_impl.h | 65 -- qwen_asr_kernels_neon.c | 274 ------- qwen_asr_safetensors.c | 481 ------------ qwen_asr_safetensors.h | 79 -- qwen_asr_tokenizer.c | 1523 -------------------------------------- qwen_asr_tokenizer.h | 61 -- qwen_kernels.c | 1745 ++++++++++++++++++++++++++++++++++++++++++++ qwen_kernels.h | 210 ++++++ qwen_kernels_avx.c | 502 +++++++++++++ qwen_kernels_generic.c | 65 ++ qwen_kernels_impl.h | 65 ++ qwen_kernels_neon.c | 274 +++++++ qwen_safetensors.c | 481 ++++++++++++ qwen_safetensors.h | 79 ++ qwen_tokenizer.c | 1523 ++++++++++++++++++++++++++++++++++++++ qwen_tokenizer.h | 61 ++ server_main.c | 2 +- tests/test_kernels.c | 2 +- 29 files changed, 5035 insertions(+), 5035 deletions(-) delete mode 100644 qwen_asr_kernels.c delete mode 100644 qwen_asr_kernels.h delete mode 100644 qwen_asr_kernels_avx.c delete mode 100644 qwen_asr_kernels_generic.c delete mode 100644 qwen_asr_kernels_impl.h delete mode 100644 qwen_asr_kernels_neon.c delete mode 100644 qwen_asr_safetensors.c delete mode 100644 qwen_asr_safetensors.h delete mode 100644 qwen_asr_tokenizer.c delete mode 100644 qwen_asr_tokenizer.h create mode 100644 qwen_kernels.c create mode 100644 qwen_kernels.h create mode 100644 qwen_kernels_avx.c create mode 100644 qwen_kernels_generic.c create mode 100644 qwen_kernels_impl.h create mode 100644 qwen_kernels_neon.c create mode 100644 qwen_safetensors.c create mode 100644 qwen_safetensors.h create mode 100644 qwen_tokenizer.c create mode 100644 qwen_tokenizer.h diff --git a/Makefile b/Makefile index 2451926..890fc7b 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,12 @@ UNAME_S := $(shell uname -s) SRCS = pplx_embed.c \ pplx_distributed.c \ pplx_server.c \ - qwen_asr_kernels.c \ - qwen_asr_kernels_generic.c \ - qwen_asr_kernels_neon.c \ - qwen_asr_kernels_avx.c \ - qwen_asr_tokenizer.c \ - qwen_asr_safetensors.c \ + qwen_kernels.c \ + qwen_kernels_generic.c \ + qwen_kernels_neon.c \ + qwen_kernels_avx.c \ + qwen_tokenizer.c \ + qwen_safetensors.c \ deps/ae/ae.c \ deps/ae/anet.c \ deps/ae/monotonic.c @@ -38,8 +38,8 @@ TARGET = pplx_embed SERVER_TARGET = pplx-embed-server LIB = libpplxembed.a -KERNEL_SRCS = qwen_asr_kernels.c qwen_asr_kernels_generic.c \ - qwen_asr_kernels_neon.c qwen_asr_kernels_avx.c +KERNEL_SRCS = qwen_kernels.c qwen_kernels_generic.c \ + qwen_kernels_neon.c qwen_kernels_avx.c ifeq ($(shell uname -s),Darwin) SHARED_LIB = libpplxembed.dylib @@ -192,40 +192,40 @@ $(SERVER_TARGET): $(LIB) server_main.o # ============================================================================= # Compile rules # ============================================================================= -pplx_embed.o: pplx_embed.c pplx_embed.h qwen_asr_kernels.h qwen_asr_safetensors.h +pplx_embed.o: pplx_embed.c pplx_embed.h qwen_kernels.h qwen_safetensors.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< pplx_distributed.o: pplx_distributed.c pplx_distributed.h pplx_embed.h pplx_embed_mlx.h $(CC) $(CFLAGS) -c -o $@ $< -pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_asr_tokenizer.h deps/ae/ae.h deps/ae/anet.h +pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h +pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -main.o: main.c pplx_embed.h pplx_distributed.h qwen_asr_kernels.h qwen_asr_tokenizer.h +main.o: main.c pplx_embed.h pplx_distributed.h qwen_kernels.h qwen_tokenizer.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_asr_kernels.h +server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_kernels.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h +qwen_kernels.o: qwen_kernels.c qwen_kernels.h qwen_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< -qwen_asr_kernels_generic.o: qwen_asr_kernels_generic.c qwen_asr_kernels_impl.h +qwen_kernels_generic.o: qwen_kernels_generic.c qwen_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< -qwen_asr_kernels_neon.o: qwen_asr_kernels_neon.c qwen_asr_kernels_impl.h +qwen_kernels_neon.o: qwen_kernels_neon.c qwen_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< -qwen_asr_kernels_avx.o: qwen_asr_kernels_avx.c qwen_asr_kernels_impl.h +qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< -qwen_asr_tokenizer.o: qwen_asr_tokenizer.c qwen_asr_tokenizer.h qwen_asr_kernels.h +qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h $(CC) $(CFLAGS) -c -o $@ $< -qwen_asr_safetensors.o: qwen_asr_safetensors.c qwen_asr_safetensors.h +qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h diff --git a/README.md b/README.md index 03b05f8..389b9f6 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Output is a line-delimited stream of JSON objects with the embedding vector, dim ```c #include "pplx_embed.h" -#include "qwen_asr_tokenizer.h" +#include "qwen_tokenizer.h" /* Load model */ pplx_ctx_t *ctx = pplx_load("/path/to/model"); diff --git a/main.c b/main.c index ef1c91a..140494b 100644 --- a/main.c +++ b/main.c @@ -2,8 +2,8 @@ #include "pplx_embed.h" #include "pplx_distributed.h" -#include "qwen_asr_kernels.h" -#include "qwen_asr_tokenizer.h" +#include "qwen_kernels.h" +#include "qwen_tokenizer.h" #ifdef USE_MLX #include "pplx_embed_mlx.h" diff --git a/pplx_embed.c b/pplx_embed.c index 4dc358e..996be9c 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -1,7 +1,7 @@ #include "pplx_embed.h" #include "pplx_embed_internal.h" -#include "qwen_asr_kernels.h" -#include "qwen_asr_safetensors.h" +#include "qwen_kernels.h" +#include "qwen_safetensors.h" #include #include diff --git a/pplx_embed_internal.h b/pplx_embed_internal.h index fa028e9..1e7e4d0 100644 --- a/pplx_embed_internal.h +++ b/pplx_embed_internal.h @@ -2,7 +2,7 @@ #define PPLX_EMBED_INTERNAL_H #include "pplx_embed.h" -#include "qwen_asr_safetensors.h" +#include "qwen_safetensors.h" typedef struct { safetensor_dtype_t dtype; diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 88129d5..602571d 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -1,6 +1,6 @@ #include "pplx_embed_mlx.h" #include "pplx_embed.h" -#include "qwen_asr_safetensors.h" +#include "qwen_safetensors.h" #include diff --git a/pplx_server.c b/pplx_server.c index f8ad5e1..d831062 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -4,8 +4,8 @@ #include "pplx_server.h" #include "pplx_embed.h" -#include "qwen_asr_safetensors.h" -#include "qwen_asr_tokenizer.h" +#include "qwen_safetensors.h" +#include "qwen_tokenizer.h" #ifdef USE_MLX #include "pplx_embed_mlx.h" diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c deleted file mode 100644 index 24c8162..0000000 --- a/qwen_asr_kernels.c +++ /dev/null @@ -1,1745 +0,0 @@ -#include "qwen_asr_kernels.h" -#include "qwen_asr_kernels_impl.h" -#include -#include -#include -#include -#include -#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include -#endif -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include -#endif -#ifdef __APPLE__ -#include -#else -#include -#endif - -#ifdef USE_BLAS -#ifdef __APPLE__ -#include -#else -#include -#endif -#endif - -#if defined(USE_OPENBLAS) -void openblas_set_num_threads(int num_threads); -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -/* ======================================================================== - * Thread Pool - * ======================================================================== */ - -#define QWEN_MAX_THREADS 16 - -typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); - -static struct { - pthread_t threads[QWEN_MAX_THREADS - 1]; - int tids[QWEN_MAX_THREADS - 1]; - int n_threads; - int shutdown; - - parallel_fn_t fn; - void *arg; - int generation; - - pthread_mutex_t mutex; - pthread_cond_t cond_work; - pthread_cond_t cond_done; - int n_done; -} tp = { - .n_threads = 1, - .shutdown = 0, - .generation = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER, - .cond_work = PTHREAD_COND_INITIALIZER, - .cond_done = PTHREAD_COND_INITIALIZER, -}; - -static void *worker_loop(void *arg) { - int tid = *(int *)arg; - int my_gen = 0; - - for (;;) { - pthread_mutex_lock(&tp.mutex); - while (tp.generation == my_gen && !tp.shutdown) - pthread_cond_wait(&tp.cond_work, &tp.mutex); - if (tp.shutdown) { - pthread_mutex_unlock(&tp.mutex); - return NULL; - } - my_gen = tp.generation; - parallel_fn_t fn = tp.fn; - void *a = tp.arg; - int nt = tp.n_threads; - pthread_mutex_unlock(&tp.mutex); - - fn(tid, nt, a); - - pthread_mutex_lock(&tp.mutex); - if (++tp.n_done >= tp.n_threads - 1) - pthread_cond_signal(&tp.cond_done); - pthread_mutex_unlock(&tp.mutex); - } -} - -void qwen_set_threads(int n) { - if (n < 1) n = 1; - if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS; - -#if defined(USE_OPENBLAS) - /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the - * auxiliary pool serial to avoid nested thread contention. */ - int blas_threads = n; - openblas_set_num_threads(blas_threads); - n = 1; -#endif - - /* Shutdown existing workers */ - if (tp.n_threads > 1) { - pthread_mutex_lock(&tp.mutex); - tp.shutdown = 1; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - for (int i = 0; i < tp.n_threads - 1; i++) - pthread_join(tp.threads[i], NULL); - tp.shutdown = 0; - tp.generation = 0; - } - - tp.n_threads = n; - if (n <= 1) { - if (qwen_verbose >= 2) { -#if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", - n, blas_threads); -#else - fprintf(stderr, "Thread pool: %d threads\n", n); -#endif - } - return; - } - - for (int i = 0; i < n - 1; i++) { - tp.tids[i] = i + 1; - pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); - } - - if (qwen_verbose >= 2) { -#if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", - n, blas_threads); -#else - fprintf(stderr, "Thread pool: %d threads\n", n); -#endif - } -} - -int qwen_get_num_cpus(void) { -#ifdef __APPLE__ - int n = 0; - size_t len = sizeof(n); - sysctlbyname("hw.ncpu", &n, &len, NULL, 0); - return n > 0 ? n : 1; -#else - int n = (int)sysconf(_SC_NPROCESSORS_ONLN); - return n > 0 ? n : 1; -#endif -} - -/* Dispatch work to all threads; main thread is tid=0 */ -static void parallel_for(parallel_fn_t fn, void *arg) { - if (tp.n_threads <= 1) { - fn(0, 1, arg); - return; - } - - pthread_mutex_lock(&tp.mutex); - tp.fn = fn; - tp.arg = arg; - tp.n_done = 0; - tp.generation++; - pthread_cond_broadcast(&tp.cond_work); - pthread_mutex_unlock(&tp.mutex); - - fn(0, tp.n_threads, arg); - - pthread_mutex_lock(&tp.mutex); - while (tp.n_done < tp.n_threads - 1) - pthread_cond_wait(&tp.cond_done, &tp.mutex); - pthread_mutex_unlock(&tp.mutex); -} - -/* ======================================================================== - * Basic Element-wise Operations - * ======================================================================== */ - -void qwen_add_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) a[i] += b[i]; -} - -void qwen_mul_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) a[i] *= b[i]; -} - -void qwen_scale(float *x, float s, int n) { - for (int i = 0; i < n; i++) x[i] *= s; -} - -void qwen_copy(float *dst, const float *src, int n) { - memcpy(dst, src, n * sizeof(float)); -} - -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ - -void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { -#ifdef USE_BLAS - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); -#else - for (int m = 0; m < M; m++) { - for (int n = 0; n < N; n++) { - float sum = 0.0f; - for (int k = 0; k < K; k++) { - sum += A[m * K + k] * B[n * K + k]; - } - C[m * N + n] = sum; - } - } -#endif -} - -void qwen_linear(float *y, const float *x, const float *W, const float *b, - int seq_len, int in_dim, int out_dim) { -#ifdef USE_BLAS - if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, - out_dim, in_dim, 1.0f, W, in_dim, x, 1, - 0.0f, y, 1); - } else { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - seq_len, out_dim, in_dim, - 1.0f, x, in_dim, W, in_dim, - 0.0f, y, out_dim); - } - if (b != NULL) { - for (int s = 0; s < seq_len; s++) { - for (int o = 0; o < out_dim; o++) { - y[s * out_dim + o] += b[o]; - } - } - } -#else - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * in_dim; - float *y_row = y + s * out_dim; - for (int o = 0; o < out_dim; o++) { - const float *w_row = W + o * in_dim; - float sum = (b != NULL) ? b[o] : 0.0f; - for (int i = 0; i < in_dim; i++) { - sum += x_row[i] * w_row[i]; - } - y_row[o] = sum; - } - } -#endif -} - -void qwen_linear_nobias(float *y, const float *x, const float *W, - int seq_len, int in_dim, int out_dim) { - qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim); -} - -/* Convert bf16 buffer to f32 buffer */ -static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { -#if defined(__ARM_NEON) || defined(__ARM_NEON__) - size_t i = 0; - uint32_t *d = (uint32_t *)(void *)dst; - for (; i + 8 <= n; i += 8) { - uint16x8_t v = vld1q_u16(src + i); - uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(v)), 16); - uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(v)), 16); - vst1q_u32(d + i, lo); - vst1q_u32(d + i + 4, hi); - } - for (; i < n; i++) - d[i] = ((uint32_t)src[i]) << 16; -#else - uint32_t *d = (uint32_t *)(void *)dst; - for (size_t i = 0; i < n; i++) - d[i] = ((uint32_t)src[i]) << 16; -#endif -} - -/* Reusable scratch buffer for bf16->f32 conversion */ -static float *bf16_scratch = NULL; -static size_t bf16_scratch_cap = 0; - -static float *bf16_get_scratch(size_t n) { - if (n > bf16_scratch_cap) { - free(bf16_scratch); - bf16_scratch = (float *)malloc(n * sizeof(float)); - bf16_scratch_cap = bf16_scratch ? n : 0; - } - return bf16_scratch; -} - -typedef struct { - const uint16_t *src; - size_t n; - float *dst_f32; -} bf16_cache_entry_t; - -static bf16_cache_entry_t *bf16_cache = NULL; -static int bf16_cache_len = 0; -static int bf16_cache_cap = 0; -static size_t bf16_cache_bytes = 0; -static size_t bf16_cache_limit_bytes = 0; -static int bf16_cache_limit_init = 0; - -static void bf16_cache_init_limit(void) { - if (bf16_cache_limit_init) return; - bf16_cache_limit_init = 1; - - /* Default OFF. Override with QWEN_BF16_CACHE_MB= to enable. */ - unsigned long long mb = 0; - const char *env = getenv("QWEN_BF16_CACHE_MB"); - if (env && env[0] != '\0') { - char *end = NULL; - unsigned long long v = strtoull(env, &end, 10); - if (end != env) mb = v; - } - bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL); - - if (qwen_verbose >= 2) { - fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb); - } -} - -static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) { - bf16_cache_init_limit(); - - for (int i = 0; i < bf16_cache_len; i++) { - if (bf16_cache[i].src == src && bf16_cache[i].n == n) { - return bf16_cache[i].dst_f32; - } - } - - if (bf16_cache_limit_bytes == 0) return NULL; - - size_t bytes = n * sizeof(float); - if (bytes > bf16_cache_limit_bytes) return NULL; - if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL; - - float *dst = (float *)malloc(bytes); - if (!dst) return NULL; - bf16_to_f32_buf(dst, src, n); - - if (bf16_cache_len == bf16_cache_cap) { - int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256; - bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc( - bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t)); - if (!tmp) { - free(dst); - return NULL; - } - bf16_cache = tmp; - bf16_cache_cap = new_cap; - } - - bf16_cache[bf16_cache_len].src = src; - bf16_cache[bf16_cache_len].n = n; - bf16_cache[bf16_cache_len].dst_f32 = dst; - bf16_cache_len++; - bf16_cache_bytes += bytes; - return dst; -} - -static const float *bf16_get_f32_view(const uint16_t *src, size_t n) { - const float *cached = bf16_get_cached_f32(src, n); - if (cached) return cached; - - float *scratch = bf16_get_scratch(n); - if (!scratch) return NULL; - bf16_to_f32_buf(scratch, src, n); - return scratch; -} - -/* - * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias - * Processes 2 output rows at a time to amortize x vector loads. - */ -static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); -} - -/* Threaded matvec: split output rows across threads */ -typedef struct { - float *y; - const float *x; - const uint16_t *W_bf16; - const float *bias; - int in_dim; - int out_dim; -} matvec_task_t; - -static void matvec_worker(int tid, int n_threads, void *arg) { - matvec_task_t *t = (matvec_task_t *)arg; - int chunk = (t->out_dim + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->out_dim) end = t->out_dim; - if (start >= end) return; - - bf16_matvec_fused(t->y + start, t->x, - t->W_bf16 + (size_t)start * t->in_dim, - t->bias ? t->bias + start : NULL, - t->in_dim, end - start); -} - -static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - if (tp.n_threads <= 1) { - bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); - return; - } - matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim }; - parallel_for(matvec_worker, &task); -} - -typedef struct { - float *y; - const float *x; - const uint16_t *W_bf16; - const float *bias; - int seq_len; - int in_dim; - int out_dim; -} bf16_linear_rows_task_t; - -static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { - bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->seq_len) end = t->seq_len; - if (start >= end) return; - - for (int s = start; s < end; s++) { - bf16_matvec_fused(t->y + (size_t)s * t->out_dim, - t->x + (size_t)s * t->in_dim, - t->W_bf16, t->bias, t->in_dim, t->out_dim); - } -} - -static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int seq_len, int in_dim, - int out_dim) { - bf16_linear_rows_task_t task = { - .y = y, - .x = x, - .W_bf16 = W_bf16, - .bias = bias, - .seq_len = seq_len, - .in_dim = in_dim, - .out_dim = out_dim, - }; - - if (tp.n_threads > 1 && seq_len >= 2) - parallel_for(bf16_linear_rows_worker, &task); - else - bf16_linear_rows_worker(0, 1, &task); -} - -typedef struct { - float *a; - float *b; - const float *x; - const uint16_t *Wa_bf16; - const uint16_t *Wb_bf16; - int seq_len; - int in_dim; - int a_dim; - int b_dim; - int total_dim; -} pair_matvec_task_t; - -static void pair_matvec_range(const pair_matvec_task_t *t, int row, - int start, int end) { - const float *x_row = t->x + (size_t)row * t->in_dim; - int a_end = t->a_dim; - int b_end = a_end + t->b_dim; - - if (start < a_end) { - int s = start; - int e = end < a_end ? end : a_end; - if (s < e) { - bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, - t->Wa_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); - } - } - - if (end > a_end && start < b_end) { - int s = start > a_end ? start - a_end : 0; - int e_abs = end < b_end ? end : b_end; - int e = e_abs - a_end; - if (s < e) { - bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, - t->Wb_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); - } - } -} - -static void pair_matvec_worker(int tid, int n_threads, void *arg) { - pair_matvec_task_t *t = (pair_matvec_task_t *)arg; - int total_work = t->seq_len * t->total_dim; - int chunk = (total_work + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > total_work) end = total_work; - if (start >= end) return; - - while (start < end) { - int row = start / t->total_dim; - int pos = start - row * t->total_dim; - int row_end = (row + 1) * t->total_dim; - int stop = end < row_end ? end : row_end; - pair_matvec_range(t, row, pos, stop - row * t->total_dim); - start = stop; - } -} - -void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, - const uint16_t *Wa_bf16, - const uint16_t *Wb_bf16, - int seq_len, int in_dim, int a_dim, - int b_dim) { - if (seq_len <= 0) return; - - if (tp.n_threads <= 1) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(a + (size_t)s * a_dim, x_row, - Wa_bf16, NULL, in_dim, a_dim); - bf16_matvec_fused(b + (size_t)s * b_dim, x_row, - Wb_bf16, NULL, in_dim, b_dim); - } - return; - } - - pair_matvec_task_t task = { - .a = a, - .b = b, - .x = x, - .Wa_bf16 = Wa_bf16, - .Wb_bf16 = Wb_bf16, - .seq_len = seq_len, - .in_dim = in_dim, - .a_dim = a_dim, - .b_dim = b_dim, - .total_dim = a_dim + b_dim, - }; - parallel_for(pair_matvec_worker, &task); -} - -typedef struct { - float *q; - float *k; - float *v; - const float *x; - const uint16_t *Wq_bf16; - const uint16_t *Wk_bf16; - const uint16_t *Wv_bf16; - int in_dim; - int q_dim; - int kv_dim; - int seq_len; - int total_dim; -} qkv_matvec_task_t; - -static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, - int start, int end) { - const float *x_row = t->x + (size_t)row * t->in_dim; - int q_end = t->q_dim; - int k_end = q_end + t->kv_dim; - int v_end = k_end + t->kv_dim; - - if (start < q_end) { - int s = start; - int e = end < q_end ? end : q_end; - if (s < e) { - bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, - t->Wq_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); - } - } - - if (end > q_end && start < k_end) { - int s = start > q_end ? start - q_end : 0; - int e_abs = end < k_end ? end : k_end; - int e = e_abs - q_end; - if (s < e) { - bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, - t->Wk_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); - } - } - - if (end > k_end && start < v_end) { - int s = start > k_end ? start - k_end : 0; - int e_abs = end < v_end ? end : v_end; - int e = e_abs - k_end; - if (s < e) { - bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, - t->Wv_bf16 + (size_t)s * t->in_dim, - NULL, t->in_dim, e - s); - } - } -} - -static void qkv_matvec_worker(int tid, int n_threads, void *arg) { - qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; - int total_work = t->seq_len * t->total_dim; - int chunk = (total_work + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > total_work) end = total_work; - if (start >= end) return; - - while (start < end) { - int row = start / t->total_dim; - int pos = start - row * t->total_dim; - int row_end = (row + 1) * t->total_dim; - int stop = end < row_end ? end : row_end; - qkv_matvec_range(t, row, pos, stop - row * t->total_dim); - start = stop; - } -} - -void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, - const uint16_t *Wq_bf16, - const uint16_t *Wk_bf16, - const uint16_t *Wv_bf16, - int seq_len, int in_dim, int q_dim, - int kv_dim) { - if (seq_len <= 0) return; - - if (tp.n_threads <= 1) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(q + (size_t)s * q_dim, x_row, - Wq_bf16, NULL, in_dim, q_dim); - bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, - Wk_bf16, NULL, in_dim, kv_dim); - bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, - Wv_bf16, NULL, in_dim, kv_dim); - } - return; - } - - qkv_matvec_task_t task = { - .q = q, - .k = k, - .v = v, - .x = x, - .Wq_bf16 = Wq_bf16, - .Wk_bf16 = Wk_bf16, - .Wv_bf16 = Wv_bf16, - .in_dim = in_dim, - .q_dim = q_dim, - .kv_dim = kv_dim, - .seq_len = seq_len, - .total_dim = q_dim + 2 * kv_dim, - }; - parallel_for(qkv_matvec_worker, &task); -} - -void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, - int seq_len, int in_dim, int out_dim) { - if (seq_len == 1) { - bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); - return; - } - if (seq_len <= 16) { - bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); - return; - } - size_t n = (size_t)out_dim * in_dim; - const float *W_f32 = bf16_get_f32_view(W_bf16, n); - if (!W_f32) return; - qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim); -} - -void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, - const float *b, int seq_len, int in_dim, int out_dim) { - if (seq_len == 1) { - bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim); - return; - } - if (seq_len <= 16) { - bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim); - return; - } - size_t n = (size_t)out_dim * in_dim; - const float *W_f32 = bf16_get_f32_view(W_bf16, n); - if (!W_f32) return; - qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim); -} - -/* Find argmax over a range of output rows [start, end). - * Uses 2-row processing to amortize x vector loads (same as bf16_matvec_fused). */ -static void argmax_bf16_range(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - qwen_argmax_bf16_range_impl(x, W_bf16, in_dim, start, end, best_out, best_val_out); -} - -typedef struct { - const float *x; - const uint16_t *W_bf16; - int in_dim; - int out_dim; - int best_idx[QWEN_MAX_THREADS]; - float best_val[QWEN_MAX_THREADS]; -} argmax_task_t; - -static void argmax_worker(int tid, int n_threads, void *arg) { - argmax_task_t *t = (argmax_task_t *)arg; - int chunk = (t->out_dim + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->out_dim) end = t->out_dim; - if (start >= end) { - t->best_val[tid] = -1e30f; - t->best_idx[tid] = 0; - return; - } - argmax_bf16_range(t->x, t->W_bf16, t->in_dim, start, end, - &t->best_idx[tid], &t->best_val[tid]); -} - -int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, - int in_dim, int out_dim) { - if (tp.n_threads <= 1) { - int best; - float best_val; - argmax_bf16_range(x, W_bf16, in_dim, 0, out_dim, &best, &best_val); - return best; - } - - argmax_task_t task; - task.x = x; - task.W_bf16 = W_bf16; - task.in_dim = in_dim; - task.out_dim = out_dim; - parallel_for(argmax_worker, &task); - - int best = task.best_idx[0]; - float best_val = task.best_val[0]; - for (int i = 1; i < tp.n_threads; i++) { - if (task.best_val[i] > best_val) { - best_val = task.best_val[i]; - best = task.best_idx[i]; - } - } - return best; -} - -void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, - int M, int K, int N) { - if (M == 1) { - bf16_matvec_threaded(C, A, B_bf16, NULL, K, N); - } else { - size_t n = (size_t)N * K; - const float *B_f32 = bf16_get_f32_view(B_bf16, n); - if (!B_f32) return; - qwen_matmul_t(C, A, B_f32, M, K, N); - } -} - -/* ======================================================================== - * 2D Convolution (im2col + BLAS sgemm) - * ======================================================================== */ - -/* - * im2col: Unroll input patches into a column matrix for GEMM-based conv2d. - * Input: [C_in, H_in, W_in] - * Output columns: [C_in * kH * kW, H_out * W_out] - */ -static void im2col(const float *in, float *cols, - int c_in, int h_in, int w_in, - int kh, int kw, int stride, int padding, - int h_out, int w_out) { - int col_len = h_out * w_out; - for (int ic = 0; ic < c_in; ic++) { - for (int ki = 0; ki < kh; ki++) { - for (int kj = 0; kj < kw; kj++) { - int col_row = (ic * kh + ki) * kw + kj; - float *col_ptr = cols + (size_t)col_row * col_len; - for (int oh = 0; oh < h_out; oh++) { - int ih = oh * stride - padding + ki; - for (int ow = 0; ow < w_out; ow++) { - int iw = ow * stride - padding + kj; - if (ih >= 0 && ih < h_in && iw >= 0 && iw < w_in) { - col_ptr[oh * w_out + ow] = in[ic * h_in * w_in + ih * w_in + iw]; - } else { - col_ptr[oh * w_out + ow] = 0.0f; - } - } - } - } - } - } -} - -void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, - int c_in, int c_out, int h_in, int w_in, - int kh, int kw, int stride, int padding) { - int h_out = (h_in + 2 * padding - kh) / stride + 1; - int w_out = (w_in + 2 * padding - kw) / stride + 1; - int patch_size = c_in * kh * kw; - int spatial_out = h_out * w_out; - - /* im2col: input -> column matrix [patch_size, spatial_out] */ - float *cols = (float *)malloc((size_t)patch_size * spatial_out * sizeof(float)); - im2col(in, cols, c_in, h_in, w_in, kh, kw, stride, padding, h_out, w_out); - - /* GEMM: weight[c_out, patch_size] @ cols[patch_size, spatial_out] = out[c_out, spatial_out] */ -#ifdef USE_BLAS - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - c_out, spatial_out, patch_size, - 1.0f, weight, patch_size, cols, spatial_out, - 0.0f, out, spatial_out); -#else - for (int oc = 0; oc < c_out; oc++) { - for (int s = 0; s < spatial_out; s++) { - float sum = 0.0f; - for (int p = 0; p < patch_size; p++) { - sum += weight[oc * patch_size + p] * cols[p * spatial_out + s]; - } - out[oc * spatial_out + s] = sum; - } - } -#endif - - free(cols); - - /* Add bias */ - if (bias) { - for (int oc = 0; oc < c_out; oc++) { - float b = bias[oc]; - float *row = out + oc * spatial_out; - for (int s = 0; s < spatial_out; s++) { - row[s] += b; - } - } - } -} - -/* ======================================================================== - * Normalization - * ======================================================================== */ - -void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, - int seq_len, int hidden, float eps) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * hidden; - float *out_row = out + s * hidden; - - /* Compute mean */ -#if defined(__AVX512F__) - __m512 sumv = _mm512_setzero_ps(); - int i = 0; - for (; i + 16 <= hidden; i += 16) { - sumv = _mm512_add_ps(sumv, _mm512_loadu_ps(x_row + i)); - } - float mean = _mm512_reduce_add_ps(sumv); - for (; i < hidden; i++) mean += x_row[i]; -#elif defined(__AVX2__) - __m256 sumv = _mm256_setzero_ps(); - int i = 0; - for (; i + 8 <= hidden; i += 8) { - sumv = _mm256_add_ps(sumv, _mm256_loadu_ps(x_row + i)); - } - __m128 sum128 = _mm_add_ps(_mm256_castps256_ps128(sumv), _mm256_extractf128_ps(sumv, 1)); - sum128 = _mm_hadd_ps(sum128, sum128); - sum128 = _mm_hadd_ps(sum128, sum128); - float mean = _mm_cvtss_f32(sum128); - for (; i < hidden; i++) mean += x_row[i]; -#else - float mean = 0.0f; - for (int i = 0; i < hidden; i++) mean += x_row[i]; -#endif - mean /= hidden; - - /* Compute variance */ -#if defined(__AVX512F__) && defined(__FMA__) - __m512 meanv = _mm512_set1_ps(mean); - __m512 accv = _mm512_setzero_ps(); - int j = 0; - for (; j + 16 <= hidden; j += 16) { - __m512 v = _mm512_sub_ps(_mm512_loadu_ps(x_row + j), meanv); - accv = _mm512_fmadd_ps(v, v, accv); - } - float var = _mm512_reduce_add_ps(accv); - for (; j < hidden; j++) { - float d = x_row[j] - mean; - var += d * d; - } -#elif defined(__AVX2__) && defined(__FMA__) - __m256 meanv = _mm256_set1_ps(mean); - __m256 accv = _mm256_setzero_ps(); - int j = 0; - for (; j + 8 <= hidden; j += 8) { - __m256 v = _mm256_sub_ps(_mm256_loadu_ps(x_row + j), meanv); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float var = _mm_cvtss_f32(acc128); - for (; j < hidden; j++) { - float d = x_row[j] - mean; - var += d * d; - } -#else - float var = 0.0f; - for (int i = 0; i < hidden; i++) { - float d = x_row[i] - mean; - var += d * d; - } -#endif - var /= hidden; - - float inv_std = 1.0f / sqrtf(var + eps); -#if defined(__AVX512F__) && defined(__FMA__) - __m512 meanv2 = _mm512_set1_ps(mean); - __m512 invv = _mm512_set1_ps(inv_std); - int k = 0; - for (; k + 16 <= hidden; k += 16) { - __m512 vx = _mm512_sub_ps(_mm512_loadu_ps(x_row + k), meanv2); - __m512 vw = _mm512_loadu_ps(weight + k); - __m512 vb = _mm512_loadu_ps(bias + k); - __m512 v = _mm512_mul_ps(_mm512_mul_ps(vx, invv), vw); - v = _mm512_add_ps(v, vb); - _mm512_storeu_ps(out_row + k, v); - } - for (; k < hidden; k++) { - out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; - } -#elif defined(__AVX2__) && defined(__FMA__) - __m256 meanv2 = _mm256_set1_ps(mean); - __m256 invv = _mm256_set1_ps(inv_std); - int k = 0; - for (; k + 8 <= hidden; k += 8) { - __m256 vx = _mm256_sub_ps(_mm256_loadu_ps(x_row + k), meanv2); - __m256 vw = _mm256_loadu_ps(weight + k); - __m256 vb = _mm256_loadu_ps(bias + k); - __m256 v = _mm256_mul_ps(_mm256_mul_ps(vx, invv), vw); - v = _mm256_add_ps(v, vb); - _mm256_storeu_ps(out_row + k, v); - } - for (; k < hidden; k++) { - out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; - } -#else - for (int i = 0; i < hidden; i++) { - out_row[i] = (x_row[i] - mean) * inv_std * weight[i] + bias[i]; - } -#endif - } -} - -#define QWEN_RMS_NORM_PARALLEL_ELEMS 262144 - -static void qwen_rms_norm_range(float *out, const float *x, - const float *weight, int start, int end, - int hidden, float eps) { - for (int s = start; s < end; s++) { - const float *x_row = x + (size_t)s * hidden; - float *out_row = out + (size_t)s * hidden; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int i = 0; - for (; i + 16 <= hidden; i += 16) { - __m512 v = _mm512_loadu_ps(x_row + i); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int i = 0; - for (; i + 8 <= hidden; i += 8) { - __m256 v = _mm256_loadu_ps(x_row + i); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; -#else - float sum_sq = 0.0f; - for (int i = 0; i < hidden; i++) { - sum_sq += x_row[i] * x_row[i]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= hidden; j += 16) { - __m512 vx = _mm512_loadu_ps(x_row + j); - __m512 vw = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= hidden; j += 8) { - __m256 vx = _mm256_loadu_ps(x_row + j); - __m256 vw = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; -#else - for (int i = 0; i < hidden; i++) { - out_row[i] = x_row[i] * rms_inv * weight[i]; - } -#endif - } -} - -typedef struct { - float *out; - const float *x; - const float *weight; - int seq_len; - int hidden; - float eps; -} rms_norm_task_t; - -static void rms_norm_worker(int tid, int n_threads, void *arg) { - rms_norm_task_t *t = (rms_norm_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->seq_len) end = t->seq_len; - if (start >= end) return; - qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, - t->eps); -} - -void qwen_rms_norm(float *out, const float *x, const float *weight, - int seq_len, int hidden, float eps) { - if (seq_len <= 0 || hidden <= 0) return; - - long long elems = (long long)seq_len * hidden; - if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) { - rms_norm_task_t task = { - .out = out, - .x = x, - .weight = weight, - .seq_len = seq_len, - .hidden = hidden, - .eps = eps, - }; - parallel_for(rms_norm_worker, &task); - } else { - qwen_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); - } -} - -void qwen_rms_norm_per_head(float *x, const float *weight, - int seq_len, int n_heads, int head_dim, float eps) { - /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ - int hidden = n_heads * head_dim; - for (int s = 0; s < seq_len; s++) { - for (int h = 0; h < n_heads; h++) { - float *vec = x + s * hidden + h * head_dim; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int d = 0; - for (; d + 16 <= head_dim; d += 16) { - __m512 v = _mm512_loadu_ps(vec + d); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int d = 0; - for (; d + 8 <= head_dim; d += 8) { - __m256 v = _mm256_loadu_ps(vec + d); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; -#else - float sum_sq = 0.0f; - for (int d = 0; d < head_dim; d++) { - sum_sq += vec[d] * vec[d]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= head_dim; j += 16) { - __m512 v = _mm512_loadu_ps(vec + j); - __m512 w = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= head_dim; j += 8) { - __m256 v = _mm256_loadu_ps(vec + j); - __m256 w = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; -#else - for (int d = 0; d < head_dim; d++) { - vec[d] = vec[d] * rms_inv * weight[d]; - } -#endif - } - } -} - -/* ======================================================================== - * Activation Functions - * ======================================================================== */ - -void qwen_silu(float *x, int n) { - for (int i = 0; i < n; i++) { - float val = x[i]; - x[i] = val / (1.0f + expf(-val)); - } -} - -void qwen_gelu(float *x, int n) { - for (int i = 0; i < n; i++) { - float val = x[i]; - float x3 = val * val * val; - float inner = 0.7978845608028654f * (val + 0.044715f * x3); - x[i] = 0.5f * val * (1.0f + tanhf(inner)); - } -} - -void qwen_silu_mul_inplace(float *gate, const float *up, int n) { -#if defined(__APPLE__) && defined(USE_BLAS) - enum { CHUNK = 4096 }; - float exp_neg[CHUNK]; - - for (int base = 0; base < n; base += CHUNK) { - int len = n - base; - if (len > CHUNK) len = CHUNK; - - for (int i = 0; i < len; i++) - exp_neg[i] = -gate[base + i]; - vvexpf(exp_neg, exp_neg, &len); - - for (int i = 0; i < len; i++) { - float g = gate[base + i]; - gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i]; - } - } -#else - for (int i = 0; i < n; i++) { - float g = gate[i]; - gate[i] = (g / (1.0f + expf(-g))) * up[i]; - } -#endif -} - -typedef struct { - float *out; - const float *gate_up; - int seq_len; - int intermediate; -} swiglu_task_t; - -static void swiglu_worker(int tid, int n_threads, void *arg) { - swiglu_task_t *t = (swiglu_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int s0 = tid * chunk; - int s1 = s0 + chunk; - if (s1 > t->seq_len) s1 = t->seq_len; - if (s0 >= s1) return; - - int inter = t->intermediate; - int alias_inplace = (t->out == t->gate_up); - for (int s = s0; s < s1; s++) { - const float *gu = t->gate_up + (size_t)s * 2 * inter; - float *o = t->out + (size_t)s * inter; - if (!alias_inplace) { -#if defined(__APPLE__) && defined(USE_BLAS) - /* Fast path for prefill: vectorized exp(-g) using Accelerate/vForce. */ - for (int j = 0; j < inter; j++) o[j] = -gu[2 * j]; - int n = inter; - vvexpf(o, o, &n); - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - o[j] = (g / (1.0f + o[j])) * u; - } -#else - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - g = g / (1.0f + expf(-g)); /* SiLU */ - o[j] = g * u; - } -#endif - } else { - /* In-place mode (decode seq=1): keep single-pass scalar to avoid alias hazards. */ - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - g = g / (1.0f + expf(-g)); /* SiLU */ - o[j] = g * u; - } - } - } -} - -void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate) { - swiglu_task_t task = { - .out = out, - .gate_up = gate_up, - .seq_len = seq_len, - .intermediate = intermediate - }; - - if (tp.n_threads > 1 && seq_len >= 2 && intermediate >= 256) { - parallel_for(swiglu_worker, &task); - } else { - swiglu_worker(0, 1, &task); - } -} - -void qwen_softmax(float *x, int rows, int cols) { - for (int r = 0; r < rows; r++) { - float *row = x + r * cols; - float max_val = row[0]; - for (int c = 1; c < cols; c++) { - if (row[c] > max_val) max_val = row[c]; - } - float sum = 0.0f; - for (int c = 0; c < cols; c++) { - row[c] = expf(row[c] - max_val); - sum += row[c]; - } - float inv_sum = 1.0f / sum; - for (int c = 0; c < cols; c++) { - row[c] *= inv_sum; - } - } -} - -/* ======================================================================== - * Attention Operations - * ======================================================================== */ - -float qwen_dot_f32(const float *a, const float *b, int n) { - return qwen_dot_f32_impl(a, b, n); -} - -static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) { - return qwen_dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void qwen_vec_scale_inplace(float *dst, float scale, int n) { - qwen_vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void qwen_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - qwen_vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void qwen_vec_scale_add(float *dst, const float *src, float correction, int n) { - qwen_vec_scale_add_impl(dst, src, correction, n); -} - -void qwen_bidirectional_attention(float *out, const float *Q, const float *K, - const float *V, int seq __attribute__((unused)), - int n_heads, int head_dim, float scale, - const int *window_starts, int n_windows) { - int hidden = n_heads * head_dim; - - for (int h = 0; h < n_heads; h++) { - for (int w = 0; w < n_windows; w++) { - int ws = window_starts[w]; - int we = window_starts[w + 1]; - - for (int i = ws; i < we; i++) { - const float *q_row = Q + i * hidden + h * head_dim; - float *o_row = out + i * hidden + h * head_dim; - - /* Online softmax */ - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - - for (int j = ws; j < we; j++) { - const float *k_row = K + j * hidden + h * head_dim; - const float *v_row = V + j * hidden + h * head_dim; - - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } - } - } -} - -static void qwen_causal_attention_heads(float *out, const float *Q, const float *K, - const float *V, int seq_q, int seq_k, - int n_heads, int n_kv_heads, int head_dim, - float scale, int q_offset, - int head_start, int head_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - - for (int h = head_start; h < head_end; h++) { - int kv_h = h / heads_per_kv; - - for (int i = 0; i < seq_q; i++) { - const float *q_row = Q + i * q_hidden + h * head_dim; - float *o_row = out + i * q_hidden + h * head_dim; - int global_pos = q_offset + i; - int k_end = global_pos + 1; - if (k_end > seq_k) k_end = seq_k; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - - for (int j = 0; j < k_end; j++) { - const float *k_row = K + j * kv_hidden + kv_h * head_dim; - const float *v_row = V + j * kv_hidden + kv_h * head_dim; - - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } - } -} - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - int seq_q, seq_k; - int n_heads, n_kv_heads; - int head_dim; - float scale; - int q_offset; -} causal_attn_task_t; - -static void causal_attn_worker(int tid, int n_threads, void *arg) { - causal_attn_task_t *t = (causal_attn_task_t *)arg; - int chunk = (t->n_heads + n_threads - 1) / n_threads; - int h0 = tid * chunk; - int h1 = h0 + chunk; - if (h1 > t->n_heads) h1 = t->n_heads; - if (h0 >= h1) return; - - qwen_causal_attention_heads(t->out, t->Q, t->K, t->V, - t->seq_q, t->seq_k, t->n_heads, t->n_kv_heads, - t->head_dim, t->scale, t->q_offset, h0, h1); -} - -void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, - int seq_q, int seq_k, int n_heads, int n_kv_heads, - int head_dim, float scale, int q_offset) { - if (tp.n_threads > 1 && n_heads >= 2 && (seq_q >= 2 || seq_k >= 128)) { - causal_attn_task_t task = { - .out = out, .Q = Q, .K = K, .V = V, - .seq_q = seq_q, .seq_k = seq_k, - .n_heads = n_heads, .n_kv_heads = n_kv_heads, - .head_dim = head_dim, .scale = scale, .q_offset = q_offset - }; - parallel_for(causal_attn_worker, &task); - return; - } - - qwen_causal_attention_heads(out, Q, K, V, - seq_q, seq_k, n_heads, n_kv_heads, - head_dim, scale, q_offset, 0, n_heads); -} - -#define QWEN_PACKED_ATTN_QUERY_TILE 32 -#define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128 - -static void qwen_bidirectional_gqa_attention_packed_online_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int end, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } -} - -#ifdef USE_BLAS -static void qwen_bidirectional_gqa_attention_packed_blas_rows( - float *out, const float *Q, const float *K, const float *V, - int start, int end, int n_heads, int n_kv_heads, int head_dim, - float scale, int h, int query_start, int query_end, float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, - 0.0f, scores, keys); - qwen_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, - 0.0f, o, q_hidden); -} -#endif - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; - int head_dim; - float scale; - float *scores; - int max_seq; -} packed_gqa_attn_task_t; - -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = t->scores - ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq - : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; - if (q1 > end) q1 = end; - if (task_id++ % n_threads != tid) continue; -#ifdef USE_BLAS - if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { - qwen_bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, - scores); - continue; - } -#endif - qwen_bidirectional_gqa_attention_packed_online_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, - t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); - } - } - } -} - -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( - const int *offsets, int batch) { -#ifdef USE_BLAS - if (!offsets || batch <= 0) return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) max_seq = len; - } - if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0; - if ((size_t)max_seq > SIZE_MAX / (sizeof(float) * - QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) - return 0; - return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * - (size_t)max_seq * sizeof(float); -#else - (void)offsets; - (void)batch; - return 0; -#endif -} - -void qwen_bidirectional_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( - offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = { - .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, - .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, - .head_dim = head_dim, .scale = scale, .scores = scores, - .max_seq = max_seq - }; - if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); - } else { - packed_gqa_attn_worker(0, 1, &task); - } -} - -void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale) { - size_t scratch_bytes = - qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - qwen_bidirectional_gqa_attention_packed_with_scratch( - out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, - scores, scratch_bytes); - free(scores); -} - -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ - -void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model) { - int half = d_model / 2; - float log_timescale = logf(10000.0f) / (float)(half - 1); - - for (int p = 0; p < n_pos; p++) { - float *row = pe + p * d_model; - for (int d = 0; d < half; d++) { - float inv_timescale = expf(-(float)d * log_timescale); - float angle = (float)p * inv_timescale; - row[d] = sinf(angle); /* first half: sin */ - row[half + d] = cosf(angle); /* second half: cos */ - } - } -} - -void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, - int seq, int head_dim, float theta) { - int half = head_dim / 2; - - for (int s = 0; s < seq; s++) { - float pos = (float)positions[s]; - for (int d = 0; d < half; d++) { - float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = pos * freq; - float c = cosf(angle); - float sn = sinf(angle); - /* Duplicate for full head_dim */ - cos_out[s * head_dim + d] = c; - cos_out[s * head_dim + half + d] = c; - sin_out[s * head_dim + d] = sn; - sin_out[s * head_dim + half + d] = sn; - } - } -} - -void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, - int seq, int n_heads, int head_dim) { - /* - * NeoX split-half style: - * x1 = x[..., :half], x2 = x[..., half:] - * rotated = cat(-x2, x1) - * result = x * cos + rotated * sin - */ - int half = head_dim / 2; - int hidden = n_heads * head_dim; - - for (int s = 0; s < seq; s++) { - const float *c = cos_vals + s * head_dim; - const float *sn = sin_vals + s * head_dim; - - for (int h = 0; h < n_heads; h++) { - float *vec = x + s * hidden + h * head_dim; - -#if defined(__AVX512F__) && defined(__FMA__) - int d = 0; - for (; d + 16 <= half; d += 16) { - __m512 x1 = _mm512_loadu_ps(vec + d); - __m512 x2 = _mm512_loadu_ps(vec + half + d); - /* RoPE cache duplicates cos/sin across halves. */ - __m512 cc = _mm512_loadu_ps(c + d); - __m512 ss = _mm512_loadu_ps(sn + d); - __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); - __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); - _mm512_storeu_ps(vec + d, new1); - _mm512_storeu_ps(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#elif defined(__AVX2__) && defined(__FMA__) - int d = 0; - for (; d + 8 <= half; d += 8) { - __m256 x1 = _mm256_loadu_ps(vec + d); - __m256 x2 = _mm256_loadu_ps(vec + half + d); - __m256 cc = _mm256_loadu_ps(c + d); - __m256 ss = _mm256_loadu_ps(sn + d); - __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); - __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); - _mm256_storeu_ps(vec + d, new1); - _mm256_storeu_ps(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#elif defined(__ARM_NEON) || defined(__ARM_NEON__) - int d = 0; - for (; d + 4 <= half; d += 4) { - float32x4_t x1 = vld1q_f32(vec + d); - float32x4_t x2 = vld1q_f32(vec + half + d); - float32x4_t cc = vld1q_f32(c + d); - float32x4_t ss = vld1q_f32(sn + d); - float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); - float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); - vst1q_f32(vec + d, new1); - vst1q_f32(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#else - for (int d = 0; d < half; d++) { - float x1 = vec[d]; /* first half */ - float x2 = vec[half + d]; /* second half */ - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#endif - } - } -} diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h deleted file mode 100644 index 63b704d..0000000 --- a/qwen_asr_kernels.h +++ /dev/null @@ -1,210 +0,0 @@ -/* - * qwen_asr_kernels.h - Math kernels for Qwen3-ASR inference - * - * Low-level math operations. All operate on float32 tensors in row-major order. - * Adapted from voxtral-realtime project. - */ - -#ifndef QWEN_ASR_KERNELS_H -#define QWEN_ASR_KERNELS_H - -#include -#include - -/* ======================================================================== - * Basic Operations - * ======================================================================== */ - -void qwen_add_inplace(float *a, const float *b, int n); -void qwen_mul_inplace(float *a, const float *b, int n); -void qwen_scale(float *x, float s, int n); -void qwen_copy(float *dst, const float *src, int n); - -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ - -/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ -void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); - -/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ -void qwen_linear(float *y, const float *x, const float *W, const float *b, - int seq_len, int in_dim, int out_dim); - -void qwen_linear_nobias(float *y, const float *x, const float *W, - int seq_len, int in_dim, int out_dim); - -/* bf16 weight variants */ -void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, - const float *b, int seq_len, int in_dim, int out_dim); - -void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, - int seq_len, int in_dim, int out_dim); - -/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ -void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, - const uint16_t *Wa_bf16, - const uint16_t *Wb_bf16, - int seq_len, int in_dim, int a_dim, - int b_dim); - -/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ -void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, - const uint16_t *Wq_bf16, - const uint16_t *Wk_bf16, - const uint16_t *Wv_bf16, - int seq_len, int in_dim, int q_dim, - int kv_dim); - -void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, - int M, int K, int N); - -/* Dot product using the best available local SIMD implementation. */ -float qwen_dot_f32(const float *a, const float *b, int n); - -/* ======================================================================== - * 2D Convolution (for audio encoder conv stem) - * ======================================================================== */ - -/* - * 2D Convolution: out = conv2d(in, weight, bias) - * in: [C_in, H, W] - * weight: [C_out, C_in, kH, kW] - * bias: [C_out] (can be NULL) - * out: [C_out, H_out, W_out] - * H_out = (H + 2*padding - kH) / stride + 1 - * W_out = (W + 2*padding - kW) / stride + 1 - */ -void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, - int c_in, int c_out, int h_in, int w_in, - int kh, int kw, int stride, int padding); - -/* ======================================================================== - * Normalization - * ======================================================================== */ - -/* LayerNorm with bias: out = (x - mean) / sqrt(var + eps) * weight + bias */ -void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, - int seq_len, int hidden, float eps); - -/* RMS Normalization: out = x / rms(x) * weight */ -void qwen_rms_norm(float *out, const float *x, const float *weight, - int seq_len, int hidden, float eps); - -/* Per-head RMS Normalization for Q/K norms in decoder - * x: [seq, n_heads, head_dim], weight: [head_dim] - * Normalizes each head independently */ -void qwen_rms_norm_per_head(float *x, const float *weight, - int seq_len, int n_heads, int head_dim, float eps); - -/* ======================================================================== - * Activation Functions - * ======================================================================== */ - -void qwen_silu(float *x, int n); -void qwen_gelu(float *x, int n); -void qwen_softmax(float *x, int rows, int cols); -/* gate = SiLU(gate) * up */ -void qwen_silu_mul_inplace(float *gate, const float *up, int n); -/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */ -void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate); - -/* ======================================================================== - * Attention Operations - * ======================================================================== */ - -/* - * Bidirectional windowed attention (encoder). - * Q, K, V: [seq, n_heads * head_dim] - * out: [seq, n_heads * head_dim] - * window_starts: array of window start positions - * window_starts[n_windows] = seq (sentinel) - * All heads have same dimensions (no GQA in encoder). - */ -void qwen_bidirectional_attention(float *out, const float *Q, const float *K, - const float *V, int seq, int n_heads, - int head_dim, float scale, - const int *window_starts, int n_windows); - -/* - * Causal attention with GQA (decoder). - * Q: [seq_q, n_heads * head_dim] - * K: [seq_k, n_kv_heads * head_dim] - * V: [seq_k, n_kv_heads * head_dim] - * q_offset: global position of first query (for causal mask) - */ -void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, - int seq_q, int seq_k, int n_heads, int n_kv_heads, - int head_dim, float scale, int q_offset); - -/* - * Bidirectional GQA attention over a packed/ragged batch. - * - * Q: [total_seq, n_heads * head_dim] - * K,V: [total_seq, n_kv_heads * head_dim] - * offsets: [batch + 1], token spans for each sequence. - * - * Attention is block-diagonal: tokens only attend within their own - * offsets[b]..offsets[b+1] span. - */ -void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale); - -/* Optional reusable scratch for the BLAS-tiled packed attention path. */ -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( - const int *offsets, int batch); -void qwen_bidirectional_gqa_attention_packed_with_scratch( - float *out, const float *Q, const float *K, const float *V, - const int *offsets, int batch, int n_heads, int n_kv_heads, - int head_dim, float scale, float *scratch, size_t scratch_bytes); - -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ - -/* - * Sinusoidal position embeddings (encoder). - * pe: output [n_pos, d_model] - * First half = sin, second half = cos. - */ -void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model); - -/* - * NeoX-style RoPE: compute cos/sin for positions. - * cos_out, sin_out: [seq, head_dim] - * cos[d] and cos[half+d] are the same (duplicated for full head_dim). - */ -void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, - int seq, int head_dim, float theta); - -/* - * Apply NeoX-style RoPE to Q or K. - * x: [seq, n_heads * head_dim] (in-place) - * cos_vals, sin_vals: [seq, head_dim] - */ -void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, - int seq, int n_heads, int head_dim); - -/* Streaming argmax: finds argmax(W_bf16 @ x) without materializing full logits. - * Returns the index of the row with highest dot product. */ -int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, - int in_dim, int out_dim); - -/* ======================================================================== - * Threading - * ======================================================================== */ - -/* Set number of threads for parallel operations (default: 1). - * Creates a persistent thread pool. Call before inference. */ -void qwen_set_threads(int n); - -/* Get number of available CPU cores */ -int qwen_get_num_cpus(void); - -/* Global verbose flag */ -extern int qwen_verbose; - -#endif /* QWEN_ASR_KERNELS_H */ diff --git a/qwen_asr_kernels_avx.c b/qwen_asr_kernels_avx.c deleted file mode 100644 index 3fa2194..0000000 --- a/qwen_asr_kernels_avx.c +++ /dev/null @@ -1,502 +0,0 @@ -/* - * qwen_asr_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) - * - * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. - * Uses AVX-512F+BW for 16-wide bf16 matvec/argmax (dominant for seq_len==1), and - * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). - * - * The bf16 matvec processes 4 output rows simultaneously to reduce instruction - * overhead and improve out-of-order execution on memory-bound workloads. - */ - -#include "qwen_asr_kernels_impl.h" - -#if defined(__AVX2__) && defined(__FMA__) - -#include -#include - -/* ===================================================================== - * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise - * ===================================================================== */ - -#if defined(__AVX512F__) && defined(__AVX512BW__) - -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - int o = 0; - - /* Process 4 output rows at a time */ - for (; o + 3 < out_dim; o += 4) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = w0 + in_dim; - const uint16_t *w2 = w1 + in_dim; - const uint16_t *w3 = w2 + in_dim; - - __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); - __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); - __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps(); - __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps(); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - __m512 xv0 = _mm512_loadu_ps(x + k); - __m512 xv1 = _mm512_loadu_ps(x + k + 16); - - /* Row 0 */ - __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); - __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); - a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); - - /* Row 1 */ - __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); - __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); - a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); - - /* Row 2 */ - __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k)); - __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16)); - a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4); - a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5); - - /* Row 3 */ - __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k)); - __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16)); - a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6); - a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7); - } - - for (; k + 16 <= in_dim; k += 16) { - __m512 xv = _mm512_loadu_ps(x + k); - __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); - __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); - __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k)); - __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); - a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4); - a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6); - } - - float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); - float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); - float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f); - float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f); - - for (; k < in_dim; k++) { - float xk = x[k]; - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; - float v0,v1,v2,v3; - memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); - s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; - } - y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3; - } - - /* Handle 2-row remainder */ - for (; o + 1 < out_dim; o += 2) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = w0 + in_dim; - __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); - __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - __m512 xv0 = _mm512_loadu_ps(x + k); - __m512 xv1 = _mm512_loadu_ps(x + k + 16); - __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); - __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); - __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); - __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); - a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); - a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); - } - for (; k + 16 <= in_dim; k += 16) { - __m512 xv = _mm512_loadu_ps(x + k); - __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); - __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); - a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); - a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); - } - - float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); - float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); - - for (; k < in_dim; k++) { - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); - s0+=v0*x[k]; s1+=v1*x[k]; - } - y[o]=s0; y[o+1]=s1; - } - - /* Single remaining row */ - for (; o < out_dim; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m512 acc = _mm512_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); - acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), - _mm512_loadu_ps(x + k), acc); - } - float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum += wv*x[k]; - } - y[o] = sum; - } -} - -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - int o = start; - - /* Process 4 rows at a time */ - for (; o + 3 < end; o += 4) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = w0 + in_dim; - const uint16_t *w2 = w1 + in_dim; - const uint16_t *w3 = w2 + in_dim; - - __m512 a0=_mm512_setzero_ps(), a1=_mm512_setzero_ps(); - __m512 a2=_mm512_setzero_ps(), a3=_mm512_setzero_ps(); - __m512 a4=_mm512_setzero_ps(), a5=_mm512_setzero_ps(); - __m512 a6=_mm512_setzero_ps(), a7=_mm512_setzero_ps(); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - __m512 xv0 = _mm512_loadu_ps(x + k); - __m512 xv1 = _mm512_loadu_ps(x + k + 16); - __m256i r0a=_mm256_loadu_si256((const __m256i*)(w0+k)); - __m256i r0b=_mm256_loadu_si256((const __m256i*)(w0+k+16)); - a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a),16)),xv0,a0); - a1=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b),16)),xv1,a1); - __m256i r1a=_mm256_loadu_si256((const __m256i*)(w1+k)); - __m256i r1b=_mm256_loadu_si256((const __m256i*)(w1+k+16)); - a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a),16)),xv0,a2); - a3=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b),16)),xv1,a3); - __m256i r2a=_mm256_loadu_si256((const __m256i*)(w2+k)); - __m256i r2b=_mm256_loadu_si256((const __m256i*)(w2+k+16)); - a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a),16)),xv0,a4); - a5=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b),16)),xv1,a5); - __m256i r3a=_mm256_loadu_si256((const __m256i*)(w3+k)); - __m256i r3b=_mm256_loadu_si256((const __m256i*)(w3+k+16)); - a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a),16)),xv0,a6); - a7=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b),16)),xv1,a7); - } - for (; k + 16 <= in_dim; k += 16) { - __m512 xv = _mm512_loadu_ps(x + k); - __m256i r0=_mm256_loadu_si256((const __m256i*)(w0+k)); - __m256i r1=_mm256_loadu_si256((const __m256i*)(w1+k)); - __m256i r2=_mm256_loadu_si256((const __m256i*)(w2+k)); - __m256i r3=_mm256_loadu_si256((const __m256i*)(w3+k)); - a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0),16)),xv,a0); - a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1),16)),xv,a2); - a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2),16)),xv,a4); - a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3),16)),xv,a6); - } - - float s0=_mm512_reduce_add_ps(_mm512_add_ps(a0,a1)); - float s1=_mm512_reduce_add_ps(_mm512_add_ps(a2,a3)); - float s2=_mm512_reduce_add_ps(_mm512_add_ps(a4,a5)); - float s3=_mm512_reduce_add_ps(_mm512_add_ps(a6,a7)); - - for (; k < in_dim; k++) { - float xk=x[k]; - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; - float v0,v1,v2,v3; - memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); - s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; - } - - if (s0>best_val){best_val=s0;best=o;} - if (s1>best_val){best_val=s1;best=o+1;} - if (s2>best_val){best_val=s2;best=o+2;} - if (s3>best_val){best_val=s3;best=o+3;} - } - - /* Remaining 1-3 rows */ - for (; o < end; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m512 acc = _mm512_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); - acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), - _mm512_loadu_ps(x + k), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum += wv*x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - - *best_out = best; - *best_val_out = best_val; -} - -#else /* AVX2 only (or AVX-512F without BW) */ - -/* Helper: bf16 to f32 for 8 elements */ -static inline __m256 bf16x8_to_f32(__m128i raw) { - return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16)); -} - -static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { - __m256i raw = _mm256_loadu_si256((const __m256i *)src); - *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw)); - *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); -} - -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - int o = 0; - - for (; o + 1 < out_dim; o += 2) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = w0 + in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); - __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256 xlo = _mm256_loadu_ps(x + k); - __m256 xhi = _mm256_loadu_ps(x + k + 8); - __m256 wlo, whi; - bf16x16_to_f32(w0 + k, &wlo, &whi); - a0 = _mm256_fmadd_ps(wlo, xlo, a0); - a1 = _mm256_fmadd_ps(whi, xhi, a1); - bf16x16_to_f32(w1 + k, &wlo, &whi); - a2 = _mm256_fmadd_ps(wlo, xlo, a2); - a3 = _mm256_fmadd_ps(whi, xhi, a3); - } - a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3); - __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); - __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); - r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0); - float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f); - float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f); - for (; k < in_dim; k++) { - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); - s0+=v0*x[k]; s1+=v1*x[k]; - } - y[o]=s0; y[o+1]=s1; - } - for (; o < out_dim; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256 wlo, whi; - bf16x16_to_f32(w + k, &wlo, &whi); - a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); - a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); - } - a0 = _mm256_add_ps(a0, a1); - __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); - r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum+=wv*x[k]; - } - y[o] = sum; - } -} - -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - - for (int o = start; o < end; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256 wlo, whi; - bf16x16_to_f32(w + k, &wlo, &whi); - a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); - a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); - } - a0 = _mm256_add_ps(a0, a1); - __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); - r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum+=wv*x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - *best_out = best; - *best_val_out = best_val; -} - -#endif /* AVX-512F+BW vs AVX2 for bf16 */ - -/* ===================================================================== - * f32 attention helpers - AVX2+FMA, with AVX-512F when available - * (operates on L1-resident head vectors) - * ===================================================================== */ - -float qwen_dot_f32_avx(const float *a, const float *b, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 acc0 = _mm512_setzero_ps(); - __m512 acc1 = _mm512_setzero_ps(); - __m512 acc2 = _mm512_setzero_ps(); - __m512 acc3 = _mm512_setzero_ps(); - for (; i + 64 <= n; i += 64) { - acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); - acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); - acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); - acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); - } - __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); - for (; i + 16 <= n; i += 16) { - acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; i < n; i++) sum += a[i] * b[i]; - return sum; -#else - int i = 0; - __m256 acc0 = _mm256_setzero_ps(); - __m256 acc1 = _mm256_setzero_ps(); - __m256 acc2 = _mm256_setzero_ps(); - __m256 acc3 = _mm256_setzero_ps(); - for (; i + 32 <= n; i += 32) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); - acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1); - acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2); - acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3); - } - acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); - for (; i + 8 <= n; i += 8) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); - } - __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); - r = _mm_hadd_ps(r, r); - r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; i < n; i++) sum += a[i] * b[i]; - return sum; -#endif -} - -void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 s = _mm512_set1_ps(scale); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); - _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); - _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - } - for (; i < n; i++) dst[i] *= scale; -#else - int i = 0; - __m256 s = _mm256_set1_ps(scale); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); - _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s)); - _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s)); - _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s)); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); - } - for (; i < n; i++) dst[i] *= scale; -#endif -} - -void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 a = _mm512_set1_ps(alpha); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - _mm512_storeu_ps(dst + i + 16, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16))); - _mm512_storeu_ps(dst + i + 32, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32))); - _mm512_storeu_ps(dst + i + 48, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - } - for (; i < n; i++) dst[i] += alpha * src[i]; -#else - int i = 0; - __m256 a = _mm256_set1_ps(alpha); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); - _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8))); - _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16))); - _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); - } - for (; i < n; i++) dst[i] += alpha * src[i]; -#endif -} - -void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 c = _mm512_set1_ps(correction); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - _mm512_storeu_ps(dst + i + 16, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16))); - _mm512_storeu_ps(dst + i + 32, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32))); - _mm512_storeu_ps(dst + i + 48, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; -#else - int i = 0; - __m256 c = _mm256_set1_ps(correction); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); - _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8))); - _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16))); - _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); - } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; -#endif -} - -#endif /* __AVX2__ && __FMA__ */ diff --git a/qwen_asr_kernels_generic.c b/qwen_asr_kernels_generic.c deleted file mode 100644 index d322d8e..0000000 --- a/qwen_asr_kernels_generic.c +++ /dev/null @@ -1,65 +0,0 @@ -/* - * qwen_asr_kernels_generic.c - architecture-generic hot kernels - */ - -#include "qwen_asr_kernels_impl.h" - -#include - -void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - for (int o = 0; o < out_dim; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = bias ? bias[o] : 0.0f; - for (int k = 0; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - y[o] = sum; - } -} - -void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - - for (int o = start; o < end; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = 0.0f; - for (int k = 0; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - if (sum > best_val) { - best_val = sum; - best = o; - } - } - - *best_out = best; - *best_val_out = best_val; -} - -float qwen_dot_f32_generic(const float *a, const float *b, int n) { - float sum = 0.0f; - for (int i = 0; i < n; i++) sum += a[i] * b[i]; - return sum; -} - -void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) { - for (int i = 0; i < n; i++) dst[i] *= scale; -} - -void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { - for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; -} - -void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { - for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; -} diff --git a/qwen_asr_kernels_impl.h b/qwen_asr_kernels_impl.h deleted file mode 100644 index 31b9cbb..0000000 --- a/qwen_asr_kernels_impl.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * qwen_asr_kernels_impl.h - internal architecture dispatch for hot kernels - */ - -#ifndef QWEN_ASR_KERNELS_IMPL_H -#define QWEN_ASR_KERNELS_IMPL_H - -#include - -void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); -float qwen_dot_f32_generic(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n); - -#ifdef __ARM_NEON -void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); -float qwen_dot_f32_neon(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); - -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_neon -#define qwen_dot_f32_impl qwen_dot_f32_neon -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon -#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon - -#elif defined(__AVX2__) && defined(__FMA__) -void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); -float qwen_dot_f32_avx(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); - -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_avx -#define qwen_dot_f32_impl qwen_dot_f32_avx -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx -#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx - -#else -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_generic -#define qwen_dot_f32_impl qwen_dot_f32_generic -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic -#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic -#endif - -#endif /* QWEN_ASR_KERNELS_IMPL_H */ diff --git a/qwen_asr_kernels_neon.c b/qwen_asr_kernels_neon.c deleted file mode 100644 index 90d8ed9..0000000 --- a/qwen_asr_kernels_neon.c +++ /dev/null @@ -1,274 +0,0 @@ -/* - * qwen_asr_kernels_neon.c - ARM NEON hot kernels - */ - -#include "qwen_asr_kernels_impl.h" - -#ifdef __ARM_NEON - -#include -#include - -void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, - const float *bias, int in_dim, int out_dim) { - int o = 0; - - /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */ - for (; o + 1 < out_dim; o += 2) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; - float s0 = bias ? bias[o] : 0.0f; - float s1 = bias ? bias[o + 1] : 0.0f; - - float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); - float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); - float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); - float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - float32x4_t x0 = vld1q_f32(x + k); - float32x4_t x1 = vld1q_f32(x + k + 4); - float32x4_t x2 = vld1q_f32(x + k + 8); - float32x4_t x3 = vld1q_f32(x + k + 12); - float32x4_t x4 = vld1q_f32(x + k + 16); - float32x4_t x5 = vld1q_f32(x + k + 20); - float32x4_t x6 = vld1q_f32(x + k + 24); - float32x4_t x7 = vld1q_f32(x + k + 28); - - uint16x8_t r0a = vld1q_u16(w0 + k); - uint16x8_t r0b = vld1q_u16(w0 + k + 8); - uint16x8_t r0c = vld1q_u16(w0 + k + 16); - uint16x8_t r0d = vld1q_u16(w0 + k + 24); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); - - uint16x8_t r1a = vld1q_u16(w1 + k); - uint16x8_t r1b = vld1q_u16(w1 + k + 8); - uint16x8_t r1c = vld1q_u16(w1 + k + 16); - uint16x8_t r1d = vld1q_u16(w1 + k + 24); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); - } - for (; k + 8 <= in_dim; k += 8) { - float32x4_t x0 = vld1q_f32(x + k); - float32x4_t x1 = vld1q_f32(x + k + 4); - uint16x8_t r0 = vld1q_u16(w0 + k); - uint16x8_t r1 = vld1q_u16(w1 + k); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0), 16)), x0); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0), 16)), x1); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1), 16)), x0); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1), 16)), x1); - } - s0 += vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); - s1 += vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); - - for (; k < in_dim; k++) { - uint32_t bits0 = ((uint32_t)w0[k]) << 16; - uint32_t bits1 = ((uint32_t)w1[k]) << 16; - float wv0, wv1; - memcpy(&wv0, &bits0, sizeof(float)); - memcpy(&wv1, &bits1, sizeof(float)); - s0 += wv0 * x[k]; - s1 += wv1 * x[k]; - } - y[o] = s0; - y[o + 1] = s1; - } - - /* Handle remaining odd row */ - for (; o < out_dim; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = bias ? bias[o] : 0.0f; - int k = 0; - - float32x4_t acc0 = vdupq_n_f32(0.0f); - float32x4_t acc1 = vdupq_n_f32(0.0f); - for (; k + 8 <= in_dim; k += 8) { - uint16x8_t bf = vld1q_u16(w_row + k); - acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), - vld1q_f32(x + k)); - acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), - vld1q_f32(x + k + 4)); - } - sum += vaddvq_f32(vaddq_f32(acc0, acc1)); - - for (; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - y[o] = sum; - } -} - -void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - int o = start; - - /* Process 2 rows at a time, 32 elements/iter, 8 accumulators per row */ - for (; o + 1 < end; o += 2) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; - float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); - float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); - float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); - float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - float32x4_t x0 = vld1q_f32(x + k); - float32x4_t x1 = vld1q_f32(x + k + 4); - float32x4_t x2 = vld1q_f32(x + k + 8); - float32x4_t x3 = vld1q_f32(x + k + 12); - float32x4_t x4 = vld1q_f32(x + k + 16); - float32x4_t x5 = vld1q_f32(x + k + 20); - float32x4_t x6 = vld1q_f32(x + k + 24); - float32x4_t x7 = vld1q_f32(x + k + 28); - - uint16x8_t r0a = vld1q_u16(w0 + k); - uint16x8_t r0b = vld1q_u16(w0 + k + 8); - uint16x8_t r0c = vld1q_u16(w0 + k + 16); - uint16x8_t r0d = vld1q_u16(w0 + k + 24); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); - - uint16x8_t r1a = vld1q_u16(w1 + k); - uint16x8_t r1b = vld1q_u16(w1 + k + 8); - uint16x8_t r1c = vld1q_u16(w1 + k + 16); - uint16x8_t r1d = vld1q_u16(w1 + k + 24); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); - } - - float s0 = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); - float s1 = vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); - - for (; k < in_dim; k++) { - uint32_t bits0 = ((uint32_t)w0[k]) << 16; - uint32_t bits1 = ((uint32_t)w1[k]) << 16; - float wv0, wv1; - memcpy(&wv0, &bits0, sizeof(float)); - memcpy(&wv1, &bits1, sizeof(float)); - s0 += wv0 * x[k]; - s1 += wv1 * x[k]; - } - - if (s0 > best_val) { best_val = s0; best = o; } - if (s1 > best_val) { best_val = s1; best = o + 1; } - } - - for (; o < end; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = 0.0f; - int k = 0; - - float32x4_t acc0 = vdupq_n_f32(0.0f), acc1 = vdupq_n_f32(0.0f); - for (; k + 8 <= in_dim; k += 8) { - uint16x8_t bf = vld1q_u16(w_row + k); - acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), - vld1q_f32(x + k)); - acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), - vld1q_f32(x + k + 4)); - } - sum += vaddvq_f32(vaddq_f32(acc0, acc1)); - - for (; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - - *best_out = best; - *best_val_out = best_val; -} - -float qwen_dot_f32_neon(const float *a, const float *b, int n) { - int i = 0; - float32x4_t acc0 = vdupq_n_f32(0.0f); - float32x4_t acc1 = vdupq_n_f32(0.0f); - for (; i + 8 <= n; i += 8) { - float32x4_t a0 = vld1q_f32(a + i); - float32x4_t b0 = vld1q_f32(b + i); - float32x4_t a1 = vld1q_f32(a + i + 4); - float32x4_t b1 = vld1q_f32(b + i + 4); - acc0 = vfmaq_f32(acc0, a0, b0); - acc1 = vfmaq_f32(acc1, a1, b1); - } - float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); - for (; i < n; i++) sum += a[i] * b[i]; - return sum; -} - -void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { - int i = 0; - float32x4_t s = vdupq_n_f32(scale); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); - vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); - } - for (; i < n; i++) dst[i] *= scale; -} - -void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { - int i = 0; - float32x4_t a = vdupq_n_f32(alpha); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); - vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); - } - for (; i < n; i++) dst[i] += alpha * src[i]; -} - -void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { - int i = 0; - float32x4_t c = vdupq_n_f32(correction); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); - vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); - } - for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; -} - -#endif /* __ARM_NEON */ diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c deleted file mode 100644 index 726ee9c..0000000 --- a/qwen_asr_safetensors.c +++ /dev/null @@ -1,481 +0,0 @@ -#include "qwen_asr_safetensors.h" -#include -#include -#include -#include -#include -#include -#include -#include - -/* ======================================================================== - * Minimal JSON parser for safetensors header - * ======================================================================== */ - -static void skip_whitespace(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; -} - -static int parse_string(const char **p, char *out, size_t max_len) { - skip_whitespace(p); - if (**p != '"') return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') out[i++] = '\n'; - else if (**p == 't') out[i++] = '\t'; - else if (**p == '"') out[i++] = '"'; - else if (**p == '\\') out[i++] = '\\'; - else out[i++] = **p; - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') return -1; - (*p)++; - return 0; -} - -static int parse_int(const char **p, int64_t *out) { - skip_whitespace(p); - int64_t val = 0; - int neg = 0; - if (**p == '-') { neg = 1; (*p)++; } - if (**p < '0' || **p > '9') return -1; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - *out = neg ? -val : val; - return 0; -} - -static safetensor_dtype_t parse_dtype(const char *s) { - if (strcmp(s, "F32") == 0) return DTYPE_F32; - if (strcmp(s, "F16") == 0) return DTYPE_F16; - if (strcmp(s, "BF16") == 0) return DTYPE_BF16; - if (strcmp(s, "I32") == 0) return DTYPE_I32; - if (strcmp(s, "I64") == 0) return DTYPE_I64; - if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL; - return DTYPE_UNKNOWN; -} - -static int parse_tensor_entry(const char **p, safetensor_t *t) { - skip_whitespace(p); - if (**p != '{') return -1; - (*p)++; - - t->dtype = DTYPE_UNKNOWN; - t->ndim = 0; - t->data_offset = 0; - t->data_size = 0; - - while (**p && **p != '}') { - skip_whitespace(p); - if (**p == ',') { (*p)++; continue; } - - char key[64]; - if (parse_string(p, key, sizeof(key)) != 0) return -1; - skip_whitespace(p); - if (**p != ':') return -1; - (*p)++; - skip_whitespace(p); - - if (strcmp(key, "dtype") == 0) { - char dtype_str[32]; - if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1; - t->dtype = parse_dtype(dtype_str); - } else if (strcmp(key, "shape") == 0) { - if (**p != '[') return -1; - (*p)++; - t->ndim = 0; - while (**p && **p != ']') { - skip_whitespace(p); - if (**p == ',') { (*p)++; continue; } - if (t->ndim >= 8) return -1; - if (parse_int(p, &t->shape[t->ndim]) != 0) return -1; - t->ndim++; - } - if (**p != ']') return -1; - (*p)++; - } else if (strcmp(key, "data_offsets") == 0) { - if (**p != '[') return -1; - (*p)++; - skip_whitespace(p); - int64_t start_i; - if (parse_int(p, &start_i) != 0) return -1; - skip_whitespace(p); - if (**p != ',') return -1; - (*p)++; - skip_whitespace(p); - int64_t end_i; - if (parse_int(p, &end_i) != 0) return -1; - if (start_i < 0 || end_i < start_i) return -1; - t->data_offset = (size_t)start_i; - t->data_size = (size_t)(end_i - start_i); - skip_whitespace(p); - if (**p != ']') return -1; - (*p)++; - } else { - /* Skip unknown value */ - if (**p == '"') { - (*p)++; - while (**p && **p != '"') { - if (**p == '\\') (*p)++; - if (**p) (*p)++; - } - if (**p == '"') (*p)++; - } else if (**p == '[') { - int depth = 1; (*p)++; - while (**p && depth > 0) { - if (**p == '[') depth++; - else if (**p == ']') depth--; - (*p)++; - } - } else if (**p == '{') { - int depth = 1; (*p)++; - while (**p && depth > 0) { - if (**p == '{') depth++; - else if (**p == '}') depth--; - (*p)++; - } - } else { - while (**p && **p != ',' && **p != '}') (*p)++; - } - } - } - if (**p != '}') return -1; - (*p)++; - return 0; -} - -static int parse_header(safetensors_file_t *sf) { - const char *p = sf->header_json; - skip_whitespace(&p); - if (*p != '{') return -1; - p++; - - sf->num_tensors = 0; - - while (*p && *p != '}') { - skip_whitespace(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; - - char name[256]; - if (parse_string(&p, name, sizeof(name)) != 0) return -1; - skip_whitespace(&p); - if (*p != ':') return -1; - p++; - - if (strcmp(name, "__metadata__") == 0) { - skip_whitespace(&p); - if (*p == '{') { - int depth = 1; p++; - while (*p && depth > 0) { - if (*p == '{') depth++; - else if (*p == '}') depth--; - p++; - } - } - continue; - } - - if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { - fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", - sf->path ? sf->path : "", SAFETENSORS_MAX_TENSORS); - return -1; - } - - safetensor_t *t = &sf->tensors[sf->num_tensors]; - snprintf(t->name, sizeof(t->name), "%s", name); - if (parse_tensor_entry(&p, t) != 0) return -1; - sf->num_tensors++; - } - if (*p != '}') return -1; - return 0; -} - -/* ======================================================================== - * Single file operations - * ======================================================================== */ - -safetensors_file_t *safetensors_open(const char *path) { - int fd = open(path, O_RDONLY); - if (fd < 0) return NULL; - - struct stat st; - if (fstat(fd, &st) < 0) { close(fd); return NULL; } - - size_t file_size = (size_t)st.st_size; - if (file_size < 8) { close(fd); return NULL; } - - void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - if (data == MAP_FAILED) return NULL; - - uint64_t header_size = 0; - memcpy(&header_size, data, 8); - if (header_size > file_size - 8) { munmap(data, file_size); return NULL; } - - safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); - if (!sf) { munmap(data, file_size); return NULL; } - - sf->path = strdup(path); - sf->data = data; - sf->file_size = file_size; - sf->header_size = (size_t)header_size; - - sf->header_json = malloc(header_size + 1); - if (!sf->header_json) { safetensors_close(sf); return NULL; } - memcpy(sf->header_json, (char *)data + 8, header_size); - sf->header_json[header_size] = '\0'; - - if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } - - return sf; -} - -void safetensors_close(safetensors_file_t *sf) { - if (!sf) return; - if (sf->data) munmap(sf->data, sf->file_size); - free(sf->path); - free(sf->header_json); - free(sf); -} - -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { - return (const char *)sf->data + 8 + sf->header_size + t->data_offset; -} - -int64_t safetensor_numel(const safetensor_t *t) { - int64_t n = 1; - for (int i = 0; i < t->ndim; i++) n *= t->shape[i]; - return n; -} - -static float bf16_to_f32(uint16_t bf16) { - uint32_t f32 = ((uint32_t)bf16) << 16; - float result; - memcpy(&result, &f32, sizeof(float)); - return result; -} - -float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { - int64_t n = safetensor_numel(t); - if (n <= 0) return NULL; - - float *out = malloc(n * sizeof(float)); - if (!out) return NULL; - - const void *data = safetensors_data(sf, t); - - switch (t->dtype) { - case DTYPE_F32: - memcpy(out, data, n * sizeof(float)); - break; - case DTYPE_BF16: { - const uint16_t *src = (const uint16_t *)data; - for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]); - break; - } - default: - free(out); - return NULL; - } - return out; -} - -int safetensor_is_bf16(const safetensor_t *t) { - return t && t->dtype == DTYPE_BF16; -} - -uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t) { - if (!sf || !t || t->dtype != DTYPE_BF16) return NULL; - return (uint16_t *)safetensors_data(sf, t); -} - -void safetensor_print(const safetensor_t *t) { - const char *dtype_names[] = {"F32", "F16", "BF16", "I32", "I64", "BOOL"}; - const char *dtype_name = t->dtype >= 0 && t->dtype <= 5 ? - dtype_names[t->dtype] : "UNKNOWN"; - printf("%s: dtype=%s, shape=[", t->name, dtype_name); - for (int i = 0; i < t->ndim; i++) { - printf("%ld%s", (long)t->shape[i], i < t->ndim - 1 ? ", " : ""); - } - printf("]\n"); -} - -void safetensors_print_all(const safetensors_file_t *sf) { - printf("File: %s (%d tensors)\n", sf->path, sf->num_tensors); - for (int i = 0; i < sf->num_tensors; i++) safetensor_print(&sf->tensors[i]); -} - -/* ======================================================================== - * Multi-shard operations - * ======================================================================== */ - -static int cmp_shard_names(const void *a, const void *b) { - const char *sa = (const char *)a; - const char *sb = (const char *)b; - return strcmp(sa, sb); -} - -static int parse_shard_name(const char *name, int *idx, int *total) { - int i = 0, n = 0, consumed = 0; - if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) - return -1; - if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) - return -1; - *idx = i; - *total = n; - return 0; -} - -multi_safetensors_t *multi_safetensors_open(const char *model_dir) { - multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); - if (!ms) return NULL; - - char path[4096]; - - /* Try single file first */ - snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); - safetensors_file_t *sf = safetensors_open(path); - if (sf) { - ms->shards[0] = sf; - ms->num_shards = 1; - return ms; - } - - /* Scan directory for shard files */ - DIR *dir = opendir(model_dir); - if (!dir) { free(ms); return NULL; } - - struct dirent *entry; - char shard_names[SAFETENSORS_MAX_SHARDS][256]; - int seen[SAFETENSORS_MAX_SHARDS] = {0}; - int n_shards = 0; - int expected_total = 0; - - while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "model-", 6) == 0 && - strstr(entry->d_name, ".safetensors") != NULL) { - int shard_idx = 0, shard_total = 0; - if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { - fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", - entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (shard_total > SAFETENSORS_MAX_SHARDS) { - fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", - shard_total, SAFETENSORS_MAX_SHARDS); - closedir(dir); - free(ms); - return NULL; - } - if (expected_total == 0) { - expected_total = shard_total; - } else if (expected_total != shard_total) { - fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", - entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (seen[shard_idx - 1]) { - fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", - shard_idx); - closedir(dir); - free(ms); - return NULL; - } - seen[shard_idx - 1] = 1; - - snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), - "%s", entry->d_name); - n_shards++; - } - } - closedir(dir); - - if (n_shards == 0) { - fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); - free(ms); - return NULL; - } - if (expected_total != n_shards) { - fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", - expected_total, n_shards, model_dir); - free(ms); - return NULL; - } - for (int i = 0; i < expected_total; i++) { - if (!seen[i]) { - fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", - i + 1, expected_total, model_dir); - free(ms); - return NULL; - } - } - - /* Sort shard names to ensure consistent ordering */ - qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); - - /* Open each shard */ - for (int i = 0; i < n_shards; i++) { - snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); - ms->shards[i] = safetensors_open(path); - if (!ms->shards[i]) { - fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); - multi_safetensors_close(ms); - return NULL; - } - } - ms->num_shards = n_shards; - return ms; -} - -void multi_safetensors_close(multi_safetensors_t *ms) { - if (!ms) return; - for (int i = 0; i < ms->num_shards; i++) { - safetensors_close(ms->shards[i]); - } - free(ms); -} - -int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, - size_t *out_nbytes) { - if (!ms || !out_nbytes) return -1; - size_t total = 0; - for (int s = 0; s < ms->num_shards; s++) { - const safetensors_file_t *sf = ms->shards[s]; - for (int i = 0; i < sf->num_tensors; i++) { - size_t n = sf->tensors[i].data_size; - if (n > SIZE_MAX - total) return -1; - total += n; - } - } - *out_nbytes = total; - return 0; -} - -const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf) { - for (int s = 0; s < ms->num_shards; s++) { - safetensors_file_t *sf = ms->shards[s]; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) { - if (out_sf) *out_sf = sf; - return &sf->tensors[i]; - } - } - } - return NULL; -} diff --git a/qwen_asr_safetensors.h b/qwen_asr_safetensors.h deleted file mode 100644 index 7bd1846..0000000 --- a/qwen_asr_safetensors.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * qwen_asr_safetensors.h - Safetensors file format reader (multi-shard support) - * Adapted from voxtral-realtime project. - */ - -#ifndef QWEN_ASR_SAFETENSORS_H -#define QWEN_ASR_SAFETENSORS_H - -#include -#include - -#define SAFETENSORS_MAX_TENSORS 1024 -#define SAFETENSORS_MAX_SHARDS 8 - -typedef enum { - DTYPE_F32 = 0, - DTYPE_F16 = 1, - DTYPE_BF16 = 2, - DTYPE_I32 = 3, - DTYPE_I64 = 4, - DTYPE_BOOL = 5, - DTYPE_UNKNOWN = -1 -} safetensor_dtype_t; - -typedef struct { - char name[256]; - safetensor_dtype_t dtype; - int ndim; - int64_t shape[8]; - size_t data_offset; - size_t data_size; -} safetensor_t; - -typedef struct { - char *path; - void *data; - size_t file_size; - size_t header_size; - char *header_json; - int num_tensors; - safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; -} safetensors_file_t; - -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ -typedef struct { - safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; - int num_shards; -} multi_safetensors_t; - -/* Open a single safetensors file (memory-mapped) */ -safetensors_file_t *safetensors_open(const char *path); -void safetensors_close(safetensors_file_t *sf); - -/* Open model from directory (auto-detects single file or multi-shard) */ -multi_safetensors_t *multi_safetensors_open(const char *model_dir); -void multi_safetensors_close(multi_safetensors_t *ms); -int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, - size_t *out_nbytes); - -/* Find a tensor by name across all shards */ -const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf); - -/* Get raw pointer to tensor data (within mmap'd region) */ -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); - -/* Get tensor data as float32 (allocates, caller must free) */ -float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); - -/* Get direct pointer to bf16 data in mmap'd region (no copy) */ -uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t); - -int safetensor_is_bf16(const safetensor_t *t); -int64_t safetensor_numel(const safetensor_t *t); -void safetensor_print(const safetensor_t *t); -void safetensors_print_all(const safetensors_file_t *sf); - -#endif /* QWEN_ASR_SAFETENSORS_H */ diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c deleted file mode 100644 index fa85017..0000000 --- a/qwen_asr_tokenizer.c +++ /dev/null @@ -1,1523 +0,0 @@ -#include "qwen_asr_tokenizer.h" -#include "qwen_asr_kernels.h" -#include -#include -#include -#include -#include - -/* ======================================================================== - * GPT-2 Bytes-to-Unicode Mapping - * ======================================================================== */ - -static int gpt2_byte_to_unicode[256]; -static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ -static int gpt2_mapping_initialized = 0; - -static void init_gpt2_mapping(void) { - if (gpt2_mapping_initialized) return; - - memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); - - int n = 0; - for (int b = 0; b < 256; b++) { - int is_normal = 0; - if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */ - if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */ - if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */ - - if (is_normal) gpt2_byte_to_unicode[b] = b; - else gpt2_byte_to_unicode[b] = 256 + n++; - } - - for (int b = 0; b < 256; b++) { - int cp = gpt2_byte_to_unicode[b]; - if (cp < 512) gpt2_unicode_to_byte[cp] = b; - } - - gpt2_mapping_initialized = 1; -} - -/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */ -static int utf8_encode_cp(int cp, char out[4]) { - if (cp < 0x80) { - out[0] = (char)cp; - return 1; - } - if (cp < 0x800) { - out[0] = (char)(0xC0 | (cp >> 6)); - out[1] = (char)(0x80 | (cp & 0x3F)); - return 2; - } - out[0] = (char)(0xE0 | (cp >> 12)); - out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[2] = (char)(0x80 | (cp & 0x3F)); - return 3; -} - -static int utf8_encode_full_cp(int cp, char out[4]) { - if (cp < 0x80) return utf8_encode_cp(cp, out); - if (cp < 0x800) return utf8_encode_cp(cp, out); - if (cp < 0x10000) return utf8_encode_cp(cp, out); - out[0] = (char)(0xF0 | (cp >> 18)); - out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); - out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[3] = (char)(0x80 | (cp & 0x3F)); - return 4; -} - -/* - * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text. - * Returns allocated string (caller must free). - */ -static char *decode_gpt2_token(const char *token_str) { - init_gpt2_mapping(); - - size_t len = strlen(token_str); - unsigned char *bytes = (unsigned char *)malloc(len + 1); - if (!bytes) return NULL; - - int byte_count = 0; - const unsigned char *p = (const unsigned char *)token_str; - const unsigned char *end = p + len; - - while (p < end) { - int cp = 0; - int nbytes = 1; - - if ((*p & 0x80) == 0) { - cp = *p; - nbytes = 1; - } else if ((*p & 0xE0) == 0xC0) { - cp = (*p & 0x1F) << 6; - if (p + 1 < end) cp |= (p[1] & 0x3F); - nbytes = 2; - } else if ((*p & 0xF0) == 0xE0) { - cp = (*p & 0x0F) << 12; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 6; - if (p + 2 < end) cp |= (p[2] & 0x3F); - nbytes = 3; - } else if ((*p & 0xF8) == 0xF0) { - cp = (*p & 0x07) << 18; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 12; - if (p + 2 < end) cp |= (p[2] & 0x3F) << 6; - if (p + 3 < end) cp |= (p[3] & 0x3F); - nbytes = 4; - } else { - cp = *p; - nbytes = 1; - } - p += nbytes; - - if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) { - bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp]; - } else { - bytes[byte_count++] = '?'; - } - } - bytes[byte_count] = '\0'; - return (char *)bytes; -} - -/* ======================================================================== - * Simple JSON parser for vocab.json - * ======================================================================== */ - -static void skip_ws(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; -} - -static int parse_json_string(const char **p, char *out, size_t max_len) { - skip_ws(p); - if (**p != '"') return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') out[i++] = '\n'; - else if (**p == 't') out[i++] = '\t'; - else if (**p == '"') out[i++] = '"'; - else if (**p == '\\') out[i++] = '\\'; - else if (**p == '/') out[i++] = '/'; - else if (**p == 'u') { - (*p)++; - unsigned int cp = 0; - for (int j = 0; j < 4 && **p; j++, (*p)++) { - cp <<= 4; - char c = **p; - if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0'); - else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10); - else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10); - } - if (cp < 0x80 && i + 1 < max_len) { - out[i++] = (char)cp; - } else if (cp < 0x800 && i + 2 < max_len) { - out[i++] = (char)(0xC0 | (cp >> 6)); - out[i++] = (char)(0x80 | (cp & 0x3F)); - } else if (i + 3 < max_len) { - out[i++] = (char)(0xE0 | (cp >> 12)); - out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[i++] = (char)(0x80 | (cp & 0x3F)); - } - continue; - } else { - out[i++] = **p; - } - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') return -1; - (*p)++; - return 0; -} - -static int64_t parse_json_int(const char **p) { - skip_ws(p); - int neg = 0; - if (**p == '-') { neg = 1; (*p)++; } - int64_t val = 0; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - return neg ? -val : val; -} - -/* ======================================================================== - * String->int hash map (open addressing) - * ======================================================================== */ - -typedef struct { - char *key; /* NULL means empty slot */ - int value; -} str_int_entry_t; - -static uint64_t fnv1a_hash(const char *s) { - uint64_t h = 1469598103934665603ULL; - while (*s) { - h ^= (unsigned char)*s++; - h *= 1099511628211ULL; - } - return h; -} - -static uint64_t fnv1a_hash_pair(const char *a, const char *b) { - uint64_t h = 1469598103934665603ULL; - while (*a) { - h ^= (unsigned char)*a++; - h *= 1099511628211ULL; - } - h ^= (unsigned char)' '; - h *= 1099511628211ULL; - while (*b) { - h ^= (unsigned char)*b++; - h *= 1099511628211ULL; - } - return h; -} - -static int pair_key_eq(const char *key, const char *a, const char *b) { - while (*a) { - if (*key++ != *a++) return 0; - } - if (*key++ != ' ') return 0; - while (*b) { - if (*key++ != *b++) return 0; - } - return *key == '\0'; -} - -static int next_pow2(int x) { - int p = 1; - while (p < x) p <<= 1; - return p; -} - -static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { - if (!map || cap <= 0 || !key) return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) { - map[idx].key = key; - map[idx].value = value; - return 0; - } - if (strcmp(map[idx].key, key) == 0) { - map[idx].value = value; - return 0; - } - } - return -1; -} - -static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) { - if (!keyp) return -1; - char *key = *keyp; - if (!map || cap <= 0 || !key) return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) { - map[idx].key = key; - map[idx].value = value; - *keyp = NULL; - return 0; - } - if (strcmp(map[idx].key, key) == 0) { - map[idx].value = value; - return 0; - } - } - return -1; -} - -static int map_get(const str_int_entry_t *map, int cap, const char *key) { - if (!map || cap <= 0 || !key) return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (strcmp(map[idx].key, key) == 0) return map[idx].value; - } - return -1; -} - -static int map_get_pair(const str_int_entry_t *map, int cap, - const char *a, const char *b) { - if (!map || cap <= 0 || !a || !b) return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (pair_key_eq(map[idx].key, a, b)) return map[idx].value; - } - return -1; -} - -/* ======================================================================== - * BPE helpers - * ======================================================================== */ - -static int utf8_char_len(unsigned char c) { - if ((c & 0x80) == 0) return 1; - if ((c & 0xE0) == 0xC0) return 2; - if ((c & 0xF0) == 0xE0) return 3; - if ((c & 0xF8) == 0xF0) return 4; - return 1; -} - -static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { - *out_syms = NULL; - *out_n = 0; - if (!s || !*s) return 0; - - int n = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { - int l = utf8_char_len(*p); - p += l; - n++; - } - if (n <= 0) return 0; - - char **syms = (char **)calloc((size_t)n, sizeof(char *)); - if (!syms) return -1; - - int i = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { - int l = utf8_char_len(*p); - syms[i] = (char *)malloc((size_t)l + 1); - if (!syms[i]) { - for (int j = 0; j < i; j++) free(syms[j]); - free(syms); - return -1; - } - memcpy(syms[i], p, (size_t)l); - syms[i][l] = '\0'; - p += l; - i++; - } - - *out_syms = syms; - *out_n = n; - return 0; -} - -static char *str_concat2(const char *a, const char *b) { - size_t la = strlen(a), lb = strlen(b); - char *r = (char *)malloc(la + lb + 1); - if (!r) return NULL; - memcpy(r, a, la); - memcpy(r + la, b, lb); - r[la + lb] = '\0'; - return r; -} - -static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { - if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; - - int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, a, b); - return rank >= 0 ? rank : INT_MAX; -} - -static int append_id(int **arr, int *n, int *cap, int id) { - if (*n >= *cap) { - int new_cap = (*cap == 0) ? 16 : (*cap * 2); - int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); - if (!tmp) return -1; - *arr = tmp; - *cap = new_cap; - } - (*arr)[(*n)++] = id; - return 0; -} - -/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */ -static char *text_to_bpe_unicode(const char *text) { - init_gpt2_mapping(); - size_t len = strlen(text); - /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ - char *out = (char *)malloc(len * 3 + 1); - if (!out) return NULL; - - size_t w = 0; - for (size_t i = 0; i < len; i++) { - int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; - char tmp[4]; - int n = utf8_encode_cp(cp, tmp); - for (int j = 0; j < n; j++) out[w++] = tmp[j]; - } - out[w] = '\0'; - return out; -} - -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, - int **out_ids, int *out_n); - -static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, - int *cp_out, int *len_out) { - if (p >= end || *p == '\0') return 0; - - unsigned char c = *p; - if ((c & 0x80) == 0) { - *cp_out = c; - *len_out = 1; - return 1; - } - if ((c & 0xE0) == 0xC0 && p + 1 < end && - (p[1] & 0xC0) == 0x80) { - *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F); - *len_out = 2; - return 1; - } - if ((c & 0xF0) == 0xE0 && p + 2 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { - *cp_out = ((c & 0x0F) << 12) | - ((p[1] & 0x3F) << 6) | - (p[2] & 0x3F); - *len_out = 3; - return 1; - } - if ((c & 0xF8) == 0xF0 && p + 3 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && - (p[3] & 0xC0) == 0x80) { - *cp_out = ((c & 0x07) << 18) | - ((p[1] & 0x3F) << 12) | - ((p[2] & 0x3F) << 6) | - (p[3] & 0x3F); - *len_out = 4; - return 1; - } - - *cp_out = c; - *len_out = 1; - return 1; -} - -static int is_ascii_alpha_cp(int cp) { - return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'); -} - -static int is_number_cp(int cp) { - if (cp >= '0' && cp <= '9') return 1; - if (cp >= 0x0660 && cp <= 0x0669) return 1; - if (cp >= 0x06F0 && cp <= 0x06F9) return 1; - if (cp >= 0x0966 && cp <= 0x096F) return 1; - if (cp >= 0xFF10 && cp <= 0xFF19) return 1; - return 0; -} - -static int is_space_cp(int cp) { - return cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r' || - cp == '\f' || cp == '\v' || cp == 0x85 || cp == 0xA0 || - cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || - cp == 0x2028 || cp == 0x2029 || cp == 0x202F || - cp == 0x205F || cp == 0x3000; -} - -static int is_symbolish_cp(int cp) { - if (cp < 0x80) return 0; - if (cp >= 0x0300 && cp <= 0x036F) return 1; - if (cp >= 0x00A1 && cp <= 0x00BF && - cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) - return 1; - if (cp == 0x00D7 || cp == 0x00F7) return 1; - if (cp >= 0x064B && cp <= 0x065F) return 1; - if ((cp >= 0x0900 && cp <= 0x0903) || - (cp >= 0x093A && cp <= 0x094F) || - (cp >= 0x0951 && cp <= 0x0957) || - (cp >= 0x0962 && cp <= 0x0963)) - return 1; - if (cp >= 0x2000 && cp <= 0x206F) return 1; - if (cp >= 0x2190 && cp <= 0x27BF) return 1; - if (cp >= 0x2E00 && cp <= 0x2E7F) return 1; - if (cp >= 0x3000 && cp <= 0x303F) return 1; - if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; - return 0; -} - -static int compose_latin_mark(int base, int mark) { - switch (mark) { - case 0x0300: - switch (base) { - case 'A': return 0x00C0; case 'E': return 0x00C8; - case 'I': return 0x00CC; case 'O': return 0x00D2; - case 'U': return 0x00D9; case 'a': return 0x00E0; - case 'e': return 0x00E8; case 'i': return 0x00EC; - case 'o': return 0x00F2; case 'u': return 0x00F9; - } - break; - case 0x0301: - switch (base) { - case 'A': return 0x00C1; case 'C': return 0x0106; - case 'E': return 0x00C9; case 'I': return 0x00CD; - case 'L': return 0x0139; case 'N': return 0x0143; - case 'O': return 0x00D3; case 'R': return 0x0154; - case 'S': return 0x015A; case 'U': return 0x00DA; - case 'Y': return 0x00DD; case 'Z': return 0x0179; - case 'a': return 0x00E1; case 'c': return 0x0107; - case 'e': return 0x00E9; case 'i': return 0x00ED; - case 'l': return 0x013A; case 'n': return 0x0144; - case 'o': return 0x00F3; case 'r': return 0x0155; - case 's': return 0x015B; case 'u': return 0x00FA; - case 'y': return 0x00FD; case 'z': return 0x017A; - } - break; - case 0x0302: - switch (base) { - case 'A': return 0x00C2; case 'E': return 0x00CA; - case 'I': return 0x00CE; case 'O': return 0x00D4; - case 'U': return 0x00DB; case 'a': return 0x00E2; - case 'e': return 0x00EA; case 'i': return 0x00EE; - case 'o': return 0x00F4; case 'u': return 0x00FB; - } - break; - case 0x0303: - switch (base) { - case 'A': return 0x00C3; case 'N': return 0x00D1; - case 'O': return 0x00D5; case 'a': return 0x00E3; - case 'n': return 0x00F1; case 'o': return 0x00F5; - } - break; - case 0x0308: - switch (base) { - case 'A': return 0x00C4; case 'E': return 0x00CB; - case 'I': return 0x00CF; case 'O': return 0x00D6; - case 'U': return 0x00DC; case 'Y': return 0x0178; - case 'a': return 0x00E4; case 'e': return 0x00EB; - case 'i': return 0x00EF; case 'o': return 0x00F6; - case 'u': return 0x00FC; case 'y': return 0x00FF; - } - break; - case 0x030A: - switch (base) { - case 'A': return 0x00C5; case 'a': return 0x00E5; - } - break; - case 0x0327: - switch (base) { - case 'C': return 0x00C7; case 'c': return 0x00E7; - } - break; - } - return 0; -} - -static char *normalize_nfc_latin(const char *text) { - const unsigned char *p = (const unsigned char *)text; - const unsigned char *end = p + strlen(text); - char *out = (char *)malloc((size_t)(end - p) * 2 + 1); - if (!out) return NULL; - - size_t w = 0; - while (p < end) { - const unsigned char *cur = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; - p += len; - - if (p < end) { - int mark = 0, mark_len = 0; - utf8_decode_cp_len(p, end, &mark, &mark_len); - int composed = compose_latin_mark(cp, mark); - if (composed) { - char tmp[4]; - int n = utf8_encode_full_cp(composed, tmp); - for (int i = 0; i < n; i++) out[w++] = tmp[i]; - p += mark_len; - continue; - } - } - - memcpy(out + w, cur, (size_t)len); - w += (size_t)len; - } - - out[w] = '\0'; - return out; -} - -static int is_letter_cp(int cp) { - if (is_ascii_alpha_cp(cp)) return 1; - if (cp < 0x80 || is_space_cp(cp) || is_number_cp(cp) || - is_symbolish_cp(cp)) - return 0; - return 1; -} - -static int is_word_cp(int cp) { - return is_letter_cp(cp) || is_number_cp(cp); -} - -static int is_newline_cp(int cp) { - return cp == '\n' || cp == '\r'; -} - -static int ascii_lower(int c) { - return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; -} - -static int match_contraction(const unsigned char *p, const unsigned char *end) { - if (p >= end || *p != '\'') return 0; - int remain = (int)(end - p); - if (remain >= 2) { - int c1 = ascii_lower(p[1]); - if (c1 == 's' || c1 == 't' || c1 == 'm' || c1 == 'd') - return 2; - } - if (remain >= 3) { - int c1 = ascii_lower(p[1]); - int c2 = ascii_lower(p[2]); - if ((c1 == 'r' && c2 == 'e') || - (c1 == 'v' && c2 == 'e') || - (c1 == 'l' && c2 == 'l')) - return 3; - } - return 0; -} - -static int append_encoded_piece(const qwen_tokenizer_t *tok, - const char *piece, size_t piece_len, - int **ids, int *n_ids, int *cap) { - if (piece_len == 0) return 0; - - char *tmp = (char *)malloc(piece_len + 1); - if (!tmp) return -1; - memcpy(tmp, piece, piece_len); - tmp[piece_len] = '\0'; - - char *mapped = text_to_bpe_unicode(tmp); - free(tmp); - if (!mapped) return -1; - - int *piece_ids = NULL; - int n_piece = 0; - if (encode_bpe_word(tok, mapped, &piece_ids, &n_piece) != 0) { - free(mapped); - free(piece_ids); - return -1; - } - free(mapped); - - for (int i = 0; i < n_piece; i++) { - if (append_id(ids, n_ids, cap, piece_ids[i]) != 0) { - free(piece_ids); - return -1; - } - } - free(piece_ids); - return 0; -} - -static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, - int **out_ids, int *out_n) { - const unsigned char *start = (const unsigned char *)text; - const unsigned char *p = start; - const unsigned char *end = start + strlen(text); - int *ids = NULL; - int n_ids = 0, cap = 0; - - while (p < end) { - const unsigned char *piece = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; - - int clen = match_contraction(p, end); - if (clen > 0) { - p += clen; - } else { - const unsigned char *next = p + len; - int next_cp = 0, next_len = 0; - int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; - p += l; - } - } else if (is_letter_cp(cp)) { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; - p += l; - } - } else if (is_number_cp(cp)) { - p += len; - } else if (is_space_cp(cp)) { - const unsigned char *q = p; - int saw_newline = 0; - while (q < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; - q += l; - if (saw_newline) { - while (q < end) { - int c2 = 0, l2 = 0; - utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; - q += l2; - } - break; - } - } - - if (saw_newline) { - p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; - p += l; - } - } else { - const unsigned char *last = p; - const unsigned char *q2 = p; - int count = 0; - while (q2 < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; - last = q2; - q2 += l; - count++; - } - if (q2 < end && count > 1) - p = last; - else - p = q2; - } - } else { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; - p += l; - } - } - } - - if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), - &ids, &n_ids, &cap) != 0) { - free(ids); - return -1; - } - } - - *out_ids = ids; - *out_n = n_ids; - return 0; -} - -/* Encode one mapped BPE unicode string to token IDs. */ -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { - *out_ids = NULL; - *out_n = 0; - if (!mapped || !*mapped) return 0; - - char **syms = NULL; - int n_syms = 0; - if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1; - if (n_syms <= 0) { - free(syms); - return 0; - } - - while (n_syms > 1) { - int best_rank = INT_MAX; - int best_i = -1; - for (int i = 0; i < n_syms - 1; i++) { - int r = merge_rank(tok, syms[i], syms[i + 1]); - if (r < best_rank) { - best_rank = r; - best_i = i; - } - } - if (best_i < 0 || best_rank == INT_MAX) break; - - char *merged = str_concat2(syms[best_i], syms[best_i + 1]); - if (!merged) { - for (int i = 0; i < n_syms; i++) free(syms[i]); - free(syms); - return -1; - } - free(syms[best_i]); - free(syms[best_i + 1]); - syms[best_i] = merged; - for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1]; - n_syms--; - } - - int *ids = NULL; - int n_ids = 0, cap = 0; - for (int i = 0; i < n_syms; i++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); - if (id < 0) { - /* Should not happen with valid vocab + merges + byte-level mapping. */ - for (int k = 0; k < n_syms; k++) free(syms[k]); - free(syms); - free(ids); - return -1; - } - if (append_id(&ids, &n_ids, &cap, id) != 0) { - for (int k = 0; k < n_syms; k++) free(syms[k]); - free(syms); - free(ids); - return -1; - } - } - - for (int i = 0; i < n_syms; i++) free(syms[i]); - free(syms); - - *out_ids = ids; - *out_n = n_ids; - return 0; -} - -typedef struct { - size_t off; -} bpe_sym_ref_t; - -struct qwen_tokenizer_workspace { - char *normalized; - size_t normalized_cap; - char *mapped; - size_t mapped_cap; - char *arena; - size_t arena_cap; - size_t arena_len; - bpe_sym_ref_t *syms; - int syms_cap; - int *ids; - int ids_cap; -}; - -static int reserve_bytes(char **buf, size_t *cap, size_t need) { - if (*cap >= need) return 0; - size_t new_cap = *cap ? *cap : 256; - while (new_cap < need) { - if (new_cap > SIZE_MAX / 2) return -1; - new_cap *= 2; - } - char *p = (char *)realloc(*buf, new_cap); - if (!p) return -1; - *buf = p; - *cap = new_cap; - return 0; -} - -static int reserve_ints(int **buf, int *cap, int need) { - if (*cap >= need) return 0; - int new_cap = *cap ? *cap : 64; - while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; - new_cap *= 2; - } - int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); - if (!p) return -1; - *buf = p; - *cap = new_cap; - return 0; -} - -static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { - if (ws->syms_cap >= need) return 0; - int new_cap = ws->syms_cap ? ws->syms_cap : 64; - while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; - new_cap *= 2; - } - bpe_sym_ref_t *p = - (bpe_sym_ref_t *)realloc(ws->syms, - (size_t)new_cap * sizeof(*ws->syms)); - if (!p) return -1; - ws->syms = p; - ws->syms_cap = new_cap; - return 0; -} - -static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, - size_t len, size_t *off) { - if (ws->arena_len > SIZE_MAX - len - 1) return -1; - size_t need = ws->arena_len + len + 1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1; - *off = ws->arena_len; - memcpy(ws->arena + ws->arena_len, s, len); - ws->arena_len += len; - ws->arena[ws->arena_len++] = '\0'; - return 0; -} - -static const char *arena_str(const qwen_tokenizer_workspace_t *ws, - bpe_sym_ref_t sym) { - return ws->arena + sym.off; -} - -static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, - const char *text, const char **out) { - const unsigned char *p = (const unsigned char *)text; - const unsigned char *end = p + strlen(text); - size_t in_len = (size_t)(end - p); - if (in_len > (SIZE_MAX - 1) / 2) return -1; - if (reserve_bytes(&ws->normalized, &ws->normalized_cap, - in_len * 2 + 1) != 0) - return -1; - - size_t w = 0; - while (p < end) { - const unsigned char *cur = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; - p += len; - - if (p < end) { - int mark = 0, mark_len = 0; - utf8_decode_cp_len(p, end, &mark, &mark_len); - int composed = compose_latin_mark(cp, mark); - if (composed) { - char tmp[4]; - int n = utf8_encode_full_cp(composed, tmp); - memcpy(ws->normalized + w, tmp, (size_t)n); - w += (size_t)n; - p += mark_len; - continue; - } - } - - memcpy(ws->normalized + w, cur, (size_t)len); - w += (size_t)len; - } - - ws->normalized[w] = '\0'; - *out = ws->normalized; - return 0; -} - -static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, - const char *text, size_t len, - const char **out) { - init_gpt2_mapping(); - if (len > (SIZE_MAX - 1) / 3) return -1; - if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) - return -1; - - size_t w = 0; - for (size_t i = 0; i < len; i++) { - int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; - char tmp[4]; - int n = utf8_encode_cp(cp, tmp); - memcpy(ws->mapped + w, tmp, (size_t)n); - w += (size_t)n; - } - ws->mapped[w] = '\0'; - *out = ws->mapped; - return 0; -} - -static int output_append_id(int *out_ids, int out_cap, int *n_ids, - int *overflow, int id) { - if (*n_ids < out_cap && out_ids) - out_ids[*n_ids] = id; - else - *overflow = 1; - (*n_ids)++; - return 0; -} - -static int encode_bpe_word_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *mapped, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (!mapped || !*mapped) return 0; - - int n_syms = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { - int l = utf8_char_len(*p); - p += l; - n_syms++; - } - if (n_syms <= 0) return 0; - if (reserve_syms(ws, n_syms) != 0) return -1; - - ws->arena_len = 0; - int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { - int l = utf8_char_len(*p); - size_t off = 0; - if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) - return -1; - ws->syms[i++].off = off; - p += l; - } - - while (n_syms > 1) { - int best_rank = INT_MAX; - int best_i = -1; - for (int j = 0; j < n_syms - 1; j++) { - int r = merge_rank(tok, arena_str(ws, ws->syms[j]), - arena_str(ws, ws->syms[j + 1])); - if (r < best_rank) { - best_rank = r; - best_i = j; - } - } - if (best_i < 0 || best_rank == INT_MAX) break; - - const char *a = arena_str(ws, ws->syms[best_i]); - const char *b = arena_str(ws, ws->syms[best_i + 1]); - size_t la = strlen(a), lb = strlen(b); - if (la > SIZE_MAX - lb) return -1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, - ws->arena_len + la + lb + 1) != 0) - return -1; - - a = arena_str(ws, ws->syms[best_i]); - b = arena_str(ws, ws->syms[best_i + 1]); - size_t off = ws->arena_len; - memcpy(ws->arena + ws->arena_len, a, la); - ws->arena_len += la; - memcpy(ws->arena + ws->arena_len, b, lb); - ws->arena_len += lb; - ws->arena[ws->arena_len++] = '\0'; - ws->syms[best_i].off = off; - for (int j = best_i + 1; j < n_syms - 1; j++) - ws->syms[j] = ws->syms[j + 1]; - n_syms--; - } - - for (int j = 0; j < n_syms; j++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, arena_str(ws, ws->syms[j])); - if (id < 0) return -1; - output_append_id(out_ids, out_cap, n_ids, overflow, id); - } - return 0; -} - -static int append_encoded_piece_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *piece, size_t piece_len, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (piece_len == 0) return 0; - - const char *mapped = NULL; - if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) - return -1; - return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, - overflow); -} - -static int encode_pretokenized_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, int out_cap, - int *out_n) { - const unsigned char *start = (const unsigned char *)text; - const unsigned char *p = start; - const unsigned char *end = start + strlen(text); - int n_ids = 0; - int overflow = 0; - - while (p < end) { - const unsigned char *piece = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; - - int clen = match_contraction(p, end); - if (clen > 0) { - p += clen; - } else { - const unsigned char *next = p + len; - int next_cp = 0, next_len = 0; - int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; - p += l; - } - } else if (is_letter_cp(cp)) { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; - p += l; - } - } else if (is_number_cp(cp)) { - p += len; - } else if (is_space_cp(cp)) { - const unsigned char *q = p; - int saw_newline = 0; - while (q < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; - q += l; - if (saw_newline) { - while (q < end) { - int c2 = 0, l2 = 0; - utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; - q += l2; - } - break; - } - } - - if (saw_newline) { - p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; - p += l; - } - } else { - const unsigned char *last = p; - const unsigned char *q2 = p; - int count = 0; - while (q2 < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; - last = q2; - q2 += l; - count++; - } - if (q2 < end && count > 1) - p = last; - else - p = q2; - } - } else { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; - p += l; - } - } - } - - if (append_encoded_piece_into(tok, ws, (const char *)piece, - (size_t)(p - piece), out_ids, out_cap, - &n_ids, &overflow) != 0) - return -1; - } - - *out_n = n_ids; - return overflow ? -2 : 0; -} - -static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { - const char *slash = strrchr(vocab_path, '/'); - if (!slash) { - if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1; - return 0; - } - size_t dir_len = (size_t)(slash - vocab_path); - if (dir_len + 12 >= out_cap) return -1; - memcpy(out_path, vocab_path, dir_len); - out_path[dir_len] = '\0'; - snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); - return 0; -} - -static void trim_newline(char *line) { - size_t n = strlen(line); - while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0'; -} - -static int parse_merge_pair(char *line, char **a, char **b) { - *a = NULL; - *b = NULL; - trim_newline(line); - if (line[0] == '\0') return -1; - if (line[0] == '#') return -1; - - char *sp = strchr(line, ' '); - if (!sp) return -1; - *sp = '\0'; - char *p2 = sp + 1; - while (*p2 == ' ') p2++; - if (*line == '\0' || *p2 == '\0') return -1; - *a = line; - *b = p2; - return 0; -} - -static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { - FILE *f = fopen(merges_path, "rb"); - if (!f) return -1; - - int n_pairs = 0; - char line[8192]; - while (fgets(line, sizeof(line), f)) { - char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) == 0) n_pairs++; - } - if (n_pairs <= 0) { - fclose(f); - return -1; - } - - tok->merge_map_cap = next_pow2(n_pairs * 2); - tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t)); - if (!tok->merge_map) { - fclose(f); - tok->merge_map_cap = 0; - return -1; - } - - if (fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - int rank = 0; - while (fgets(line, sizeof(line), f)) { - char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) != 0) continue; - - size_t la = strlen(a), lb = strlen(b); - char *key = (char *)malloc(la + 1 + lb + 1); - if (!key) continue; - memcpy(key, a, la); - key[la] = ' '; - memcpy(key + la + 1, b, lb); - key[la + 1 + lb] = '\0'; - - (void)map_insert_owned((str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, &key, rank); - free(key); - rank++; - } - fclose(f); - return 0; -} - -/* ======================================================================== - * Public API - * ======================================================================== */ - -qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { - FILE *f = fopen(vocab_json_path, "rb"); - if (!f) { - fprintf(stderr, "qwen_tokenizer_load: cannot open %s\n", vocab_json_path); - return NULL; - } - - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return NULL; - } - long file_size = ftell(f); - if (file_size < 0 || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return NULL; - } - - char *json = (char *)malloc((size_t)file_size + 1); - if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) { - fclose(f); - free(json); - return NULL; - } - fclose(f); - json[file_size] = '\0'; - - int max_id = 0; - const char *p = json; - skip_ws(&p); - if (*p != '{') { - free(json); - return NULL; - } - p++; - - const char *saved = p; - while (*p && *p != '}') { - skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; - - char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; - skip_ws(&p); - if (*p != ':') break; - p++; - int64_t id = parse_json_int(&p); - if (id > max_id) max_id = (int)id; - } - - int vocab_size = max_id + 1; - qwen_tokenizer_t *tok = (qwen_tokenizer_t *)calloc(1, sizeof(qwen_tokenizer_t)); - if (!tok) { - free(json); - return NULL; - } - tok->vocab_size = vocab_size; - tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); - tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); - if (!tok->id_to_text || !tok->id_to_bpe) { - qwen_tokenizer_free(tok); - free(json); - return NULL; - } - - p = saved; - while (*p && *p != '}') { - skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; - - char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; - skip_ws(&p); - if (*p != ':') break; - p++; - int64_t id = parse_json_int(&p); - - if (id >= 0 && id < vocab_size) { - free(tok->id_to_bpe[id]); - free(tok->id_to_text[id]); - tok->id_to_bpe[id] = strdup(key); - tok->id_to_text[id] = decode_gpt2_token(key); - } - } - free(json); - - int n_vocab_entries = 0; - for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++; - tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); - tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); - if (!tok->vocab_map) { - qwen_tokenizer_free(tok); - return NULL; - } - for (int i = 0; i < vocab_size; i++) { - if (tok->id_to_bpe[i]) { - map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i); - } - } - - char merges_path[1024]; - if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { - if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { - fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", - merges_path); - } - } - - return tok; -} - -const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { - if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; - return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; -} - -int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !text || text[0] == '\0') return NULL; - - char *normalized = normalize_nfc_latin(text); - if (!normalized) return NULL; - - int *ids = NULL; - int n_ids = 0; - if (encode_pretokenized(tok, normalized, &ids, &n_ids) != 0) { - free(normalized); - free(ids); - return NULL; - } - free(normalized); - - if (out_n_tokens) *out_n_tokens = n_ids; - return ids; -} - -qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { - return (qwen_tokenizer_workspace_t *)calloc(1, - sizeof(qwen_tokenizer_workspace_t)); -} - -void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { - if (!ws) return; - free(ws->normalized); - free(ws->mapped); - free(ws->arena); - free(ws->syms); - free(ws->ids); - free(ws); -} - -int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, int out_cap, - int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || out_cap < 0) return -1; - if (text[0] == '\0') return 0; - - const char *normalized = NULL; - if (normalize_nfc_latin_into(ws, text, &normalized) != 0) - return -1; - - int n_ids = 0; - int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, - &n_ids); - if (out_n_tokens) *out_n_tokens = n_ids; - return rc; -} - -int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || text[0] == '\0') return NULL; - - size_t len = strlen(text); - size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; - if (cap_sz > (size_t)INT_MAX) return NULL; - int cap = (int)cap_sz; - - if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) - return NULL; - - int n_ids = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); - if (rc == -2) { - if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) - return NULL; - rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); - } - if (rc != 0 || n_ids <= 0) return NULL; - - int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); - if (!ids) return NULL; - memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); - if (out_n_tokens) *out_n_tokens = n_ids; - return ids; -} - -void qwen_tokenizer_free(qwen_tokenizer_t *tok) { - if (!tok) return; - - if (tok->id_to_text) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]); - free(tok->id_to_text); - } - if (tok->id_to_bpe) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]); - free(tok->id_to_bpe); - } - - if (tok->merge_map) { - str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; - for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key); - free(tok->merge_map); - } - free(tok->vocab_map); - free(tok); -} diff --git a/qwen_asr_tokenizer.h b/qwen_asr_tokenizer.h deleted file mode 100644 index 742d0d1..0000000 --- a/qwen_asr_tokenizer.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * qwen_asr_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) - * - * Supports: - * - decode token ID -> UTF-8 text - * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) - */ - -#ifndef QWEN_ASR_TOKENIZER_H -#define QWEN_ASR_TOKENIZER_H - -typedef struct { - char **id_to_text; /* [vocab_size] decoded text strings */ - char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ - int vocab_size; - - /* Internal hash maps (opaque to callers) */ - void *vocab_map; - int vocab_map_cap; - void *merge_map; - int merge_map_cap; -} qwen_tokenizer_t; - -typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; - -/* Load tokenizer from vocab.json in model directory */ -qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); - -/* Decode a single token ID to text. Returns pointer to internal string. */ -const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); - -/* Encode UTF-8 text into token IDs using BPE. - * Returns malloc'd array of token IDs and sets *out_n_tokens. - * Returns NULL on error (and sets *out_n_tokens to 0). */ -int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); - -/* Reusable scratch for allocation-light repeated encoding. */ -qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); -void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); - -/* Encode into caller-provided out_ids[0..out_cap-1]. - * Returns 0 on success, -1 on error, -2 if out_cap is too small. - * In all cases *out_n_tokens is set to the number of tokens needed/emitted - * when out_n_tokens is non-NULL. */ -int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, int out_cap, - int *out_n_tokens); - -/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and - * performs only the final returned int[] allocation. */ -int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens); - -/* Free tokenizer */ -void qwen_tokenizer_free(qwen_tokenizer_t *tok); - -#endif /* QWEN_ASR_TOKENIZER_H */ diff --git a/qwen_kernels.c b/qwen_kernels.c new file mode 100644 index 0000000..5cf38cd --- /dev/null +++ b/qwen_kernels.c @@ -0,0 +1,1745 @@ +#include "qwen_kernels.h" +#include "qwen_kernels_impl.h" +#include +#include +#include +#include +#include +#if (defined(__AVX512F__) || defined(__AVX2__)) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif +#ifdef __APPLE__ +#include +#else +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#if defined(USE_OPENBLAS) +void openblas_set_num_threads(int num_threads); +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* ======================================================================== + * Thread Pool + * ======================================================================== */ + +#define QWEN_MAX_THREADS 16 + +typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); + +static struct { + pthread_t threads[QWEN_MAX_THREADS - 1]; + int tids[QWEN_MAX_THREADS - 1]; + int n_threads; + int shutdown; + + parallel_fn_t fn; + void *arg; + int generation; + + pthread_mutex_t mutex; + pthread_cond_t cond_work; + pthread_cond_t cond_done; + int n_done; +} tp = { + .n_threads = 1, + .shutdown = 0, + .generation = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond_work = PTHREAD_COND_INITIALIZER, + .cond_done = PTHREAD_COND_INITIALIZER, +}; + +static void *worker_loop(void *arg) { + int tid = *(int *)arg; + int my_gen = 0; + + for (;;) { + pthread_mutex_lock(&tp.mutex); + while (tp.generation == my_gen && !tp.shutdown) + pthread_cond_wait(&tp.cond_work, &tp.mutex); + if (tp.shutdown) { + pthread_mutex_unlock(&tp.mutex); + return NULL; + } + my_gen = tp.generation; + parallel_fn_t fn = tp.fn; + void *a = tp.arg; + int nt = tp.n_threads; + pthread_mutex_unlock(&tp.mutex); + + fn(tid, nt, a); + + pthread_mutex_lock(&tp.mutex); + if (++tp.n_done >= tp.n_threads - 1) + pthread_cond_signal(&tp.cond_done); + pthread_mutex_unlock(&tp.mutex); + } +} + +void qwen_set_threads(int n) { + if (n < 1) n = 1; + if (n > QWEN_MAX_THREADS) n = QWEN_MAX_THREADS; + +#if defined(USE_OPENBLAS) + /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the + * auxiliary pool serial to avoid nested thread contention. */ + int blas_threads = n; + openblas_set_num_threads(blas_threads); + n = 1; +#endif + + /* Shutdown existing workers */ + if (tp.n_threads > 1) { + pthread_mutex_lock(&tp.mutex); + tp.shutdown = 1; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + for (int i = 0; i < tp.n_threads - 1; i++) + pthread_join(tp.threads[i], NULL); + tp.shutdown = 0; + tp.generation = 0; + } + + tp.n_threads = n; + if (n <= 1) { + if (qwen_verbose >= 2) { +#if defined(USE_OPENBLAS) + fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", + n, blas_threads); +#else + fprintf(stderr, "Thread pool: %d threads\n", n); +#endif + } + return; + } + + for (int i = 0; i < n - 1; i++) { + tp.tids[i] = i + 1; + pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); + } + + if (qwen_verbose >= 2) { +#if defined(USE_OPENBLAS) + fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", + n, blas_threads); +#else + fprintf(stderr, "Thread pool: %d threads\n", n); +#endif + } +} + +int qwen_get_num_cpus(void) { +#ifdef __APPLE__ + int n = 0; + size_t len = sizeof(n); + sysctlbyname("hw.ncpu", &n, &len, NULL, 0); + return n > 0 ? n : 1; +#else + int n = (int)sysconf(_SC_NPROCESSORS_ONLN); + return n > 0 ? n : 1; +#endif +} + +/* Dispatch work to all threads; main thread is tid=0 */ +static void parallel_for(parallel_fn_t fn, void *arg) { + if (tp.n_threads <= 1) { + fn(0, 1, arg); + return; + } + + pthread_mutex_lock(&tp.mutex); + tp.fn = fn; + tp.arg = arg; + tp.n_done = 0; + tp.generation++; + pthread_cond_broadcast(&tp.cond_work); + pthread_mutex_unlock(&tp.mutex); + + fn(0, tp.n_threads, arg); + + pthread_mutex_lock(&tp.mutex); + while (tp.n_done < tp.n_threads - 1) + pthread_cond_wait(&tp.cond_done, &tp.mutex); + pthread_mutex_unlock(&tp.mutex); +} + +/* ======================================================================== + * Basic Element-wise Operations + * ======================================================================== */ + +void qwen_add_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) a[i] += b[i]; +} + +void qwen_mul_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) a[i] *= b[i]; +} + +void qwen_scale(float *x, float s, int n) { + for (int i = 0; i < n; i++) x[i] *= s; +} + +void qwen_copy(float *dst, const float *src, int n) { + memcpy(dst, src, n * sizeof(float)); +} + +/* ======================================================================== + * Matrix Operations + * ======================================================================== */ + +void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); +#else + for (int m = 0; m < M; m++) { + for (int n = 0; n < N; n++) { + float sum = 0.0f; + for (int k = 0; k < K; k++) { + sum += A[m * K + k] * B[n * K + k]; + } + C[m * N + n] = sum; + } + } +#endif +} + +void qwen_linear(float *y, const float *x, const float *W, const float *b, + int seq_len, int in_dim, int out_dim) { +#ifdef USE_BLAS + if (seq_len == 1) { + cblas_sgemv(CblasRowMajor, CblasNoTrans, + out_dim, in_dim, 1.0f, W, in_dim, x, 1, + 0.0f, y, 1); + } else { + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + seq_len, out_dim, in_dim, + 1.0f, x, in_dim, W, in_dim, + 0.0f, y, out_dim); + } + if (b != NULL) { + for (int s = 0; s < seq_len; s++) { + for (int o = 0; o < out_dim; o++) { + y[s * out_dim + o] += b[o]; + } + } + } +#else + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * in_dim; + float *y_row = y + s * out_dim; + for (int o = 0; o < out_dim; o++) { + const float *w_row = W + o * in_dim; + float sum = (b != NULL) ? b[o] : 0.0f; + for (int i = 0; i < in_dim; i++) { + sum += x_row[i] * w_row[i]; + } + y_row[o] = sum; + } + } +#endif +} + +void qwen_linear_nobias(float *y, const float *x, const float *W, + int seq_len, int in_dim, int out_dim) { + qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim); +} + +/* Convert bf16 buffer to f32 buffer */ +static void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + size_t i = 0; + uint32_t *d = (uint32_t *)(void *)dst; + for (; i + 8 <= n; i += 8) { + uint16x8_t v = vld1q_u16(src + i); + uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(v)), 16); + uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(v)), 16); + vst1q_u32(d + i, lo); + vst1q_u32(d + i + 4, hi); + } + for (; i < n; i++) + d[i] = ((uint32_t)src[i]) << 16; +#else + uint32_t *d = (uint32_t *)(void *)dst; + for (size_t i = 0; i < n; i++) + d[i] = ((uint32_t)src[i]) << 16; +#endif +} + +/* Reusable scratch buffer for bf16->f32 conversion */ +static float *bf16_scratch = NULL; +static size_t bf16_scratch_cap = 0; + +static float *bf16_get_scratch(size_t n) { + if (n > bf16_scratch_cap) { + free(bf16_scratch); + bf16_scratch = (float *)malloc(n * sizeof(float)); + bf16_scratch_cap = bf16_scratch ? n : 0; + } + return bf16_scratch; +} + +typedef struct { + const uint16_t *src; + size_t n; + float *dst_f32; +} bf16_cache_entry_t; + +static bf16_cache_entry_t *bf16_cache = NULL; +static int bf16_cache_len = 0; +static int bf16_cache_cap = 0; +static size_t bf16_cache_bytes = 0; +static size_t bf16_cache_limit_bytes = 0; +static int bf16_cache_limit_init = 0; + +static void bf16_cache_init_limit(void) { + if (bf16_cache_limit_init) return; + bf16_cache_limit_init = 1; + + /* Default OFF. Override with QWEN_BF16_CACHE_MB= to enable. */ + unsigned long long mb = 0; + const char *env = getenv("QWEN_BF16_CACHE_MB"); + if (env && env[0] != '\0') { + char *end = NULL; + unsigned long long v = strtoull(env, &end, 10); + if (end != env) mb = v; + } + bf16_cache_limit_bytes = (size_t)(mb * 1024ULL * 1024ULL); + + if (qwen_verbose >= 2) { + fprintf(stderr, "BF16 cache: limit=%llu MB\n", mb); + } +} + +static const float *bf16_get_cached_f32(const uint16_t *src, size_t n) { + bf16_cache_init_limit(); + + for (int i = 0; i < bf16_cache_len; i++) { + if (bf16_cache[i].src == src && bf16_cache[i].n == n) { + return bf16_cache[i].dst_f32; + } + } + + if (bf16_cache_limit_bytes == 0) return NULL; + + size_t bytes = n * sizeof(float); + if (bytes > bf16_cache_limit_bytes) return NULL; + if (bf16_cache_bytes + bytes > bf16_cache_limit_bytes) return NULL; + + float *dst = (float *)malloc(bytes); + if (!dst) return NULL; + bf16_to_f32_buf(dst, src, n); + + if (bf16_cache_len == bf16_cache_cap) { + int new_cap = bf16_cache_cap > 0 ? bf16_cache_cap * 2 : 256; + bf16_cache_entry_t *tmp = (bf16_cache_entry_t *)realloc( + bf16_cache, (size_t)new_cap * sizeof(bf16_cache_entry_t)); + if (!tmp) { + free(dst); + return NULL; + } + bf16_cache = tmp; + bf16_cache_cap = new_cap; + } + + bf16_cache[bf16_cache_len].src = src; + bf16_cache[bf16_cache_len].n = n; + bf16_cache[bf16_cache_len].dst_f32 = dst; + bf16_cache_len++; + bf16_cache_bytes += bytes; + return dst; +} + +static const float *bf16_get_f32_view(const uint16_t *src, size_t n) { + const float *cached = bf16_get_cached_f32(src, n); + if (cached) return cached; + + float *scratch = bf16_get_scratch(n); + if (!scratch) return NULL; + bf16_to_f32_buf(scratch, src, n); + return scratch; +} + +/* + * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias + * Processes 2 output rows at a time to amortize x vector loads. + */ +static void bf16_matvec_fused(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); +} + +/* Threaded matvec: split output rows across threads */ +typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int in_dim; + int out_dim; +} matvec_task_t; + +static void matvec_worker(int tid, int n_threads, void *arg) { + matvec_task_t *t = (matvec_task_t *)arg; + int chunk = (t->out_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->out_dim) end = t->out_dim; + if (start >= end) return; + + bf16_matvec_fused(t->y + start, t->x, + t->W_bf16 + (size_t)start * t->in_dim, + t->bias ? t->bias + start : NULL, + t->in_dim, end - start); +} + +static void bf16_matvec_threaded(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + if (tp.n_threads <= 1) { + bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); + return; + } + matvec_task_t task = { y, x, W_bf16, bias, in_dim, out_dim }; + parallel_for(matvec_worker, &task); +} + +typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int seq_len; + int in_dim; + int out_dim; +} bf16_linear_rows_task_t; + +static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { + bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) end = t->seq_len; + if (start >= end) return; + + for (int s = start; s < end; s++) { + bf16_matvec_fused(t->y + (size_t)s * t->out_dim, + t->x + (size_t)s * t->in_dim, + t->W_bf16, t->bias, t->in_dim, t->out_dim); + } +} + +static void bf16_linear_rows(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int seq_len, int in_dim, + int out_dim) { + bf16_linear_rows_task_t task = { + .y = y, + .x = x, + .W_bf16 = W_bf16, + .bias = bias, + .seq_len = seq_len, + .in_dim = in_dim, + .out_dim = out_dim, + }; + + if (tp.n_threads > 1 && seq_len >= 2) + parallel_for(bf16_linear_rows_worker, &task); + else + bf16_linear_rows_worker(0, 1, &task); +} + +typedef struct { + float *a; + float *b; + const float *x; + const uint16_t *Wa_bf16; + const uint16_t *Wb_bf16; + int seq_len; + int in_dim; + int a_dim; + int b_dim; + int total_dim; +} pair_matvec_task_t; + +static void pair_matvec_range(const pair_matvec_task_t *t, int row, + int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; + int a_end = t->a_dim; + int b_end = a_end + t->b_dim; + + if (start < a_end) { + int s = start; + int e = end < a_end ? end : a_end; + if (s < e) { + bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, + t->Wa_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > a_end && start < b_end) { + int s = start > a_end ? start - a_end : 0; + int e_abs = end < b_end ? end : b_end; + int e = e_abs - a_end; + if (s < e) { + bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, + t->Wb_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } +} + +static void pair_matvec_worker(int tid, int n_threads, void *arg) { + pair_matvec_task_t *t = (pair_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) end = total_work; + if (start >= end) return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + pair_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + +void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, int in_dim, int a_dim, + int b_dim) { + if (seq_len <= 0) return; + + if (tp.n_threads <= 1) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(a + (size_t)s * a_dim, x_row, + Wa_bf16, NULL, in_dim, a_dim); + bf16_matvec_fused(b + (size_t)s * b_dim, x_row, + Wb_bf16, NULL, in_dim, b_dim); + } + return; + } + + pair_matvec_task_t task = { + .a = a, + .b = b, + .x = x, + .Wa_bf16 = Wa_bf16, + .Wb_bf16 = Wb_bf16, + .seq_len = seq_len, + .in_dim = in_dim, + .a_dim = a_dim, + .b_dim = b_dim, + .total_dim = a_dim + b_dim, + }; + parallel_for(pair_matvec_worker, &task); +} + +typedef struct { + float *q; + float *k; + float *v; + const float *x; + const uint16_t *Wq_bf16; + const uint16_t *Wk_bf16; + const uint16_t *Wv_bf16; + int in_dim; + int q_dim; + int kv_dim; + int seq_len; + int total_dim; +} qkv_matvec_task_t; + +static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, + int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; + int q_end = t->q_dim; + int k_end = q_end + t->kv_dim; + int v_end = k_end + t->kv_dim; + + if (start < q_end) { + int s = start; + int e = end < q_end ? end : q_end; + if (s < e) { + bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, + t->Wq_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > q_end && start < k_end) { + int s = start > q_end ? start - q_end : 0; + int e_abs = end < k_end ? end : k_end; + int e = e_abs - q_end; + if (s < e) { + bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, + t->Wk_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } + + if (end > k_end && start < v_end) { + int s = start > k_end ? start - k_end : 0; + int e_abs = end < v_end ? end : v_end; + int e = e_abs - k_end; + if (s < e) { + bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, + t->Wv_bf16 + (size_t)s * t->in_dim, + NULL, t->in_dim, e - s); + } + } +} + +static void qkv_matvec_worker(int tid, int n_threads, void *arg) { + qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) end = total_work; + if (start >= end) return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + qkv_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + +void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int seq_len, int in_dim, int q_dim, + int kv_dim) { + if (seq_len <= 0) return; + + if (tp.n_threads <= 1) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(q + (size_t)s * q_dim, x_row, + Wq_bf16, NULL, in_dim, q_dim); + bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, + Wk_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, + Wv_bf16, NULL, in_dim, kv_dim); + } + return; + } + + qkv_matvec_task_t task = { + .q = q, + .k = k, + .v = v, + .x = x, + .Wq_bf16 = Wq_bf16, + .Wk_bf16 = Wk_bf16, + .Wv_bf16 = Wv_bf16, + .in_dim = in_dim, + .q_dim = q_dim, + .kv_dim = kv_dim, + .seq_len = seq_len, + .total_dim = q_dim + 2 * kv_dim, + }; + parallel_for(qkv_matvec_worker, &task); +} + +void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, + int seq_len, int in_dim, int out_dim) { + if (seq_len == 1) { + bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); + return; + } + if (seq_len <= 16) { + bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); + return; + } + size_t n = (size_t)out_dim * in_dim; + const float *W_f32 = bf16_get_f32_view(W_bf16, n); + if (!W_f32) return; + qwen_linear_nobias(y, x, W_f32, seq_len, in_dim, out_dim); +} + +void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, + const float *b, int seq_len, int in_dim, int out_dim) { + if (seq_len == 1) { + bf16_matvec_threaded(y, x, W_bf16, b, in_dim, out_dim); + return; + } + if (seq_len <= 16) { + bf16_linear_rows(y, x, W_bf16, b, seq_len, in_dim, out_dim); + return; + } + size_t n = (size_t)out_dim * in_dim; + const float *W_f32 = bf16_get_f32_view(W_bf16, n); + if (!W_f32) return; + qwen_linear(y, x, W_f32, b, seq_len, in_dim, out_dim); +} + +/* Find argmax over a range of output rows [start, end). + * Uses 2-row processing to amortize x vector loads (same as bf16_matvec_fused). */ +static void argmax_bf16_range(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + qwen_argmax_bf16_range_impl(x, W_bf16, in_dim, start, end, best_out, best_val_out); +} + +typedef struct { + const float *x; + const uint16_t *W_bf16; + int in_dim; + int out_dim; + int best_idx[QWEN_MAX_THREADS]; + float best_val[QWEN_MAX_THREADS]; +} argmax_task_t; + +static void argmax_worker(int tid, int n_threads, void *arg) { + argmax_task_t *t = (argmax_task_t *)arg; + int chunk = (t->out_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->out_dim) end = t->out_dim; + if (start >= end) { + t->best_val[tid] = -1e30f; + t->best_idx[tid] = 0; + return; + } + argmax_bf16_range(t->x, t->W_bf16, t->in_dim, start, end, + &t->best_idx[tid], &t->best_val[tid]); +} + +int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, + int in_dim, int out_dim) { + if (tp.n_threads <= 1) { + int best; + float best_val; + argmax_bf16_range(x, W_bf16, in_dim, 0, out_dim, &best, &best_val); + return best; + } + + argmax_task_t task; + task.x = x; + task.W_bf16 = W_bf16; + task.in_dim = in_dim; + task.out_dim = out_dim; + parallel_for(argmax_worker, &task); + + int best = task.best_idx[0]; + float best_val = task.best_val[0]; + for (int i = 1; i < tp.n_threads; i++) { + if (task.best_val[i] > best_val) { + best_val = task.best_val[i]; + best = task.best_idx[i]; + } + } + return best; +} + +void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, + int M, int K, int N) { + if (M == 1) { + bf16_matvec_threaded(C, A, B_bf16, NULL, K, N); + } else { + size_t n = (size_t)N * K; + const float *B_f32 = bf16_get_f32_view(B_bf16, n); + if (!B_f32) return; + qwen_matmul_t(C, A, B_f32, M, K, N); + } +} + +/* ======================================================================== + * 2D Convolution (im2col + BLAS sgemm) + * ======================================================================== */ + +/* + * im2col: Unroll input patches into a column matrix for GEMM-based conv2d. + * Input: [C_in, H_in, W_in] + * Output columns: [C_in * kH * kW, H_out * W_out] + */ +static void im2col(const float *in, float *cols, + int c_in, int h_in, int w_in, + int kh, int kw, int stride, int padding, + int h_out, int w_out) { + int col_len = h_out * w_out; + for (int ic = 0; ic < c_in; ic++) { + for (int ki = 0; ki < kh; ki++) { + for (int kj = 0; kj < kw; kj++) { + int col_row = (ic * kh + ki) * kw + kj; + float *col_ptr = cols + (size_t)col_row * col_len; + for (int oh = 0; oh < h_out; oh++) { + int ih = oh * stride - padding + ki; + for (int ow = 0; ow < w_out; ow++) { + int iw = ow * stride - padding + kj; + if (ih >= 0 && ih < h_in && iw >= 0 && iw < w_in) { + col_ptr[oh * w_out + ow] = in[ic * h_in * w_in + ih * w_in + iw]; + } else { + col_ptr[oh * w_out + ow] = 0.0f; + } + } + } + } + } + } +} + +void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, + int c_in, int c_out, int h_in, int w_in, + int kh, int kw, int stride, int padding) { + int h_out = (h_in + 2 * padding - kh) / stride + 1; + int w_out = (w_in + 2 * padding - kw) / stride + 1; + int patch_size = c_in * kh * kw; + int spatial_out = h_out * w_out; + + /* im2col: input -> column matrix [patch_size, spatial_out] */ + float *cols = (float *)malloc((size_t)patch_size * spatial_out * sizeof(float)); + im2col(in, cols, c_in, h_in, w_in, kh, kw, stride, padding, h_out, w_out); + + /* GEMM: weight[c_out, patch_size] @ cols[patch_size, spatial_out] = out[c_out, spatial_out] */ +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + c_out, spatial_out, patch_size, + 1.0f, weight, patch_size, cols, spatial_out, + 0.0f, out, spatial_out); +#else + for (int oc = 0; oc < c_out; oc++) { + for (int s = 0; s < spatial_out; s++) { + float sum = 0.0f; + for (int p = 0; p < patch_size; p++) { + sum += weight[oc * patch_size + p] * cols[p * spatial_out + s]; + } + out[oc * spatial_out + s] = sum; + } + } +#endif + + free(cols); + + /* Add bias */ + if (bias) { + for (int oc = 0; oc < c_out; oc++) { + float b = bias[oc]; + float *row = out + oc * spatial_out; + for (int s = 0; s < spatial_out; s++) { + row[s] += b; + } + } + } +} + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, + int seq_len, int hidden, float eps) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * hidden; + float *out_row = out + s * hidden; + + /* Compute mean */ +#if defined(__AVX512F__) + __m512 sumv = _mm512_setzero_ps(); + int i = 0; + for (; i + 16 <= hidden; i += 16) { + sumv = _mm512_add_ps(sumv, _mm512_loadu_ps(x_row + i)); + } + float mean = _mm512_reduce_add_ps(sumv); + for (; i < hidden; i++) mean += x_row[i]; +#elif defined(__AVX2__) + __m256 sumv = _mm256_setzero_ps(); + int i = 0; + for (; i + 8 <= hidden; i += 8) { + sumv = _mm256_add_ps(sumv, _mm256_loadu_ps(x_row + i)); + } + __m128 sum128 = _mm_add_ps(_mm256_castps256_ps128(sumv), _mm256_extractf128_ps(sumv, 1)); + sum128 = _mm_hadd_ps(sum128, sum128); + sum128 = _mm_hadd_ps(sum128, sum128); + float mean = _mm_cvtss_f32(sum128); + for (; i < hidden; i++) mean += x_row[i]; +#else + float mean = 0.0f; + for (int i = 0; i < hidden; i++) mean += x_row[i]; +#endif + mean /= hidden; + + /* Compute variance */ +#if defined(__AVX512F__) && defined(__FMA__) + __m512 meanv = _mm512_set1_ps(mean); + __m512 accv = _mm512_setzero_ps(); + int j = 0; + for (; j + 16 <= hidden; j += 16) { + __m512 v = _mm512_sub_ps(_mm512_loadu_ps(x_row + j), meanv); + accv = _mm512_fmadd_ps(v, v, accv); + } + float var = _mm512_reduce_add_ps(accv); + for (; j < hidden; j++) { + float d = x_row[j] - mean; + var += d * d; + } +#elif defined(__AVX2__) && defined(__FMA__) + __m256 meanv = _mm256_set1_ps(mean); + __m256 accv = _mm256_setzero_ps(); + int j = 0; + for (; j + 8 <= hidden; j += 8) { + __m256 v = _mm256_sub_ps(_mm256_loadu_ps(x_row + j), meanv); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float var = _mm_cvtss_f32(acc128); + for (; j < hidden; j++) { + float d = x_row[j] - mean; + var += d * d; + } +#else + float var = 0.0f; + for (int i = 0; i < hidden; i++) { + float d = x_row[i] - mean; + var += d * d; + } +#endif + var /= hidden; + + float inv_std = 1.0f / sqrtf(var + eps); +#if defined(__AVX512F__) && defined(__FMA__) + __m512 meanv2 = _mm512_set1_ps(mean); + __m512 invv = _mm512_set1_ps(inv_std); + int k = 0; + for (; k + 16 <= hidden; k += 16) { + __m512 vx = _mm512_sub_ps(_mm512_loadu_ps(x_row + k), meanv2); + __m512 vw = _mm512_loadu_ps(weight + k); + __m512 vb = _mm512_loadu_ps(bias + k); + __m512 v = _mm512_mul_ps(_mm512_mul_ps(vx, invv), vw); + v = _mm512_add_ps(v, vb); + _mm512_storeu_ps(out_row + k, v); + } + for (; k < hidden; k++) { + out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; + } +#elif defined(__AVX2__) && defined(__FMA__) + __m256 meanv2 = _mm256_set1_ps(mean); + __m256 invv = _mm256_set1_ps(inv_std); + int k = 0; + for (; k + 8 <= hidden; k += 8) { + __m256 vx = _mm256_sub_ps(_mm256_loadu_ps(x_row + k), meanv2); + __m256 vw = _mm256_loadu_ps(weight + k); + __m256 vb = _mm256_loadu_ps(bias + k); + __m256 v = _mm256_mul_ps(_mm256_mul_ps(vx, invv), vw); + v = _mm256_add_ps(v, vb); + _mm256_storeu_ps(out_row + k, v); + } + for (; k < hidden; k++) { + out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; + } +#else + for (int i = 0; i < hidden; i++) { + out_row[i] = (x_row[i] - mean) * inv_std * weight[i] + bias[i]; + } +#endif + } +} + +#define QWEN_RMS_NORM_PARALLEL_ELEMS 262144 + +static void qwen_rms_norm_range(float *out, const float *x, + const float *weight, int start, int end, + int hidden, float eps) { + for (int s = start; s < end; s++) { + const float *x_row = x + (size_t)s * hidden; + float *out_row = out + (size_t)s * hidden; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int i = 0; + for (; i + 16 <= hidden; i += 16) { + __m512 v = _mm512_loadu_ps(x_row + i); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int i = 0; + for (; i + 8 <= hidden; i += 8) { + __m256 v = _mm256_loadu_ps(x_row + i); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; i < hidden; i++) sum_sq += x_row[i] * x_row[i]; +#else + float sum_sq = 0.0f; + for (int i = 0; i < hidden; i++) { + sum_sq += x_row[i] * x_row[i]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= hidden; j += 16) { + __m512 vx = _mm512_loadu_ps(x_row + j); + __m512 vw = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= hidden; j += 8) { + __m256 vx = _mm256_loadu_ps(x_row + j); + __m256 vw = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) out_row[j] = x_row[j] * rms_inv * weight[j]; +#else + for (int i = 0; i < hidden; i++) { + out_row[i] = x_row[i] * rms_inv * weight[i]; + } +#endif + } +} + +typedef struct { + float *out; + const float *x; + const float *weight; + int seq_len; + int hidden; + float eps; +} rms_norm_task_t; + +static void rms_norm_worker(int tid, int n_threads, void *arg) { + rms_norm_task_t *t = (rms_norm_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) end = t->seq_len; + if (start >= end) return; + qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, + t->eps); +} + +void qwen_rms_norm(float *out, const float *x, const float *weight, + int seq_len, int hidden, float eps) { + if (seq_len <= 0 || hidden <= 0) return; + + long long elems = (long long)seq_len * hidden; + if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) { + rms_norm_task_t task = { + .out = out, + .x = x, + .weight = weight, + .seq_len = seq_len, + .hidden = hidden, + .eps = eps, + }; + parallel_for(rms_norm_worker, &task); + } else { + qwen_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); + } +} + +void qwen_rms_norm_per_head(float *x, const float *weight, + int seq_len, int n_heads, int head_dim, float eps) { + /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ + int hidden = n_heads * head_dim; + for (int s = 0; s < seq_len; s++) { + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int d = 0; + for (; d + 16 <= head_dim; d += 16) { + __m512 v = _mm512_loadu_ps(vec + d); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int d = 0; + for (; d + 8 <= head_dim; d += 8) { + __m256 v = _mm256_loadu_ps(vec + d); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; d < head_dim; d++) sum_sq += vec[d] * vec[d]; +#else + float sum_sq = 0.0f; + for (int d = 0; d < head_dim; d++) { + sum_sq += vec[d] * vec[d]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= head_dim; j += 16) { + __m512 v = _mm512_loadu_ps(vec + j); + __m512 w = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= head_dim; j += 8) { + __m256 v = _mm256_loadu_ps(vec + j); + __m256 w = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) vec[j] = vec[j] * rms_inv * weight[j]; +#else + for (int d = 0; d < head_dim; d++) { + vec[d] = vec[d] * rms_inv * weight[d]; + } +#endif + } + } +} + +/* ======================================================================== + * Activation Functions + * ======================================================================== */ + +void qwen_silu(float *x, int n) { + for (int i = 0; i < n; i++) { + float val = x[i]; + x[i] = val / (1.0f + expf(-val)); + } +} + +void qwen_gelu(float *x, int n) { + for (int i = 0; i < n; i++) { + float val = x[i]; + float x3 = val * val * val; + float inner = 0.7978845608028654f * (val + 0.044715f * x3); + x[i] = 0.5f * val * (1.0f + tanhf(inner)); + } +} + +void qwen_silu_mul_inplace(float *gate, const float *up, int n) { +#if defined(__APPLE__) && defined(USE_BLAS) + enum { CHUNK = 4096 }; + float exp_neg[CHUNK]; + + for (int base = 0; base < n; base += CHUNK) { + int len = n - base; + if (len > CHUNK) len = CHUNK; + + for (int i = 0; i < len; i++) + exp_neg[i] = -gate[base + i]; + vvexpf(exp_neg, exp_neg, &len); + + for (int i = 0; i < len; i++) { + float g = gate[base + i]; + gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i]; + } + } +#else + for (int i = 0; i < n; i++) { + float g = gate[i]; + gate[i] = (g / (1.0f + expf(-g))) * up[i]; + } +#endif +} + +typedef struct { + float *out; + const float *gate_up; + int seq_len; + int intermediate; +} swiglu_task_t; + +static void swiglu_worker(int tid, int n_threads, void *arg) { + swiglu_task_t *t = (swiglu_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int s0 = tid * chunk; + int s1 = s0 + chunk; + if (s1 > t->seq_len) s1 = t->seq_len; + if (s0 >= s1) return; + + int inter = t->intermediate; + int alias_inplace = (t->out == t->gate_up); + for (int s = s0; s < s1; s++) { + const float *gu = t->gate_up + (size_t)s * 2 * inter; + float *o = t->out + (size_t)s * inter; + if (!alias_inplace) { +#if defined(__APPLE__) && defined(USE_BLAS) + /* Fast path for prefill: vectorized exp(-g) using Accelerate/vForce. */ + for (int j = 0; j < inter; j++) o[j] = -gu[2 * j]; + int n = inter; + vvexpf(o, o, &n); + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + o[j] = (g / (1.0f + o[j])) * u; + } +#else + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + g = g / (1.0f + expf(-g)); /* SiLU */ + o[j] = g * u; + } +#endif + } else { + /* In-place mode (decode seq=1): keep single-pass scalar to avoid alias hazards. */ + for (int j = 0; j < inter; j++) { + float g = gu[2 * j]; + float u = gu[2 * j + 1]; + g = g / (1.0f + expf(-g)); /* SiLU */ + o[j] = g * u; + } + } + } +} + +void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate) { + swiglu_task_t task = { + .out = out, + .gate_up = gate_up, + .seq_len = seq_len, + .intermediate = intermediate + }; + + if (tp.n_threads > 1 && seq_len >= 2 && intermediate >= 256) { + parallel_for(swiglu_worker, &task); + } else { + swiglu_worker(0, 1, &task); + } +} + +void qwen_softmax(float *x, int rows, int cols) { + for (int r = 0; r < rows; r++) { + float *row = x + r * cols; + float max_val = row[0]; + for (int c = 1; c < cols; c++) { + if (row[c] > max_val) max_val = row[c]; + } + float sum = 0.0f; + for (int c = 0; c < cols; c++) { + row[c] = expf(row[c] - max_val); + sum += row[c]; + } + float inv_sum = 1.0f / sum; + for (int c = 0; c < cols; c++) { + row[c] *= inv_sum; + } + } +} + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +float qwen_dot_f32(const float *a, const float *b, int n) { + return qwen_dot_f32_impl(a, b, n); +} + +static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) { + return qwen_dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void qwen_vec_scale_inplace(float *dst, float scale, int n) { + qwen_vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void qwen_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + qwen_vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void qwen_vec_scale_add(float *dst, const float *src, float correction, int n) { + qwen_vec_scale_add_impl(dst, src, correction, n); +} + +void qwen_bidirectional_attention(float *out, const float *Q, const float *K, + const float *V, int seq __attribute__((unused)), + int n_heads, int head_dim, float scale, + const int *window_starts, int n_windows) { + int hidden = n_heads * head_dim; + + for (int h = 0; h < n_heads; h++) { + for (int w = 0; w < n_windows; w++) { + int ws = window_starts[w]; + int we = window_starts[w + 1]; + + for (int i = ws; i < we; i++) { + const float *q_row = Q + i * hidden + h * head_dim; + float *o_row = out + i * hidden + h * head_dim; + + /* Online softmax */ + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = ws; j < we; j++) { + const float *k_row = K + j * hidden + h * head_dim; + const float *v_row = V + j * hidden + h * head_dim; + + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } + } + } +} + +static void qwen_causal_attention_heads(float *out, const float *Q, const float *K, + const float *V, int seq_q, int seq_k, + int n_heads, int n_kv_heads, int head_dim, + float scale, int q_offset, + int head_start, int head_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + + for (int h = head_start; h < head_end; h++) { + int kv_h = h / heads_per_kv; + + for (int i = 0; i < seq_q; i++) { + const float *q_row = Q + i * q_hidden + h * head_dim; + float *o_row = out + i * q_hidden + h * head_dim; + int global_pos = q_offset + i; + int k_end = global_pos + 1; + if (k_end > seq_k) k_end = seq_k; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = 0; j < k_end; j++) { + const float *k_row = K + j * kv_hidden + kv_h * head_dim; + const float *v_row = V + j * kv_hidden + kv_h * head_dim; + + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } + } +} + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + int seq_q, seq_k; + int n_heads, n_kv_heads; + int head_dim; + float scale; + int q_offset; +} causal_attn_task_t; + +static void causal_attn_worker(int tid, int n_threads, void *arg) { + causal_attn_task_t *t = (causal_attn_task_t *)arg; + int chunk = (t->n_heads + n_threads - 1) / n_threads; + int h0 = tid * chunk; + int h1 = h0 + chunk; + if (h1 > t->n_heads) h1 = t->n_heads; + if (h0 >= h1) return; + + qwen_causal_attention_heads(t->out, t->Q, t->K, t->V, + t->seq_q, t->seq_k, t->n_heads, t->n_kv_heads, + t->head_dim, t->scale, t->q_offset, h0, h1); +} + +void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, + int seq_q, int seq_k, int n_heads, int n_kv_heads, + int head_dim, float scale, int q_offset) { + if (tp.n_threads > 1 && n_heads >= 2 && (seq_q >= 2 || seq_k >= 128)) { + causal_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, + .seq_q = seq_q, .seq_k = seq_k, + .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale, .q_offset = q_offset + }; + parallel_for(causal_attn_worker, &task); + return; + } + + qwen_causal_attention_heads(out, Q, K, V, + seq_q, seq_k, n_heads, n_kv_heads, + head_dim, scale, q_offset, 0, n_heads); +} + +#define QWEN_PACKED_ATTN_QUERY_TILE 32 +#define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128 + +static void qwen_bidirectional_gqa_attention_packed_online_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int end, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } +} + +#ifdef USE_BLAS +static void qwen_bidirectional_gqa_attention_packed_blas_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int end, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end, float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, + 0.0f, scores, keys); + qwen_softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, + 0.0f, o, q_hidden); +} +#endif + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; + float *scores; + int max_seq; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores + ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq + : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; + if (q1 > end) q1 = end; + if (task_id++ % n_threads != tid) continue; +#ifdef USE_BLAS + if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { + qwen_bidirectional_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, + scores); + continue; + } +#endif + qwen_bidirectional_gqa_attention_packed_online_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); + } + } + } +} + +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( + const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) max_seq = len; + } + if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0; + if ((size_t)max_seq > SIZE_MAX / (sizeof(float) * + QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) + return 0; + return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * + (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void qwen_bidirectional_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + if (len > max_seq) max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( + offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, + .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale, .scores = scores, + .max_seq = max_seq + }; + if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) { + parallel_for(packed_gqa_attn_worker, &task); + } else { + packed_gqa_attn_worker(0, 1, &task); + } +} + +void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale) { + size_t scratch_bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + qwen_bidirectional_gqa_attention_packed_with_scratch( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, + scores, scratch_bytes); + free(scores); +} + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model) { + int half = d_model / 2; + float log_timescale = logf(10000.0f) / (float)(half - 1); + + for (int p = 0; p < n_pos; p++) { + float *row = pe + p * d_model; + for (int d = 0; d < half; d++) { + float inv_timescale = expf(-(float)d * log_timescale); + float angle = (float)p * inv_timescale; + row[d] = sinf(angle); /* first half: sin */ + row[half + d] = cosf(angle); /* second half: cos */ + } + } +} + +void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, + int seq, int head_dim, float theta) { + int half = head_dim / 2; + + for (int s = 0; s < seq; s++) { + float pos = (float)positions[s]; + for (int d = 0; d < half; d++) { + float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = pos * freq; + float c = cosf(angle); + float sn = sinf(angle); + /* Duplicate for full head_dim */ + cos_out[s * head_dim + d] = c; + cos_out[s * head_dim + half + d] = c; + sin_out[s * head_dim + d] = sn; + sin_out[s * head_dim + half + d] = sn; + } + } +} + +void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, + int seq, int n_heads, int head_dim) { + /* + * NeoX split-half style: + * x1 = x[..., :half], x2 = x[..., half:] + * rotated = cat(-x2, x1) + * result = x * cos + rotated * sin + */ + int half = head_dim / 2; + int hidden = n_heads * head_dim; + + for (int s = 0; s < seq; s++) { + const float *c = cos_vals + s * head_dim; + const float *sn = sin_vals + s * head_dim; + + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + int d = 0; + for (; d + 16 <= half; d += 16) { + __m512 x1 = _mm512_loadu_ps(vec + d); + __m512 x2 = _mm512_loadu_ps(vec + half + d); + /* RoPE cache duplicates cos/sin across halves. */ + __m512 cc = _mm512_loadu_ps(c + d); + __m512 ss = _mm512_loadu_ps(sn + d); + __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); + __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); + _mm512_storeu_ps(vec + d, new1); + _mm512_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__AVX2__) && defined(__FMA__) + int d = 0; + for (; d + 8 <= half; d += 8) { + __m256 x1 = _mm256_loadu_ps(vec + d); + __m256 x2 = _mm256_loadu_ps(vec + half + d); + __m256 cc = _mm256_loadu_ps(c + d); + __m256 ss = _mm256_loadu_ps(sn + d); + __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); + __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); + _mm256_storeu_ps(vec + d, new1); + _mm256_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + int d = 0; + for (; d + 4 <= half; d += 4) { + float32x4_t x1 = vld1q_f32(vec + d); + float32x4_t x2 = vld1q_f32(vec + half + d); + float32x4_t cc = vld1q_f32(c + d); + float32x4_t ss = vld1q_f32(sn + d); + float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); + float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); + vst1q_f32(vec + d, new1); + vst1q_f32(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#else + for (int d = 0; d < half; d++) { + float x1 = vec[d]; /* first half */ + float x2 = vec[half + d]; /* second half */ + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#endif + } + } +} diff --git a/qwen_kernels.h b/qwen_kernels.h new file mode 100644 index 0000000..e92f944 --- /dev/null +++ b/qwen_kernels.h @@ -0,0 +1,210 @@ +/* + * qwen_kernels.h - Math kernels for Qwen3-ASR inference + * + * Low-level math operations. All operate on float32 tensors in row-major order. + * Adapted from voxtral-realtime project. + */ + +#ifndef QWEN_KERNELS_H +#define QWEN_KERNELS_H + +#include +#include + +/* ======================================================================== + * Basic Operations + * ======================================================================== */ + +void qwen_add_inplace(float *a, const float *b, int n); +void qwen_mul_inplace(float *a, const float *b, int n); +void qwen_scale(float *x, float s, int n); +void qwen_copy(float *dst, const float *src, int n); + +/* ======================================================================== + * Matrix Operations + * ======================================================================== */ + +/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ +void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); + +/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ +void qwen_linear(float *y, const float *x, const float *W, const float *b, + int seq_len, int in_dim, int out_dim); + +void qwen_linear_nobias(float *y, const float *x, const float *W, + int seq_len, int in_dim, int out_dim); + +/* bf16 weight variants */ +void qwen_linear_bf16(float *y, const float *x, const uint16_t *W_bf16, + const float *b, int seq_len, int in_dim, int out_dim); + +void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, + int seq_len, int in_dim, int out_dim); + +/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ +void qwen_linear_nobias_bf16_pair(float *a, float *b, const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, int in_dim, int a_dim, + int b_dim); + +/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ +void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int seq_len, int in_dim, int q_dim, + int kv_dim); + +void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, + int M, int K, int N); + +/* Dot product using the best available local SIMD implementation. */ +float qwen_dot_f32(const float *a, const float *b, int n); + +/* ======================================================================== + * 2D Convolution (for audio encoder conv stem) + * ======================================================================== */ + +/* + * 2D Convolution: out = conv2d(in, weight, bias) + * in: [C_in, H, W] + * weight: [C_out, C_in, kH, kW] + * bias: [C_out] (can be NULL) + * out: [C_out, H_out, W_out] + * H_out = (H + 2*padding - kH) / stride + 1 + * W_out = (W + 2*padding - kW) / stride + 1 + */ +void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, + int c_in, int c_out, int h_in, int w_in, + int kh, int kw, int stride, int padding); + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +/* LayerNorm with bias: out = (x - mean) / sqrt(var + eps) * weight + bias */ +void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, + int seq_len, int hidden, float eps); + +/* RMS Normalization: out = x / rms(x) * weight */ +void qwen_rms_norm(float *out, const float *x, const float *weight, + int seq_len, int hidden, float eps); + +/* Per-head RMS Normalization for Q/K norms in decoder + * x: [seq, n_heads, head_dim], weight: [head_dim] + * Normalizes each head independently */ +void qwen_rms_norm_per_head(float *x, const float *weight, + int seq_len, int n_heads, int head_dim, float eps); + +/* ======================================================================== + * Activation Functions + * ======================================================================== */ + +void qwen_silu(float *x, int n); +void qwen_gelu(float *x, int n); +void qwen_softmax(float *x, int rows, int cols); +/* gate = SiLU(gate) * up */ +void qwen_silu_mul_inplace(float *gate, const float *up, int n); +/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */ +void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate); + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +/* + * Bidirectional windowed attention (encoder). + * Q, K, V: [seq, n_heads * head_dim] + * out: [seq, n_heads * head_dim] + * window_starts: array of window start positions + * window_starts[n_windows] = seq (sentinel) + * All heads have same dimensions (no GQA in encoder). + */ +void qwen_bidirectional_attention(float *out, const float *Q, const float *K, + const float *V, int seq, int n_heads, + int head_dim, float scale, + const int *window_starts, int n_windows); + +/* + * Causal attention with GQA (decoder). + * Q: [seq_q, n_heads * head_dim] + * K: [seq_k, n_kv_heads * head_dim] + * V: [seq_k, n_kv_heads * head_dim] + * q_offset: global position of first query (for causal mask) + */ +void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, + int seq_q, int seq_k, int n_heads, int n_kv_heads, + int head_dim, float scale, int q_offset); + +/* + * Bidirectional GQA attention over a packed/ragged batch. + * + * Q: [total_seq, n_heads * head_dim] + * K,V: [total_seq, n_kv_heads * head_dim] + * offsets: [batch + 1], token spans for each sequence. + * + * Attention is block-diagonal: tokens only attend within their own + * offsets[b]..offsets[b+1] span. + */ +void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale); + +/* Optional reusable scratch for the BLAS-tiled packed attention path. */ +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( + const int *offsets, int batch); +void qwen_bidirectional_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes); + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +/* + * Sinusoidal position embeddings (encoder). + * pe: output [n_pos, d_model] + * First half = sin, second half = cos. + */ +void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model); + +/* + * NeoX-style RoPE: compute cos/sin for positions. + * cos_out, sin_out: [seq, head_dim] + * cos[d] and cos[half+d] are the same (duplicated for full head_dim). + */ +void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, + int seq, int head_dim, float theta); + +/* + * Apply NeoX-style RoPE to Q or K. + * x: [seq, n_heads * head_dim] (in-place) + * cos_vals, sin_vals: [seq, head_dim] + */ +void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, + int seq, int n_heads, int head_dim); + +/* Streaming argmax: finds argmax(W_bf16 @ x) without materializing full logits. + * Returns the index of the row with highest dot product. */ +int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, + int in_dim, int out_dim); + +/* ======================================================================== + * Threading + * ======================================================================== */ + +/* Set number of threads for parallel operations (default: 1). + * Creates a persistent thread pool. Call before inference. */ +void qwen_set_threads(int n); + +/* Get number of available CPU cores */ +int qwen_get_num_cpus(void); + +/* Global verbose flag */ +extern int qwen_verbose; + +#endif /* QWEN_KERNELS_H */ diff --git a/qwen_kernels_avx.c b/qwen_kernels_avx.c new file mode 100644 index 0000000..f47725f --- /dev/null +++ b/qwen_kernels_avx.c @@ -0,0 +1,502 @@ +/* + * qwen_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) + * + * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. + * Uses AVX-512F+BW for 16-wide bf16 matvec/argmax (dominant for seq_len==1), and + * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). + * + * The bf16 matvec processes 4 output rows simultaneously to reduce instruction + * overhead and improve out-of-order execution on memory-bound workloads. + */ + +#include "qwen_kernels_impl.h" + +#if defined(__AVX2__) && defined(__FMA__) + +#include +#include + +/* ===================================================================== + * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise + * ===================================================================== */ + +#if defined(__AVX512F__) && defined(__AVX512BW__) + +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + /* Process 4 output rows at a time */ + for (; o + 3 < out_dim; o += 4) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + const uint16_t *w2 = w1 + in_dim; + const uint16_t *w3 = w2 + in_dim; + + __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); + __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); + __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps(); + __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + + /* Row 0 */ + __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + + /* Row 1 */ + __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + + /* Row 2 */ + __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k)); + __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16)); + a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4); + a5 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5); + + /* Row 3 */ + __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k)); + __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16)); + a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6); + a7 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7); + } + + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + a4 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4); + a6 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6); + } + + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); + float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o+2] : 0.0f); + float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o+3] : 0.0f); + + for (; k < in_dim; k++) { + float xk = x[k]; + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; + float v0,v1,v2,v3; + memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); + s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; + } + y[o]=s0; y[o+1]=s1; y[o+2]=s2; y[o+3]=s3; + } + + /* Handle 2-row remainder */ + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); + __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16)); + __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k)); + __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0); + a1 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2); + a3 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3); + } + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k)); + a0 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0); + a2 = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2); + } + + float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f); + float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o+1] : 0.0f); + + for (; k < in_dim; k++) { + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); + s0+=v0*x[k]; s1+=v1*x[k]; + } + y[o]=s0; y[o+1]=s1; + } + + /* Single remaining row */ + for (; o < out_dim; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m512 acc = _mm512_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); + acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), + _mm512_loadu_ps(x + k), acc); + } + float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum += wv*x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + int o = start; + + /* Process 4 rows at a time */ + for (; o + 3 < end; o += 4) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + const uint16_t *w2 = w1 + in_dim; + const uint16_t *w3 = w2 + in_dim; + + __m512 a0=_mm512_setzero_ps(), a1=_mm512_setzero_ps(); + __m512 a2=_mm512_setzero_ps(), a3=_mm512_setzero_ps(); + __m512 a4=_mm512_setzero_ps(), a5=_mm512_setzero_ps(); + __m512 a6=_mm512_setzero_ps(), a7=_mm512_setzero_ps(); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + __m512 xv0 = _mm512_loadu_ps(x + k); + __m512 xv1 = _mm512_loadu_ps(x + k + 16); + __m256i r0a=_mm256_loadu_si256((const __m256i*)(w0+k)); + __m256i r0b=_mm256_loadu_si256((const __m256i*)(w0+k+16)); + a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a),16)),xv0,a0); + a1=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b),16)),xv1,a1); + __m256i r1a=_mm256_loadu_si256((const __m256i*)(w1+k)); + __m256i r1b=_mm256_loadu_si256((const __m256i*)(w1+k+16)); + a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a),16)),xv0,a2); + a3=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b),16)),xv1,a3); + __m256i r2a=_mm256_loadu_si256((const __m256i*)(w2+k)); + __m256i r2b=_mm256_loadu_si256((const __m256i*)(w2+k+16)); + a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a),16)),xv0,a4); + a5=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b),16)),xv1,a5); + __m256i r3a=_mm256_loadu_si256((const __m256i*)(w3+k)); + __m256i r3b=_mm256_loadu_si256((const __m256i*)(w3+k+16)); + a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a),16)),xv0,a6); + a7=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b),16)),xv1,a7); + } + for (; k + 16 <= in_dim; k += 16) { + __m512 xv = _mm512_loadu_ps(x + k); + __m256i r0=_mm256_loadu_si256((const __m256i*)(w0+k)); + __m256i r1=_mm256_loadu_si256((const __m256i*)(w1+k)); + __m256i r2=_mm256_loadu_si256((const __m256i*)(w2+k)); + __m256i r3=_mm256_loadu_si256((const __m256i*)(w3+k)); + a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0),16)),xv,a0); + a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1),16)),xv,a2); + a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2),16)),xv,a4); + a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3),16)),xv,a6); + } + + float s0=_mm512_reduce_add_ps(_mm512_add_ps(a0,a1)); + float s1=_mm512_reduce_add_ps(_mm512_add_ps(a2,a3)); + float s2=_mm512_reduce_add_ps(_mm512_add_ps(a4,a5)); + float s3=_mm512_reduce_add_ps(_mm512_add_ps(a6,a7)); + + for (; k < in_dim; k++) { + float xk=x[k]; + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; + float v0,v1,v2,v3; + memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); + s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; + } + + if (s0>best_val){best_val=s0;best=o;} + if (s1>best_val){best_val=s1;best=o+1;} + if (s2>best_val){best_val=s2;best=o+2;} + if (s3>best_val){best_val=s3;best=o+3;} + } + + /* Remaining 1-3 rows */ + for (; o < end; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m512 acc = _mm512_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); + acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), + _mm512_loadu_ps(x + k), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum += wv*x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + + *best_out = best; + *best_val_out = best_val; +} + +#else /* AVX2 only (or AVX-512F without BW) */ + +/* Helper: bf16 to f32 for 8 elements */ +static inline __m256 bf16x8_to_f32(__m128i raw) { + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16)); +} + +static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { + __m256i raw = _mm256_loadu_si256((const __m256i *)src); + *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw)); + *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); +} + +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = w0 + in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + __m256 a2=_mm256_setzero_ps(), a3=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 xlo = _mm256_loadu_ps(x + k); + __m256 xhi = _mm256_loadu_ps(x + k + 8); + __m256 wlo, whi; + bf16x16_to_f32(w0 + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, xlo, a0); + a1 = _mm256_fmadd_ps(whi, xhi, a1); + bf16x16_to_f32(w1 + k, &wlo, &whi); + a2 = _mm256_fmadd_ps(wlo, xlo, a2); + a3 = _mm256_fmadd_ps(whi, xhi, a3); + } + a0 = _mm256_add_ps(a0, a1); a2 = _mm256_add_ps(a2, a3); + __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1)); + r0 = _mm_hadd_ps(r0, r1); r0 = _mm_hadd_ps(r0, r0); + float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f); + float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0,r0,1)) + (bias ? bias[o+1] : 0.0f); + for (; k < in_dim; k++) { + uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; + float v0,v1; memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); + s0+=v0*x[k]; s1+=v1*x[k]; + } + y[o]=s0; y[o+1]=s1; + } + for (; o < out_dim; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 wlo, whi; + bf16x16_to_f32(w + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); + a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); + } + a0 = _mm256_add_ps(a0, a1); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum+=wv*x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + + for (int o = start; o < end; o++) { + const uint16_t *w = W_bf16 + (size_t)o * in_dim; + __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); + int k = 0; + for (; k + 16 <= in_dim; k += 16) { + __m256 wlo, whi; + bf16x16_to_f32(w + k, &wlo, &whi); + a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); + a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); + } + a0 = _mm256_add_ps(a0, a1); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); + r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; k < in_dim; k++) { + uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); + sum+=wv*x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + *best_out = best; + *best_val_out = best_val; +} + +#endif /* AVX-512F+BW vs AVX2 for bf16 */ + +/* ===================================================================== + * f32 attention helpers - AVX2+FMA, with AVX-512F when available + * (operates on L1-resident head vectors) + * ===================================================================== */ + +float qwen_dot_f32_avx(const float *a, const float *b, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); + } + __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); + for (; i + 16 <= n; i += 16) { + acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +#else + int i = 0; + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+8), _mm256_loadu_ps(b+i+8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+16), _mm256_loadu_ps(b+i+16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i+24), _mm256_loadu_ps(b+i+24), acc3); + } + acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + for (; i + 8 <= n; i += 8) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc0); + } + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +#endif +} + +void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 s = _mm512_set1_ps(scale); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); + _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); + _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + } + for (; i < n; i++) dst[i] *= scale; +#else + int i = 0; + __m256 s = _mm256_set1_ps(scale); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); + _mm256_storeu_ps(dst+i+8, _mm256_mul_ps(_mm256_loadu_ps(dst+i+8), s)); + _mm256_storeu_ps(dst+i+16, _mm256_mul_ps(_mm256_loadu_ps(dst+i+16), s)); + _mm256_storeu_ps(dst+i+24, _mm256_mul_ps(_mm256_loadu_ps(dst+i+24), s)); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_mul_ps(_mm256_loadu_ps(dst+i), s)); + } + for (; i < n; i++) dst[i] *= scale; +#endif +} + +void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 a = _mm512_set1_ps(alpha); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + _mm512_storeu_ps(dst + i + 16, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, _mm512_loadu_ps(dst + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +#else + int i = 0; + __m256 a = _mm256_set1_ps(alpha); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); + _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+8), a, _mm256_loadu_ps(dst+i+8))); + _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+16), a, _mm256_loadu_ps(dst+i+16))); + _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(src+i+24), a, _mm256_loadu_ps(dst+i+24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(src+i), a, _mm256_loadu_ps(dst+i))); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +#endif +} + +void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { +#if defined(__AVX512F__) + int i = 0; + __m512 c = _mm512_set1_ps(correction); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + _mm512_storeu_ps(dst + i + 16, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, _mm512_loadu_ps(src + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +#else + int i = 0; + __m256 c = _mm256_set1_ps(correction); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); + _mm256_storeu_ps(dst+i+8, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+8), c, _mm256_loadu_ps(src+i+8))); + _mm256_storeu_ps(dst+i+16, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+16), c, _mm256_loadu_ps(src+i+16))); + _mm256_storeu_ps(dst+i+24, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i+24), c, _mm256_loadu_ps(src+i+24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst+i, _mm256_fmadd_ps(_mm256_loadu_ps(dst+i), c, _mm256_loadu_ps(src+i))); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +#endif +} + +#endif /* __AVX2__ && __FMA__ */ diff --git a/qwen_kernels_generic.c b/qwen_kernels_generic.c new file mode 100644 index 0000000..cc25d3a --- /dev/null +++ b/qwen_kernels_generic.c @@ -0,0 +1,65 @@ +/* + * qwen_kernels_generic.c - architecture-generic hot kernels + */ + +#include "qwen_kernels_impl.h" + +#include + +void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + for (int o = 0; o < out_dim; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = bias ? bias[o] : 0.0f; + for (int k = 0; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + + for (int o = start; o < end; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = 0.0f; + for (int k = 0; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + if (sum > best_val) { + best_val = sum; + best = o; + } + } + + *best_out = best; + *best_val_out = best_val; +} + +float qwen_dot_f32_generic(const float *a, const float *b, int n) { + float sum = 0.0f; + for (int i = 0; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) { + for (int i = 0; i < n; i++) dst[i] *= scale; +} + +void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { + for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; +} + +void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { + for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; +} diff --git a/qwen_kernels_impl.h b/qwen_kernels_impl.h new file mode 100644 index 0000000..4a351a3 --- /dev/null +++ b/qwen_kernels_impl.h @@ -0,0 +1,65 @@ +/* + * qwen_kernels_impl.h - internal architecture dispatch for hot kernels + */ + +#ifndef QWEN_KERNELS_IMPL_H +#define QWEN_KERNELS_IMPL_H + +#include + +void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_generic(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON +void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_neon(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); + +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_neon +#define qwen_dot_f32_impl qwen_dot_f32_neon +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon +#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon + +#elif defined(__AVX2__) && defined(__FMA__) +void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim); +void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out); +float qwen_dot_f32_avx(const float *a, const float *b, int n); +void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); +void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); +void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); + +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_avx +#define qwen_dot_f32_impl qwen_dot_f32_avx +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx +#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx + +#else +#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic +#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_generic +#define qwen_dot_f32_impl qwen_dot_f32_generic +#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic +#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic +#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic +#endif + +#endif /* QWEN_KERNELS_IMPL_H */ diff --git a/qwen_kernels_neon.c b/qwen_kernels_neon.c new file mode 100644 index 0000000..0060e88 --- /dev/null +++ b/qwen_kernels_neon.c @@ -0,0 +1,274 @@ +/* + * qwen_kernels_neon.c - ARM NEON hot kernels + */ + +#include "qwen_kernels_impl.h" + +#ifdef __ARM_NEON + +#include +#include + +void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, + const float *bias, int in_dim, int out_dim) { + int o = 0; + + /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */ + for (; o + 1 < out_dim; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; + float s0 = bias ? bias[o] : 0.0f; + float s1 = bias ? bias[o + 1] : 0.0f; + + float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); + float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); + float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); + float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + float32x4_t x2 = vld1q_f32(x + k + 8); + float32x4_t x3 = vld1q_f32(x + k + 12); + float32x4_t x4 = vld1q_f32(x + k + 16); + float32x4_t x5 = vld1q_f32(x + k + 20); + float32x4_t x6 = vld1q_f32(x + k + 24); + float32x4_t x7 = vld1q_f32(x + k + 28); + + uint16x8_t r0a = vld1q_u16(w0 + k); + uint16x8_t r0b = vld1q_u16(w0 + k + 8); + uint16x8_t r0c = vld1q_u16(w0 + k + 16); + uint16x8_t r0d = vld1q_u16(w0 + k + 24); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); + + uint16x8_t r1a = vld1q_u16(w1 + k); + uint16x8_t r1b = vld1q_u16(w1 + k + 8); + uint16x8_t r1c = vld1q_u16(w1 + k + 16); + uint16x8_t r1d = vld1q_u16(w1 + k + 24); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); + } + for (; k + 8 <= in_dim; k += 8) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + uint16x8_t r0 = vld1q_u16(w0 + k); + uint16x8_t r1 = vld1q_u16(w1 + k); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0), 16)), x1); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1), 16)), x1); + } + s0 += vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); + s1 += vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); + + for (; k < in_dim; k++) { + uint32_t bits0 = ((uint32_t)w0[k]) << 16; + uint32_t bits1 = ((uint32_t)w1[k]) << 16; + float wv0, wv1; + memcpy(&wv0, &bits0, sizeof(float)); + memcpy(&wv1, &bits1, sizeof(float)); + s0 += wv0 * x[k]; + s1 += wv1 * x[k]; + } + y[o] = s0; + y[o + 1] = s1; + } + + /* Handle remaining odd row */ + for (; o < out_dim; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = bias ? bias[o] : 0.0f; + int k = 0; + + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; k + 8 <= in_dim; k += 8) { + uint16x8_t bf = vld1q_u16(w_row + k); + acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), + vld1q_f32(x + k)); + acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), + vld1q_f32(x + k + 4)); + } + sum += vaddvq_f32(vaddq_f32(acc0, acc1)); + + for (; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + y[o] = sum; + } +} + +void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, + int in_dim, int start, int end, + int *best_out, float *best_val_out) { + int best = start; + float best_val = -1e30f; + int o = start; + + /* Process 2 rows at a time, 32 elements/iter, 8 accumulators per row */ + for (; o + 1 < end; o += 2) { + const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; + const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; + float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); + float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); + float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); + float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); + int k = 0; + + for (; k + 32 <= in_dim; k += 32) { + float32x4_t x0 = vld1q_f32(x + k); + float32x4_t x1 = vld1q_f32(x + k + 4); + float32x4_t x2 = vld1q_f32(x + k + 8); + float32x4_t x3 = vld1q_f32(x + k + 12); + float32x4_t x4 = vld1q_f32(x + k + 16); + float32x4_t x5 = vld1q_f32(x + k + 20); + float32x4_t x6 = vld1q_f32(x + k + 24); + float32x4_t x7 = vld1q_f32(x + k + 28); + + uint16x8_t r0a = vld1q_u16(w0 + k); + uint16x8_t r0b = vld1q_u16(w0 + k + 8); + uint16x8_t r0c = vld1q_u16(w0 + k + 16); + uint16x8_t r0d = vld1q_u16(w0 + k + 24); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); + a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); + a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); + a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); + a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); + + uint16x8_t r1a = vld1q_u16(w1 + k); + uint16x8_t r1b = vld1q_u16(w1 + k + 8); + uint16x8_t r1c = vld1q_u16(w1 + k + 16); + uint16x8_t r1d = vld1q_u16(w1 + k + 24); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); + b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); + b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); + b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); + b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); + } + + float s0 = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); + float s1 = vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); + + for (; k < in_dim; k++) { + uint32_t bits0 = ((uint32_t)w0[k]) << 16; + uint32_t bits1 = ((uint32_t)w1[k]) << 16; + float wv0, wv1; + memcpy(&wv0, &bits0, sizeof(float)); + memcpy(&wv1, &bits1, sizeof(float)); + s0 += wv0 * x[k]; + s1 += wv1 * x[k]; + } + + if (s0 > best_val) { best_val = s0; best = o; } + if (s1 > best_val) { best_val = s1; best = o + 1; } + } + + for (; o < end; o++) { + const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; + float sum = 0.0f; + int k = 0; + + float32x4_t acc0 = vdupq_n_f32(0.0f), acc1 = vdupq_n_f32(0.0f); + for (; k + 8 <= in_dim; k += 8) { + uint16x8_t bf = vld1q_u16(w_row + k); + acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), + vld1q_f32(x + k)); + acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), + vld1q_f32(x + k + 4)); + } + sum += vaddvq_f32(vaddq_f32(acc0, acc1)); + + for (; k < in_dim; k++) { + uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; + float w_val; + memcpy(&w_val, &f32_bits, sizeof(float)); + sum += w_val * x[k]; + } + if (sum > best_val) { best_val = sum; best = o; } + } + + *best_out = best; + *best_val_out = best_val; +} + +float qwen_dot_f32_neon(const float *a, const float *b, int n) { + int i = 0; + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; i + 8 <= n; i += 8) { + float32x4_t a0 = vld1q_f32(a + i); + float32x4_t b0 = vld1q_f32(b + i); + float32x4_t a1 = vld1q_f32(a + i + 4); + float32x4_t b1 = vld1q_f32(b + i + 4); + acc0 = vfmaq_f32(acc0, a0, b0); + acc1 = vfmaq_f32(acc1, a1, b1); + } + float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { + int i = 0; + float32x4_t s = vdupq_n_f32(scale); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); + vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); + } + for (; i < n; i++) dst[i] *= scale; +} + +void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { + int i = 0; + float32x4_t a = vdupq_n_f32(alpha); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); + vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); + } + for (; i < n; i++) dst[i] += alpha * src[i]; +} + +void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { + int i = 0; + float32x4_t c = vdupq_n_f32(correction); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); + vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); + } + for (; i < n; i++) dst[i] = dst[i] * correction + src[i]; +} + +#endif /* __ARM_NEON */ diff --git a/qwen_safetensors.c b/qwen_safetensors.c new file mode 100644 index 0000000..37a5d3e --- /dev/null +++ b/qwen_safetensors.c @@ -0,0 +1,481 @@ +#include "qwen_safetensors.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* ======================================================================== + * Minimal JSON parser for safetensors header + * ======================================================================== */ + +static void skip_whitespace(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; +} + +static int parse_string(const char **p, char *out, size_t max_len) { + skip_whitespace(p); + if (**p != '"') return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') out[i++] = '\n'; + else if (**p == 't') out[i++] = '\t'; + else if (**p == '"') out[i++] = '"'; + else if (**p == '\\') out[i++] = '\\'; + else out[i++] = **p; + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') return -1; + (*p)++; + return 0; +} + +static int parse_int(const char **p, int64_t *out) { + skip_whitespace(p); + int64_t val = 0; + int neg = 0; + if (**p == '-') { neg = 1; (*p)++; } + if (**p < '0' || **p > '9') return -1; + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + *out = neg ? -val : val; + return 0; +} + +static safetensor_dtype_t parse_dtype(const char *s) { + if (strcmp(s, "F32") == 0) return DTYPE_F32; + if (strcmp(s, "F16") == 0) return DTYPE_F16; + if (strcmp(s, "BF16") == 0) return DTYPE_BF16; + if (strcmp(s, "I32") == 0) return DTYPE_I32; + if (strcmp(s, "I64") == 0) return DTYPE_I64; + if (strcmp(s, "BOOL") == 0) return DTYPE_BOOL; + return DTYPE_UNKNOWN; +} + +static int parse_tensor_entry(const char **p, safetensor_t *t) { + skip_whitespace(p); + if (**p != '{') return -1; + (*p)++; + + t->dtype = DTYPE_UNKNOWN; + t->ndim = 0; + t->data_offset = 0; + t->data_size = 0; + + while (**p && **p != '}') { + skip_whitespace(p); + if (**p == ',') { (*p)++; continue; } + + char key[64]; + if (parse_string(p, key, sizeof(key)) != 0) return -1; + skip_whitespace(p); + if (**p != ':') return -1; + (*p)++; + skip_whitespace(p); + + if (strcmp(key, "dtype") == 0) { + char dtype_str[32]; + if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) return -1; + t->dtype = parse_dtype(dtype_str); + } else if (strcmp(key, "shape") == 0) { + if (**p != '[') return -1; + (*p)++; + t->ndim = 0; + while (**p && **p != ']') { + skip_whitespace(p); + if (**p == ',') { (*p)++; continue; } + if (t->ndim >= 8) return -1; + if (parse_int(p, &t->shape[t->ndim]) != 0) return -1; + t->ndim++; + } + if (**p != ']') return -1; + (*p)++; + } else if (strcmp(key, "data_offsets") == 0) { + if (**p != '[') return -1; + (*p)++; + skip_whitespace(p); + int64_t start_i; + if (parse_int(p, &start_i) != 0) return -1; + skip_whitespace(p); + if (**p != ',') return -1; + (*p)++; + skip_whitespace(p); + int64_t end_i; + if (parse_int(p, &end_i) != 0) return -1; + if (start_i < 0 || end_i < start_i) return -1; + t->data_offset = (size_t)start_i; + t->data_size = (size_t)(end_i - start_i); + skip_whitespace(p); + if (**p != ']') return -1; + (*p)++; + } else { + /* Skip unknown value */ + if (**p == '"') { + (*p)++; + while (**p && **p != '"') { + if (**p == '\\') (*p)++; + if (**p) (*p)++; + } + if (**p == '"') (*p)++; + } else if (**p == '[') { + int depth = 1; (*p)++; + while (**p && depth > 0) { + if (**p == '[') depth++; + else if (**p == ']') depth--; + (*p)++; + } + } else if (**p == '{') { + int depth = 1; (*p)++; + while (**p && depth > 0) { + if (**p == '{') depth++; + else if (**p == '}') depth--; + (*p)++; + } + } else { + while (**p && **p != ',' && **p != '}') (*p)++; + } + } + } + if (**p != '}') return -1; + (*p)++; + return 0; +} + +static int parse_header(safetensors_file_t *sf) { + const char *p = sf->header_json; + skip_whitespace(&p); + if (*p != '{') return -1; + p++; + + sf->num_tensors = 0; + + while (*p && *p != '}') { + skip_whitespace(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char name[256]; + if (parse_string(&p, name, sizeof(name)) != 0) return -1; + skip_whitespace(&p); + if (*p != ':') return -1; + p++; + + if (strcmp(name, "__metadata__") == 0) { + skip_whitespace(&p); + if (*p == '{') { + int depth = 1; p++; + while (*p && depth > 0) { + if (*p == '{') depth++; + else if (*p == '}') depth--; + p++; + } + } + continue; + } + + if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { + fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", + sf->path ? sf->path : "", SAFETENSORS_MAX_TENSORS); + return -1; + } + + safetensor_t *t = &sf->tensors[sf->num_tensors]; + snprintf(t->name, sizeof(t->name), "%s", name); + if (parse_tensor_entry(&p, t) != 0) return -1; + sf->num_tensors++; + } + if (*p != '}') return -1; + return 0; +} + +/* ======================================================================== + * Single file operations + * ======================================================================== */ + +safetensors_file_t *safetensors_open(const char *path) { + int fd = open(path, O_RDONLY); + if (fd < 0) return NULL; + + struct stat st; + if (fstat(fd, &st) < 0) { close(fd); return NULL; } + + size_t file_size = (size_t)st.st_size; + if (file_size < 8) { close(fd); return NULL; } + + void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (data == MAP_FAILED) return NULL; + + uint64_t header_size = 0; + memcpy(&header_size, data, 8); + if (header_size > file_size - 8) { munmap(data, file_size); return NULL; } + + safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); + if (!sf) { munmap(data, file_size); return NULL; } + + sf->path = strdup(path); + sf->data = data; + sf->file_size = file_size; + sf->header_size = (size_t)header_size; + + sf->header_json = malloc(header_size + 1); + if (!sf->header_json) { safetensors_close(sf); return NULL; } + memcpy(sf->header_json, (char *)data + 8, header_size); + sf->header_json[header_size] = '\0'; + + if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } + + return sf; +} + +void safetensors_close(safetensors_file_t *sf) { + if (!sf) return; + if (sf->data) munmap(sf->data, sf->file_size); + free(sf->path); + free(sf->header_json); + free(sf); +} + +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { + return (const char *)sf->data + 8 + sf->header_size + t->data_offset; +} + +int64_t safetensor_numel(const safetensor_t *t) { + int64_t n = 1; + for (int i = 0; i < t->ndim; i++) n *= t->shape[i]; + return n; +} + +static float bf16_to_f32(uint16_t bf16) { + uint32_t f32 = ((uint32_t)bf16) << 16; + float result; + memcpy(&result, &f32, sizeof(float)); + return result; +} + +float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { + int64_t n = safetensor_numel(t); + if (n <= 0) return NULL; + + float *out = malloc(n * sizeof(float)); + if (!out) return NULL; + + const void *data = safetensors_data(sf, t); + + switch (t->dtype) { + case DTYPE_F32: + memcpy(out, data, n * sizeof(float)); + break; + case DTYPE_BF16: { + const uint16_t *src = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) out[i] = bf16_to_f32(src[i]); + break; + } + default: + free(out); + return NULL; + } + return out; +} + +int safetensor_is_bf16(const safetensor_t *t) { + return t && t->dtype == DTYPE_BF16; +} + +uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t) { + if (!sf || !t || t->dtype != DTYPE_BF16) return NULL; + return (uint16_t *)safetensors_data(sf, t); +} + +void safetensor_print(const safetensor_t *t) { + const char *dtype_names[] = {"F32", "F16", "BF16", "I32", "I64", "BOOL"}; + const char *dtype_name = t->dtype >= 0 && t->dtype <= 5 ? + dtype_names[t->dtype] : "UNKNOWN"; + printf("%s: dtype=%s, shape=[", t->name, dtype_name); + for (int i = 0; i < t->ndim; i++) { + printf("%ld%s", (long)t->shape[i], i < t->ndim - 1 ? ", " : ""); + } + printf("]\n"); +} + +void safetensors_print_all(const safetensors_file_t *sf) { + printf("File: %s (%d tensors)\n", sf->path, sf->num_tensors); + for (int i = 0; i < sf->num_tensors; i++) safetensor_print(&sf->tensors[i]); +} + +/* ======================================================================== + * Multi-shard operations + * ======================================================================== */ + +static int cmp_shard_names(const void *a, const void *b) { + const char *sa = (const char *)a; + const char *sb = (const char *)b; + return strcmp(sa, sb); +} + +static int parse_shard_name(const char *name, int *idx, int *total) { + int i = 0, n = 0, consumed = 0; + if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) + return -1; + if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) + return -1; + *idx = i; + *total = n; + return 0; +} + +multi_safetensors_t *multi_safetensors_open(const char *model_dir) { + multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); + if (!ms) return NULL; + + char path[4096]; + + /* Try single file first */ + snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); + safetensors_file_t *sf = safetensors_open(path); + if (sf) { + ms->shards[0] = sf; + ms->num_shards = 1; + return ms; + } + + /* Scan directory for shard files */ + DIR *dir = opendir(model_dir); + if (!dir) { free(ms); return NULL; } + + struct dirent *entry; + char shard_names[SAFETENSORS_MAX_SHARDS][256]; + int seen[SAFETENSORS_MAX_SHARDS] = {0}; + int n_shards = 0; + int expected_total = 0; + + while ((entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "model-", 6) == 0 && + strstr(entry->d_name, ".safetensors") != NULL) { + int shard_idx = 0, shard_total = 0; + if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { + fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", + entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (shard_total > SAFETENSORS_MAX_SHARDS) { + fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", + shard_total, SAFETENSORS_MAX_SHARDS); + closedir(dir); + free(ms); + return NULL; + } + if (expected_total == 0) { + expected_total = shard_total; + } else if (expected_total != shard_total) { + fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", + entry->d_name); + closedir(dir); + free(ms); + return NULL; + } + if (seen[shard_idx - 1]) { + fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", + shard_idx); + closedir(dir); + free(ms); + return NULL; + } + seen[shard_idx - 1] = 1; + + snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), + "%s", entry->d_name); + n_shards++; + } + } + closedir(dir); + + if (n_shards == 0) { + fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); + free(ms); + return NULL; + } + if (expected_total != n_shards) { + fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", + expected_total, n_shards, model_dir); + free(ms); + return NULL; + } + for (int i = 0; i < expected_total; i++) { + if (!seen[i]) { + fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", + i + 1, expected_total, model_dir); + free(ms); + return NULL; + } + } + + /* Sort shard names to ensure consistent ordering */ + qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); + + /* Open each shard */ + for (int i = 0; i < n_shards; i++) { + snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); + ms->shards[i] = safetensors_open(path); + if (!ms->shards[i]) { + fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); + multi_safetensors_close(ms); + return NULL; + } + } + ms->num_shards = n_shards; + return ms; +} + +void multi_safetensors_close(multi_safetensors_t *ms) { + if (!ms) return; + for (int i = 0; i < ms->num_shards; i++) { + safetensors_close(ms->shards[i]); + } + free(ms); +} + +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, + size_t *out_nbytes) { + if (!ms || !out_nbytes) return -1; + size_t total = 0; + for (int s = 0; s < ms->num_shards; s++) { + const safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + size_t n = sf->tensors[i].data_size; + if (n > SIZE_MAX - total) return -1; + total += n; + } + } + *out_nbytes = total; + return 0; +} + +const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, + const char *name, + safetensors_file_t **out_sf) { + for (int s = 0; s < ms->num_shards; s++) { + safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) { + if (out_sf) *out_sf = sf; + return &sf->tensors[i]; + } + } + } + return NULL; +} diff --git a/qwen_safetensors.h b/qwen_safetensors.h new file mode 100644 index 0000000..a93e720 --- /dev/null +++ b/qwen_safetensors.h @@ -0,0 +1,79 @@ +/* + * qwen_safetensors.h - Safetensors file format reader (multi-shard support) + * Adapted from voxtral-realtime project. + */ + +#ifndef QWEN_SAFETENSORS_H +#define QWEN_SAFETENSORS_H + +#include +#include + +#define SAFETENSORS_MAX_TENSORS 1024 +#define SAFETENSORS_MAX_SHARDS 8 + +typedef enum { + DTYPE_F32 = 0, + DTYPE_F16 = 1, + DTYPE_BF16 = 2, + DTYPE_I32 = 3, + DTYPE_I64 = 4, + DTYPE_BOOL = 5, + DTYPE_UNKNOWN = -1 +} safetensor_dtype_t; + +typedef struct { + char name[256]; + safetensor_dtype_t dtype; + int ndim; + int64_t shape[8]; + size_t data_offset; + size_t data_size; +} safetensor_t; + +typedef struct { + char *path; + void *data; + size_t file_size; + size_t header_size; + char *header_json; + int num_tensors; + safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; +} safetensors_file_t; + +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ +typedef struct { + safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; + int num_shards; +} multi_safetensors_t; + +/* Open a single safetensors file (memory-mapped) */ +safetensors_file_t *safetensors_open(const char *path); +void safetensors_close(safetensors_file_t *sf); + +/* Open model from directory (auto-detects single file or multi-shard) */ +multi_safetensors_t *multi_safetensors_open(const char *model_dir); +void multi_safetensors_close(multi_safetensors_t *ms); +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, + size_t *out_nbytes); + +/* Find a tensor by name across all shards */ +const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, + const char *name, + safetensors_file_t **out_sf); + +/* Get raw pointer to tensor data (within mmap'd region) */ +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); + +/* Get tensor data as float32 (allocates, caller must free) */ +float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); + +/* Get direct pointer to bf16 data in mmap'd region (no copy) */ +uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t); + +int safetensor_is_bf16(const safetensor_t *t); +int64_t safetensor_numel(const safetensor_t *t); +void safetensor_print(const safetensor_t *t); +void safetensors_print_all(const safetensors_file_t *sf); + +#endif /* QWEN_SAFETENSORS_H */ diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c new file mode 100644 index 0000000..38ecb4e --- /dev/null +++ b/qwen_tokenizer.c @@ -0,0 +1,1523 @@ +#include "qwen_tokenizer.h" +#include "qwen_kernels.h" +#include +#include +#include +#include +#include + +/* ======================================================================== + * GPT-2 Bytes-to-Unicode Mapping + * ======================================================================== */ + +static int gpt2_byte_to_unicode[256]; +static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ +static int gpt2_mapping_initialized = 0; + +static void init_gpt2_mapping(void) { + if (gpt2_mapping_initialized) return; + + memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); + + int n = 0; + for (int b = 0; b < 256; b++) { + int is_normal = 0; + if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */ + if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */ + if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */ + + if (is_normal) gpt2_byte_to_unicode[b] = b; + else gpt2_byte_to_unicode[b] = 256 + n++; + } + + for (int b = 0; b < 256; b++) { + int cp = gpt2_byte_to_unicode[b]; + if (cp < 512) gpt2_unicode_to_byte[cp] = b; + } + + gpt2_mapping_initialized = 1; +} + +/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */ +static int utf8_encode_cp(int cp, char out[4]) { + if (cp < 0x80) { + out[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; +} + +static int utf8_encode_full_cp(int cp, char out[4]) { + if (cp < 0x80) return utf8_encode_cp(cp, out); + if (cp < 0x800) return utf8_encode_cp(cp, out); + if (cp < 0x10000) return utf8_encode_cp(cp, out); + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +/* + * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text. + * Returns allocated string (caller must free). + */ +static char *decode_gpt2_token(const char *token_str) { + init_gpt2_mapping(); + + size_t len = strlen(token_str); + unsigned char *bytes = (unsigned char *)malloc(len + 1); + if (!bytes) return NULL; + + int byte_count = 0; + const unsigned char *p = (const unsigned char *)token_str; + const unsigned char *end = p + len; + + while (p < end) { + int cp = 0; + int nbytes = 1; + + if ((*p & 0x80) == 0) { + cp = *p; + nbytes = 1; + } else if ((*p & 0xE0) == 0xC0) { + cp = (*p & 0x1F) << 6; + if (p + 1 < end) cp |= (p[1] & 0x3F); + nbytes = 2; + } else if ((*p & 0xF0) == 0xE0) { + cp = (*p & 0x0F) << 12; + if (p + 1 < end) cp |= (p[1] & 0x3F) << 6; + if (p + 2 < end) cp |= (p[2] & 0x3F); + nbytes = 3; + } else if ((*p & 0xF8) == 0xF0) { + cp = (*p & 0x07) << 18; + if (p + 1 < end) cp |= (p[1] & 0x3F) << 12; + if (p + 2 < end) cp |= (p[2] & 0x3F) << 6; + if (p + 3 < end) cp |= (p[3] & 0x3F); + nbytes = 4; + } else { + cp = *p; + nbytes = 1; + } + p += nbytes; + + if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) { + bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp]; + } else { + bytes[byte_count++] = '?'; + } + } + bytes[byte_count] = '\0'; + return (char *)bytes; +} + +/* ======================================================================== + * Simple JSON parser for vocab.json + * ======================================================================== */ + +static void skip_ws(const char **p) { + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; +} + +static int parse_json_string(const char **p, char *out, size_t max_len) { + skip_ws(p); + if (**p != '"') return -1; + (*p)++; + size_t i = 0; + while (**p && **p != '"' && i < max_len - 1) { + if (**p == '\\') { + (*p)++; + if (**p == 'n') out[i++] = '\n'; + else if (**p == 't') out[i++] = '\t'; + else if (**p == '"') out[i++] = '"'; + else if (**p == '\\') out[i++] = '\\'; + else if (**p == '/') out[i++] = '/'; + else if (**p == 'u') { + (*p)++; + unsigned int cp = 0; + for (int j = 0; j < 4 && **p; j++, (*p)++) { + cp <<= 4; + char c = **p; + if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0'); + else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10); + } + if (cp < 0x80 && i + 1 < max_len) { + out[i++] = (char)cp; + } else if (cp < 0x800 && i + 2 < max_len) { + out[i++] = (char)(0xC0 | (cp >> 6)); + out[i++] = (char)(0x80 | (cp & 0x3F)); + } else if (i + 3 < max_len) { + out[i++] = (char)(0xE0 | (cp >> 12)); + out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[i++] = (char)(0x80 | (cp & 0x3F)); + } + continue; + } else { + out[i++] = **p; + } + } else { + out[i++] = **p; + } + (*p)++; + } + out[i] = '\0'; + if (**p != '"') return -1; + (*p)++; + return 0; +} + +static int64_t parse_json_int(const char **p) { + skip_ws(p); + int neg = 0; + if (**p == '-') { neg = 1; (*p)++; } + int64_t val = 0; + while (**p >= '0' && **p <= '9') { + val = val * 10 + (**p - '0'); + (*p)++; + } + return neg ? -val : val; +} + +/* ======================================================================== + * String->int hash map (open addressing) + * ======================================================================== */ + +typedef struct { + char *key; /* NULL means empty slot */ + int value; +} str_int_entry_t; + +static uint64_t fnv1a_hash(const char *s) { + uint64_t h = 1469598103934665603ULL; + while (*s) { + h ^= (unsigned char)*s++; + h *= 1099511628211ULL; + } + return h; +} + +static uint64_t fnv1a_hash_pair(const char *a, const char *b) { + uint64_t h = 1469598103934665603ULL; + while (*a) { + h ^= (unsigned char)*a++; + h *= 1099511628211ULL; + } + h ^= (unsigned char)' '; + h *= 1099511628211ULL; + while (*b) { + h ^= (unsigned char)*b++; + h *= 1099511628211ULL; + } + return h; +} + +static int pair_key_eq(const char *key, const char *a, const char *b) { + while (*a) { + if (*key++ != *a++) return 0; + } + if (*key++ != ' ') return 0; + while (*b) { + if (*key++ != *b++) return 0; + } + return *key == '\0'; +} + +static int next_pow2(int x) { + int p = 1; + while (p < x) p <<= 1; + return p; +} + +static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { + if (!map || cap <= 0 || !key) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) { + map[idx].key = key; + map[idx].value = value; + return 0; + } + if (strcmp(map[idx].key, key) == 0) { + map[idx].value = value; + return 0; + } + } + return -1; +} + +static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) { + if (!keyp) return -1; + char *key = *keyp; + if (!map || cap <= 0 || !key) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) { + map[idx].key = key; + map[idx].value = value; + *keyp = NULL; + return 0; + } + if (strcmp(map[idx].key, key) == 0) { + map[idx].value = value; + return 0; + } + } + return -1; +} + +static int map_get(const str_int_entry_t *map, int cap, const char *key) { + if (!map || cap <= 0 || !key) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) return -1; + if (strcmp(map[idx].key, key) == 0) return map[idx].value; + } + return -1; +} + +static int map_get_pair(const str_int_entry_t *map, int cap, + const char *a, const char *b) { + if (!map || cap <= 0 || !a || !b) return -1; + int mask = cap - 1; + int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask); + for (int i = 0; i < cap; i++) { + int idx = (pos + i) & mask; + if (!map[idx].key) return -1; + if (pair_key_eq(map[idx].key, a, b)) return map[idx].value; + } + return -1; +} + +/* ======================================================================== + * BPE helpers + * ======================================================================== */ + +static int utf8_char_len(unsigned char c) { + if ((c & 0x80) == 0) return 1; + if ((c & 0xE0) == 0xC0) return 2; + if ((c & 0xF0) == 0xE0) return 3; + if ((c & 0xF8) == 0xF0) return 4; + return 1; +} + +static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { + *out_syms = NULL; + *out_n = 0; + if (!s || !*s) return 0; + + int n = 0; + for (const unsigned char *p = (const unsigned char *)s; *p; ) { + int l = utf8_char_len(*p); + p += l; + n++; + } + if (n <= 0) return 0; + + char **syms = (char **)calloc((size_t)n, sizeof(char *)); + if (!syms) return -1; + + int i = 0; + for (const unsigned char *p = (const unsigned char *)s; *p; ) { + int l = utf8_char_len(*p); + syms[i] = (char *)malloc((size_t)l + 1); + if (!syms[i]) { + for (int j = 0; j < i; j++) free(syms[j]); + free(syms); + return -1; + } + memcpy(syms[i], p, (size_t)l); + syms[i][l] = '\0'; + p += l; + i++; + } + + *out_syms = syms; + *out_n = n; + return 0; +} + +static char *str_concat2(const char *a, const char *b) { + size_t la = strlen(a), lb = strlen(b); + char *r = (char *)malloc(la + lb + 1); + if (!r) return NULL; + memcpy(r, a, la); + memcpy(r + la, b, lb); + r[la + lb] = '\0'; + return r; +} + +static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { + if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; + + int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, + tok->merge_map_cap, a, b); + return rank >= 0 ? rank : INT_MAX; +} + +static int append_id(int **arr, int *n, int *cap, int id) { + if (*n >= *cap) { + int new_cap = (*cap == 0) ? 16 : (*cap * 2); + int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); + if (!tmp) return -1; + *arr = tmp; + *cap = new_cap; + } + (*arr)[(*n)++] = id; + return 0; +} + +/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */ +static char *text_to_bpe_unicode(const char *text) { + init_gpt2_mapping(); + size_t len = strlen(text); + /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ + char *out = (char *)malloc(len * 3 + 1); + if (!out) return NULL; + + size_t w = 0; + for (size_t i = 0; i < len; i++) { + int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; + char tmp[4]; + int n = utf8_encode_cp(cp, tmp); + for (int j = 0; j < n; j++) out[w++] = tmp[j]; + } + out[w] = '\0'; + return out; +} + +static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, + int **out_ids, int *out_n); + +static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, + int *cp_out, int *len_out) { + if (p >= end || *p == '\0') return 0; + + unsigned char c = *p; + if ((c & 0x80) == 0) { + *cp_out = c; + *len_out = 1; + return 1; + } + if ((c & 0xE0) == 0xC0 && p + 1 < end && + (p[1] & 0xC0) == 0x80) { + *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F); + *len_out = 2; + return 1; + } + if ((c & 0xF0) == 0xE0 && p + 2 < end && + (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { + *cp_out = ((c & 0x0F) << 12) | + ((p[1] & 0x3F) << 6) | + (p[2] & 0x3F); + *len_out = 3; + return 1; + } + if ((c & 0xF8) == 0xF0 && p + 3 < end && + (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && + (p[3] & 0xC0) == 0x80) { + *cp_out = ((c & 0x07) << 18) | + ((p[1] & 0x3F) << 12) | + ((p[2] & 0x3F) << 6) | + (p[3] & 0x3F); + *len_out = 4; + return 1; + } + + *cp_out = c; + *len_out = 1; + return 1; +} + +static int is_ascii_alpha_cp(int cp) { + return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'); +} + +static int is_number_cp(int cp) { + if (cp >= '0' && cp <= '9') return 1; + if (cp >= 0x0660 && cp <= 0x0669) return 1; + if (cp >= 0x06F0 && cp <= 0x06F9) return 1; + if (cp >= 0x0966 && cp <= 0x096F) return 1; + if (cp >= 0xFF10 && cp <= 0xFF19) return 1; + return 0; +} + +static int is_space_cp(int cp) { + return cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r' || + cp == '\f' || cp == '\v' || cp == 0x85 || cp == 0xA0 || + cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || + cp == 0x2028 || cp == 0x2029 || cp == 0x202F || + cp == 0x205F || cp == 0x3000; +} + +static int is_symbolish_cp(int cp) { + if (cp < 0x80) return 0; + if (cp >= 0x0300 && cp <= 0x036F) return 1; + if (cp >= 0x00A1 && cp <= 0x00BF && + cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) + return 1; + if (cp == 0x00D7 || cp == 0x00F7) return 1; + if (cp >= 0x064B && cp <= 0x065F) return 1; + if ((cp >= 0x0900 && cp <= 0x0903) || + (cp >= 0x093A && cp <= 0x094F) || + (cp >= 0x0951 && cp <= 0x0957) || + (cp >= 0x0962 && cp <= 0x0963)) + return 1; + if (cp >= 0x2000 && cp <= 0x206F) return 1; + if (cp >= 0x2190 && cp <= 0x27BF) return 1; + if (cp >= 0x2E00 && cp <= 0x2E7F) return 1; + if (cp >= 0x3000 && cp <= 0x303F) return 1; + if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; + return 0; +} + +static int compose_latin_mark(int base, int mark) { + switch (mark) { + case 0x0300: + switch (base) { + case 'A': return 0x00C0; case 'E': return 0x00C8; + case 'I': return 0x00CC; case 'O': return 0x00D2; + case 'U': return 0x00D9; case 'a': return 0x00E0; + case 'e': return 0x00E8; case 'i': return 0x00EC; + case 'o': return 0x00F2; case 'u': return 0x00F9; + } + break; + case 0x0301: + switch (base) { + case 'A': return 0x00C1; case 'C': return 0x0106; + case 'E': return 0x00C9; case 'I': return 0x00CD; + case 'L': return 0x0139; case 'N': return 0x0143; + case 'O': return 0x00D3; case 'R': return 0x0154; + case 'S': return 0x015A; case 'U': return 0x00DA; + case 'Y': return 0x00DD; case 'Z': return 0x0179; + case 'a': return 0x00E1; case 'c': return 0x0107; + case 'e': return 0x00E9; case 'i': return 0x00ED; + case 'l': return 0x013A; case 'n': return 0x0144; + case 'o': return 0x00F3; case 'r': return 0x0155; + case 's': return 0x015B; case 'u': return 0x00FA; + case 'y': return 0x00FD; case 'z': return 0x017A; + } + break; + case 0x0302: + switch (base) { + case 'A': return 0x00C2; case 'E': return 0x00CA; + case 'I': return 0x00CE; case 'O': return 0x00D4; + case 'U': return 0x00DB; case 'a': return 0x00E2; + case 'e': return 0x00EA; case 'i': return 0x00EE; + case 'o': return 0x00F4; case 'u': return 0x00FB; + } + break; + case 0x0303: + switch (base) { + case 'A': return 0x00C3; case 'N': return 0x00D1; + case 'O': return 0x00D5; case 'a': return 0x00E3; + case 'n': return 0x00F1; case 'o': return 0x00F5; + } + break; + case 0x0308: + switch (base) { + case 'A': return 0x00C4; case 'E': return 0x00CB; + case 'I': return 0x00CF; case 'O': return 0x00D6; + case 'U': return 0x00DC; case 'Y': return 0x0178; + case 'a': return 0x00E4; case 'e': return 0x00EB; + case 'i': return 0x00EF; case 'o': return 0x00F6; + case 'u': return 0x00FC; case 'y': return 0x00FF; + } + break; + case 0x030A: + switch (base) { + case 'A': return 0x00C5; case 'a': return 0x00E5; + } + break; + case 0x0327: + switch (base) { + case 'C': return 0x00C7; case 'c': return 0x00E7; + } + break; + } + return 0; +} + +static char *normalize_nfc_latin(const char *text) { + const unsigned char *p = (const unsigned char *)text; + const unsigned char *end = p + strlen(text); + char *out = (char *)malloc((size_t)(end - p) * 2 + 1); + if (!out) return NULL; + + size_t w = 0; + while (p < end) { + const unsigned char *cur = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + p += len; + + if (p < end) { + int mark = 0, mark_len = 0; + utf8_decode_cp_len(p, end, &mark, &mark_len); + int composed = compose_latin_mark(cp, mark); + if (composed) { + char tmp[4]; + int n = utf8_encode_full_cp(composed, tmp); + for (int i = 0; i < n; i++) out[w++] = tmp[i]; + p += mark_len; + continue; + } + } + + memcpy(out + w, cur, (size_t)len); + w += (size_t)len; + } + + out[w] = '\0'; + return out; +} + +static int is_letter_cp(int cp) { + if (is_ascii_alpha_cp(cp)) return 1; + if (cp < 0x80 || is_space_cp(cp) || is_number_cp(cp) || + is_symbolish_cp(cp)) + return 0; + return 1; +} + +static int is_word_cp(int cp) { + return is_letter_cp(cp) || is_number_cp(cp); +} + +static int is_newline_cp(int cp) { + return cp == '\n' || cp == '\r'; +} + +static int ascii_lower(int c) { + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static int match_contraction(const unsigned char *p, const unsigned char *end) { + if (p >= end || *p != '\'') return 0; + int remain = (int)(end - p); + if (remain >= 2) { + int c1 = ascii_lower(p[1]); + if (c1 == 's' || c1 == 't' || c1 == 'm' || c1 == 'd') + return 2; + } + if (remain >= 3) { + int c1 = ascii_lower(p[1]); + int c2 = ascii_lower(p[2]); + if ((c1 == 'r' && c2 == 'e') || + (c1 == 'v' && c2 == 'e') || + (c1 == 'l' && c2 == 'l')) + return 3; + } + return 0; +} + +static int append_encoded_piece(const qwen_tokenizer_t *tok, + const char *piece, size_t piece_len, + int **ids, int *n_ids, int *cap) { + if (piece_len == 0) return 0; + + char *tmp = (char *)malloc(piece_len + 1); + if (!tmp) return -1; + memcpy(tmp, piece, piece_len); + tmp[piece_len] = '\0'; + + char *mapped = text_to_bpe_unicode(tmp); + free(tmp); + if (!mapped) return -1; + + int *piece_ids = NULL; + int n_piece = 0; + if (encode_bpe_word(tok, mapped, &piece_ids, &n_piece) != 0) { + free(mapped); + free(piece_ids); + return -1; + } + free(mapped); + + for (int i = 0; i < n_piece; i++) { + if (append_id(ids, n_ids, cap, piece_ids[i]) != 0) { + free(piece_ids); + return -1; + } + } + free(piece_ids); + return 0; +} + +static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, + int **out_ids, int *out_n) { + const unsigned char *start = (const unsigned char *)text; + const unsigned char *p = start; + const unsigned char *end = start + strlen(text); + int *ids = NULL; + int n_ids = 0, cap = 0; + + while (p < end) { + const unsigned char *piece = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + + int clen = match_contraction(p, end); + if (clen > 0) { + p += clen; + } else { + const unsigned char *next = p + len; + int next_cp = 0, next_len = 0; + int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); + + if (!is_newline_cp(cp) && !is_word_cp(cp) && + has_next && is_letter_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_letter_cp(cp)) { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_number_cp(cp)) { + p += len; + } else if (is_space_cp(cp)) { + const unsigned char *q = p; + int saw_newline = 0; + while (q < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q, end, &c, &l); + if (!is_space_cp(c)) break; + if (is_newline_cp(c)) saw_newline = 1; + q += l; + if (saw_newline) { + while (q < end) { + int c2 = 0, l2 = 0; + utf8_decode_cp_len(q, end, &c2, &l2); + if (!is_newline_cp(c2)) break; + q += l2; + } + break; + } + } + + if (saw_newline) { + p = q; + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && + !is_word_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } else { + const unsigned char *last = p; + const unsigned char *q2 = p; + int count = 0; + while (q2 < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q2, end, &c, &l); + if (!is_space_cp(c)) break; + last = q2; + q2 += l; + count++; + } + if (q2 < end && count > 1) + p = last; + else + p = q2; + } + } else { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } + } + + if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), + &ids, &n_ids, &cap) != 0) { + free(ids); + return -1; + } + } + + *out_ids = ids; + *out_n = n_ids; + return 0; +} + +/* Encode one mapped BPE unicode string to token IDs. */ +static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { + *out_ids = NULL; + *out_n = 0; + if (!mapped || !*mapped) return 0; + + char **syms = NULL; + int n_syms = 0; + if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1; + if (n_syms <= 0) { + free(syms); + return 0; + } + + while (n_syms > 1) { + int best_rank = INT_MAX; + int best_i = -1; + for (int i = 0; i < n_syms - 1; i++) { + int r = merge_rank(tok, syms[i], syms[i + 1]); + if (r < best_rank) { + best_rank = r; + best_i = i; + } + } + if (best_i < 0 || best_rank == INT_MAX) break; + + char *merged = str_concat2(syms[best_i], syms[best_i + 1]); + if (!merged) { + for (int i = 0; i < n_syms; i++) free(syms[i]); + free(syms); + return -1; + } + free(syms[best_i]); + free(syms[best_i + 1]); + syms[best_i] = merged; + for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1]; + n_syms--; + } + + int *ids = NULL; + int n_ids = 0, cap = 0; + for (int i = 0; i < n_syms; i++) { + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); + if (id < 0) { + /* Should not happen with valid vocab + merges + byte-level mapping. */ + for (int k = 0; k < n_syms; k++) free(syms[k]); + free(syms); + free(ids); + return -1; + } + if (append_id(&ids, &n_ids, &cap, id) != 0) { + for (int k = 0; k < n_syms; k++) free(syms[k]); + free(syms); + free(ids); + return -1; + } + } + + for (int i = 0; i < n_syms; i++) free(syms[i]); + free(syms); + + *out_ids = ids; + *out_n = n_ids; + return 0; +} + +typedef struct { + size_t off; +} bpe_sym_ref_t; + +struct qwen_tokenizer_workspace { + char *normalized; + size_t normalized_cap; + char *mapped; + size_t mapped_cap; + char *arena; + size_t arena_cap; + size_t arena_len; + bpe_sym_ref_t *syms; + int syms_cap; + int *ids; + int ids_cap; +}; + +static int reserve_bytes(char **buf, size_t *cap, size_t need) { + if (*cap >= need) return 0; + size_t new_cap = *cap ? *cap : 256; + while (new_cap < need) { + if (new_cap > SIZE_MAX / 2) return -1; + new_cap *= 2; + } + char *p = (char *)realloc(*buf, new_cap); + if (!p) return -1; + *buf = p; + *cap = new_cap; + return 0; +} + +static int reserve_ints(int **buf, int *cap, int need) { + if (*cap >= need) return 0; + int new_cap = *cap ? *cap : 64; + while (new_cap < need) { + if (new_cap > INT_MAX / 2) return -1; + new_cap *= 2; + } + int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); + if (!p) return -1; + *buf = p; + *cap = new_cap; + return 0; +} + +static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { + if (ws->syms_cap >= need) return 0; + int new_cap = ws->syms_cap ? ws->syms_cap : 64; + while (new_cap < need) { + if (new_cap > INT_MAX / 2) return -1; + new_cap *= 2; + } + bpe_sym_ref_t *p = + (bpe_sym_ref_t *)realloc(ws->syms, + (size_t)new_cap * sizeof(*ws->syms)); + if (!p) return -1; + ws->syms = p; + ws->syms_cap = new_cap; + return 0; +} + +static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, + size_t len, size_t *off) { + if (ws->arena_len > SIZE_MAX - len - 1) return -1; + size_t need = ws->arena_len + len + 1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1; + *off = ws->arena_len; + memcpy(ws->arena + ws->arena_len, s, len); + ws->arena_len += len; + ws->arena[ws->arena_len++] = '\0'; + return 0; +} + +static const char *arena_str(const qwen_tokenizer_workspace_t *ws, + bpe_sym_ref_t sym) { + return ws->arena + sym.off; +} + +static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, + const char *text, const char **out) { + const unsigned char *p = (const unsigned char *)text; + const unsigned char *end = p + strlen(text); + size_t in_len = (size_t)(end - p); + if (in_len > (SIZE_MAX - 1) / 2) return -1; + if (reserve_bytes(&ws->normalized, &ws->normalized_cap, + in_len * 2 + 1) != 0) + return -1; + + size_t w = 0; + while (p < end) { + const unsigned char *cur = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + p += len; + + if (p < end) { + int mark = 0, mark_len = 0; + utf8_decode_cp_len(p, end, &mark, &mark_len); + int composed = compose_latin_mark(cp, mark); + if (composed) { + char tmp[4]; + int n = utf8_encode_full_cp(composed, tmp); + memcpy(ws->normalized + w, tmp, (size_t)n); + w += (size_t)n; + p += mark_len; + continue; + } + } + + memcpy(ws->normalized + w, cur, (size_t)len); + w += (size_t)len; + } + + ws->normalized[w] = '\0'; + *out = ws->normalized; + return 0; +} + +static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, + const char *text, size_t len, + const char **out) { + init_gpt2_mapping(); + if (len > (SIZE_MAX - 1) / 3) return -1; + if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) + return -1; + + size_t w = 0; + for (size_t i = 0; i < len; i++) { + int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; + char tmp[4]; + int n = utf8_encode_cp(cp, tmp); + memcpy(ws->mapped + w, tmp, (size_t)n); + w += (size_t)n; + } + ws->mapped[w] = '\0'; + *out = ws->mapped; + return 0; +} + +static int output_append_id(int *out_ids, int out_cap, int *n_ids, + int *overflow, int id) { + if (*n_ids < out_cap && out_ids) + out_ids[*n_ids] = id; + else + *overflow = 1; + (*n_ids)++; + return 0; +} + +static int encode_bpe_word_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *mapped, + int *out_ids, int out_cap, + int *n_ids, int *overflow) { + if (!mapped || !*mapped) return 0; + + int n_syms = 0; + for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + int l = utf8_char_len(*p); + p += l; + n_syms++; + } + if (n_syms <= 0) return 0; + if (reserve_syms(ws, n_syms) != 0) return -1; + + ws->arena_len = 0; + int i = 0; + for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + int l = utf8_char_len(*p); + size_t off = 0; + if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) + return -1; + ws->syms[i++].off = off; + p += l; + } + + while (n_syms > 1) { + int best_rank = INT_MAX; + int best_i = -1; + for (int j = 0; j < n_syms - 1; j++) { + int r = merge_rank(tok, arena_str(ws, ws->syms[j]), + arena_str(ws, ws->syms[j + 1])); + if (r < best_rank) { + best_rank = r; + best_i = j; + } + } + if (best_i < 0 || best_rank == INT_MAX) break; + + const char *a = arena_str(ws, ws->syms[best_i]); + const char *b = arena_str(ws, ws->syms[best_i + 1]); + size_t la = strlen(a), lb = strlen(b); + if (la > SIZE_MAX - lb) return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, + ws->arena_len + la + lb + 1) != 0) + return -1; + + a = arena_str(ws, ws->syms[best_i]); + b = arena_str(ws, ws->syms[best_i + 1]); + size_t off = ws->arena_len; + memcpy(ws->arena + ws->arena_len, a, la); + ws->arena_len += la; + memcpy(ws->arena + ws->arena_len, b, lb); + ws->arena_len += lb; + ws->arena[ws->arena_len++] = '\0'; + ws->syms[best_i].off = off; + for (int j = best_i + 1; j < n_syms - 1; j++) + ws->syms[j] = ws->syms[j + 1]; + n_syms--; + } + + for (int j = 0; j < n_syms; j++) { + int id = map_get((const str_int_entry_t *)tok->vocab_map, + tok->vocab_map_cap, arena_str(ws, ws->syms[j])); + if (id < 0) return -1; + output_append_id(out_ids, out_cap, n_ids, overflow, id); + } + return 0; +} + +static int append_encoded_piece_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *piece, size_t piece_len, + int *out_ids, int out_cap, + int *n_ids, int *overflow) { + if (piece_len == 0) return 0; + + const char *mapped = NULL; + if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) + return -1; + return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, + overflow); +} + +static int encode_pretokenized_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n) { + const unsigned char *start = (const unsigned char *)text; + const unsigned char *p = start; + const unsigned char *end = start + strlen(text); + int n_ids = 0; + int overflow = 0; + + while (p < end) { + const unsigned char *piece = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + + int clen = match_contraction(p, end); + if (clen > 0) { + p += clen; + } else { + const unsigned char *next = p + len; + int next_cp = 0, next_len = 0; + int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); + + if (!is_newline_cp(cp) && !is_word_cp(cp) && + has_next && is_letter_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_letter_cp(cp)) { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_number_cp(cp)) { + p += len; + } else if (is_space_cp(cp)) { + const unsigned char *q = p; + int saw_newline = 0; + while (q < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q, end, &c, &l); + if (!is_space_cp(c)) break; + if (is_newline_cp(c)) saw_newline = 1; + q += l; + if (saw_newline) { + while (q < end) { + int c2 = 0, l2 = 0; + utf8_decode_cp_len(q, end, &c2, &l2); + if (!is_newline_cp(c2)) break; + q += l2; + } + break; + } + } + + if (saw_newline) { + p = q; + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && + !is_word_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } else { + const unsigned char *last = p; + const unsigned char *q2 = p; + int count = 0; + while (q2 < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q2, end, &c, &l); + if (!is_space_cp(c)) break; + last = q2; + q2 += l; + count++; + } + if (q2 < end && count > 1) + p = last; + else + p = q2; + } + } else { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } + } + + if (append_encoded_piece_into(tok, ws, (const char *)piece, + (size_t)(p - piece), out_ids, out_cap, + &n_ids, &overflow) != 0) + return -1; + } + + *out_n = n_ids; + return overflow ? -2 : 0; +} + +static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { + const char *slash = strrchr(vocab_path, '/'); + if (!slash) { + if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1; + return 0; + } + size_t dir_len = (size_t)(slash - vocab_path); + if (dir_len + 12 >= out_cap) return -1; + memcpy(out_path, vocab_path, dir_len); + out_path[dir_len] = '\0'; + snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); + return 0; +} + +static void trim_newline(char *line) { + size_t n = strlen(line); + while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0'; +} + +static int parse_merge_pair(char *line, char **a, char **b) { + *a = NULL; + *b = NULL; + trim_newline(line); + if (line[0] == '\0') return -1; + if (line[0] == '#') return -1; + + char *sp = strchr(line, ' '); + if (!sp) return -1; + *sp = '\0'; + char *p2 = sp + 1; + while (*p2 == ' ') p2++; + if (*line == '\0' || *p2 == '\0') return -1; + *a = line; + *b = p2; + return 0; +} + +static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { + FILE *f = fopen(merges_path, "rb"); + if (!f) return -1; + + int n_pairs = 0; + char line[8192]; + while (fgets(line, sizeof(line), f)) { + char *a = NULL, *b = NULL; + if (parse_merge_pair(line, &a, &b) == 0) n_pairs++; + } + if (n_pairs <= 0) { + fclose(f); + return -1; + } + + tok->merge_map_cap = next_pow2(n_pairs * 2); + tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t)); + if (!tok->merge_map) { + fclose(f); + tok->merge_map_cap = 0; + return -1; + } + + if (fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + int rank = 0; + while (fgets(line, sizeof(line), f)) { + char *a = NULL, *b = NULL; + if (parse_merge_pair(line, &a, &b) != 0) continue; + + size_t la = strlen(a), lb = strlen(b); + char *key = (char *)malloc(la + 1 + lb + 1); + if (!key) continue; + memcpy(key, a, la); + key[la] = ' '; + memcpy(key + la + 1, b, lb); + key[la + 1 + lb] = '\0'; + + (void)map_insert_owned((str_int_entry_t *)tok->merge_map, + tok->merge_map_cap, &key, rank); + free(key); + rank++; + } + fclose(f); + return 0; +} + +/* ======================================================================== + * Public API + * ======================================================================== */ + +qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { + FILE *f = fopen(vocab_json_path, "rb"); + if (!f) { + fprintf(stderr, "qwen_tokenizer_load: cannot open %s\n", vocab_json_path); + return NULL; + } + + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return NULL; + } + long file_size = ftell(f); + if (file_size < 0 || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return NULL; + } + + char *json = (char *)malloc((size_t)file_size + 1); + if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) { + fclose(f); + free(json); + return NULL; + } + fclose(f); + json[file_size] = '\0'; + + int max_id = 0; + const char *p = json; + skip_ws(&p); + if (*p != '{') { + free(json); + return NULL; + } + p++; + + const char *saved = p; + while (*p && *p != '}') { + skip_ws(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char key[4096] = {0}; + if (parse_json_string(&p, key, sizeof(key)) != 0) break; + skip_ws(&p); + if (*p != ':') break; + p++; + int64_t id = parse_json_int(&p); + if (id > max_id) max_id = (int)id; + } + + int vocab_size = max_id + 1; + qwen_tokenizer_t *tok = (qwen_tokenizer_t *)calloc(1, sizeof(qwen_tokenizer_t)); + if (!tok) { + free(json); + return NULL; + } + tok->vocab_size = vocab_size; + tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); + tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); + if (!tok->id_to_text || !tok->id_to_bpe) { + qwen_tokenizer_free(tok); + free(json); + return NULL; + } + + p = saved; + while (*p && *p != '}') { + skip_ws(&p); + if (*p == ',') { p++; continue; } + if (*p == '}') break; + + char key[4096] = {0}; + if (parse_json_string(&p, key, sizeof(key)) != 0) break; + skip_ws(&p); + if (*p != ':') break; + p++; + int64_t id = parse_json_int(&p); + + if (id >= 0 && id < vocab_size) { + free(tok->id_to_bpe[id]); + free(tok->id_to_text[id]); + tok->id_to_bpe[id] = strdup(key); + tok->id_to_text[id] = decode_gpt2_token(key); + } + } + free(json); + + int n_vocab_entries = 0; + for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++; + tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); + tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); + if (!tok->vocab_map) { + qwen_tokenizer_free(tok); + return NULL; + } + for (int i = 0; i < vocab_size; i++) { + if (tok->id_to_bpe[i]) { + map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i); + } + } + + char merges_path[1024]; + if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { + if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { + fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", + merges_path); + } + } + + return tok; +} + +const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { + if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; + return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; +} + +int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !text || text[0] == '\0') return NULL; + + char *normalized = normalize_nfc_latin(text); + if (!normalized) return NULL; + + int *ids = NULL; + int n_ids = 0; + if (encode_pretokenized(tok, normalized, &ids, &n_ids) != 0) { + free(normalized); + free(ids); + return NULL; + } + free(normalized); + + if (out_n_tokens) *out_n_tokens = n_ids; + return ids; +} + +qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { + return (qwen_tokenizer_workspace_t *)calloc(1, + sizeof(qwen_tokenizer_workspace_t)); +} + +void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { + if (!ws) return; + free(ws->normalized); + free(ws->mapped); + free(ws->arena); + free(ws->syms); + free(ws->ids); + free(ws); +} + +int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !ws || !text || out_cap < 0) return -1; + if (text[0] == '\0') return 0; + + const char *normalized = NULL; + if (normalize_nfc_latin_into(ws, text, &normalized) != 0) + return -1; + + int n_ids = 0; + int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, + &n_ids); + if (out_n_tokens) *out_n_tokens = n_ids; + return rc; +} + +int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !ws || !text || text[0] == '\0') return NULL; + + size_t len = strlen(text); + size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; + if (cap_sz > (size_t)INT_MAX) return NULL; + int cap = (int)cap_sz; + + if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) + return NULL; + + int n_ids = 0; + int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, + &n_ids); + if (rc == -2) { + if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) + return NULL; + rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, + &n_ids); + } + if (rc != 0 || n_ids <= 0) return NULL; + + int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); + if (!ids) return NULL; + memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); + if (out_n_tokens) *out_n_tokens = n_ids; + return ids; +} + +void qwen_tokenizer_free(qwen_tokenizer_t *tok) { + if (!tok) return; + + if (tok->id_to_text) { + for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]); + free(tok->id_to_text); + } + if (tok->id_to_bpe) { + for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]); + free(tok->id_to_bpe); + } + + if (tok->merge_map) { + str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; + for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key); + free(tok->merge_map); + } + free(tok->vocab_map); + free(tok); +} diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h new file mode 100644 index 0000000..5d80237 --- /dev/null +++ b/qwen_tokenizer.h @@ -0,0 +1,61 @@ +/* + * qwen_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * + * Supports: + * - decode token ID -> UTF-8 text + * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) + */ + +#ifndef QWEN_TOKENIZER_H +#define QWEN_TOKENIZER_H + +typedef struct { + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + int vocab_size; + + /* Internal hash maps (opaque to callers) */ + void *vocab_map; + int vocab_map_cap; + void *merge_map; + int merge_map_cap; +} qwen_tokenizer_t; + +typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; + +/* Load tokenizer from vocab.json in model directory */ +qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); + +/* Decode a single token ID to text. Returns pointer to internal string. */ +const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); + +/* Encode UTF-8 text into token IDs using BPE. + * Returns malloc'd array of token IDs and sets *out_n_tokens. + * Returns NULL on error (and sets *out_n_tokens to 0). */ +int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); + +/* Reusable scratch for allocation-light repeated encoding. */ +qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); +void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, -2 if out_cap is too small. + * In all cases *out_n_tokens is set to the number of tokens needed/emitted + * when out_n_tokens is non-NULL. */ +int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n_tokens); + +/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and + * performs only the final returned int[] allocation. */ +int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens); + +/* Free tokenizer */ +void qwen_tokenizer_free(qwen_tokenizer_t *tok); + +#endif /* QWEN_TOKENIZER_H */ diff --git a/server_main.c b/server_main.c index b4c7a34..ace0100 100644 --- a/server_main.c +++ b/server_main.c @@ -2,7 +2,7 @@ #include "pplx_embed.h" #include "pplx_server.h" -#include "qwen_asr_kernels.h" +#include "qwen_kernels.h" #ifdef USE_CUDA #include "pplx_embed_cuda.h" diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 113c731..72a205a 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -2,7 +2,7 @@ * Runs via `make test` (generic and BLAS variants) and * scripts/check_kernel_golden.py. No model files required. */ -#include "qwen_asr_kernels.h" +#include "qwen_kernels.h" #include #include -- cgit v1.3.1