diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 15:43:25 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | e8259865f41ceb50dd28b267186d11c62a465135 (patch) | |
| tree | cd5c8a4dff6936c5a9d511f2eea484393b698152 | |
| parent | 5f73e5dc901f47cda6b2a26eea31cf9fc6178bc7 (diff) | |
| download | ffwd-e8259865f41ceb50dd28b267186d11c62a465135.tar.gz ffwd-e8259865f41ceb50dd28b267186d11c62a465135.zip | |
kernels: delete dead audio/causal-LM leftovers and unused exports
| -rw-r--r-- | qwen_kernels.c | 499 | ||||
| -rw-r--r-- | qwen_kernels.h | 67 | ||||
| -rw-r--r-- | qwen_kernels_avx.c | 126 | ||||
| -rw-r--r-- | qwen_kernels_generic.c | 25 | ||||
| -rw-r--r-- | qwen_kernels_impl.h | 12 | ||||
| -rw-r--r-- | qwen_kernels_neon.c | 99 | ||||
| -rw-r--r-- | qwen_safetensors.c | 25 | ||||
| -rw-r--r-- | qwen_safetensors.h | 6 | ||||
| -rw-r--r-- | tests/test_kernels.c | 10 |
9 files changed, 4 insertions, 865 deletions
diff --git a/qwen_kernels.c b/qwen_kernels.c index bfa0691..85871d4 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -186,18 +186,6 @@ void qwen_add_inplace(float *a, const float *b, int n) { for (int i = 0; i < n; i++) a[i] += b[i]; } -void qwen_mul_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) a[i] *= b[i]; -} - -void qwen_scale(float *x, float s, int n) { - for (int i = 0; i < n; i++) x[i] *= s; -} - -void qwen_copy(float *dst, const float *src, int n) { - memcpy(dst, src, n * sizeof(float)); -} - /* ======================================================================== * Matrix Operations * ======================================================================== */ @@ -585,259 +573,13 @@ void qwen_linear_nobias_bf16(float *y, const float *x, const uint16_t *W_bf16, bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } -/* Find argmax over a range of output rows [start, end). - * Uses 2-row processing to amortize x vector loads (same as bf16_matvec_fused). */ -static void argmax_bf16_range(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - qwen_argmax_bf16_range_impl(x, W_bf16, in_dim, start, end, best_out, best_val_out); -} - -typedef struct { - const float *x; - const uint16_t *W_bf16; - int in_dim; - int out_dim; - int best_idx[QWEN_MAX_THREADS]; - float best_val[QWEN_MAX_THREADS]; -} argmax_task_t; - -static void argmax_worker(int tid, int n_threads, void *arg) { - argmax_task_t *t = (argmax_task_t *)arg; - int chunk = (t->out_dim + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->out_dim) end = t->out_dim; - if (start >= end) { - t->best_val[tid] = -1e30f; - t->best_idx[tid] = 0; - return; - } - argmax_bf16_range(t->x, t->W_bf16, t->in_dim, start, end, - &t->best_idx[tid], &t->best_val[tid]); -} - -int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, - int in_dim, int out_dim) { - if (tp.n_threads <= 1) { - int best; - float best_val; - argmax_bf16_range(x, W_bf16, in_dim, 0, out_dim, &best, &best_val); - return best; - } - - argmax_task_t task; - task.x = x; - task.W_bf16 = W_bf16; - task.in_dim = in_dim; - task.out_dim = out_dim; - parallel_for(argmax_worker, &task); - - int best = task.best_idx[0]; - float best_val = task.best_val[0]; - for (int i = 1; i < tp.n_threads; i++) { - if (task.best_val[i] > best_val) { - best_val = task.best_val[i]; - best = task.best_idx[i]; - } - } - return best; -} - - -/* ======================================================================== - * 2D Convolution (im2col + BLAS sgemm) - * ======================================================================== */ - -/* - * im2col: Unroll input patches into a column matrix for GEMM-based conv2d. - * Input: [C_in, H_in, W_in] - * Output columns: [C_in * kH * kW, H_out * W_out] - */ -static void im2col(const float *in, float *cols, - int c_in, int h_in, int w_in, - int kh, int kw, int stride, int padding, - int h_out, int w_out) { - int col_len = h_out * w_out; - for (int ic = 0; ic < c_in; ic++) { - for (int ki = 0; ki < kh; ki++) { - for (int kj = 0; kj < kw; kj++) { - int col_row = (ic * kh + ki) * kw + kj; - float *col_ptr = cols + (size_t)col_row * col_len; - for (int oh = 0; oh < h_out; oh++) { - int ih = oh * stride - padding + ki; - for (int ow = 0; ow < w_out; ow++) { - int iw = ow * stride - padding + kj; - if (ih >= 0 && ih < h_in && iw >= 0 && iw < w_in) { - col_ptr[oh * w_out + ow] = in[ic * h_in * w_in + ih * w_in + iw]; - } else { - col_ptr[oh * w_out + ow] = 0.0f; - } - } - } - } - } - } -} - -void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, - int c_in, int c_out, int h_in, int w_in, - int kh, int kw, int stride, int padding) { - int h_out = (h_in + 2 * padding - kh) / stride + 1; - int w_out = (w_in + 2 * padding - kw) / stride + 1; - int patch_size = c_in * kh * kw; - int spatial_out = h_out * w_out; - - /* im2col: input -> column matrix [patch_size, spatial_out] */ - float *cols = (float *)malloc((size_t)patch_size * spatial_out * sizeof(float)); - im2col(in, cols, c_in, h_in, w_in, kh, kw, stride, padding, h_out, w_out); - - /* GEMM: weight[c_out, patch_size] @ cols[patch_size, spatial_out] = out[c_out, spatial_out] */ -#ifdef USE_BLAS - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - c_out, spatial_out, patch_size, - 1.0f, weight, patch_size, cols, spatial_out, - 0.0f, out, spatial_out); -#else - for (int oc = 0; oc < c_out; oc++) { - for (int s = 0; s < spatial_out; s++) { - float sum = 0.0f; - for (int p = 0; p < patch_size; p++) { - sum += weight[oc * patch_size + p] * cols[p * spatial_out + s]; - } - out[oc * spatial_out + s] = sum; - } - } -#endif - - free(cols); - - /* Add bias */ - if (bias) { - for (int oc = 0; oc < c_out; oc++) { - float b = bias[oc]; - float *row = out + oc * spatial_out; - for (int s = 0; s < spatial_out; s++) { - row[s] += b; - } - } - } -} /* ======================================================================== * Normalization * ======================================================================== */ -void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, - int seq_len, int hidden, float eps) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * hidden; - float *out_row = out + s * hidden; - - /* Compute mean */ -#if defined(__AVX512F__) - __m512 sumv = _mm512_setzero_ps(); - int i = 0; - for (; i + 16 <= hidden; i += 16) { - sumv = _mm512_add_ps(sumv, _mm512_loadu_ps(x_row + i)); - } - float mean = _mm512_reduce_add_ps(sumv); - for (; i < hidden; i++) mean += x_row[i]; -#elif defined(__AVX2__) - __m256 sumv = _mm256_setzero_ps(); - int i = 0; - for (; i + 8 <= hidden; i += 8) { - sumv = _mm256_add_ps(sumv, _mm256_loadu_ps(x_row + i)); - } - __m128 sum128 = _mm_add_ps(_mm256_castps256_ps128(sumv), _mm256_extractf128_ps(sumv, 1)); - sum128 = _mm_hadd_ps(sum128, sum128); - sum128 = _mm_hadd_ps(sum128, sum128); - float mean = _mm_cvtss_f32(sum128); - for (; i < hidden; i++) mean += x_row[i]; -#else - float mean = 0.0f; - for (int i = 0; i < hidden; i++) mean += x_row[i]; -#endif - mean /= hidden; - - /* Compute variance */ -#if defined(__AVX512F__) && defined(__FMA__) - __m512 meanv = _mm512_set1_ps(mean); - __m512 accv = _mm512_setzero_ps(); - int j = 0; - for (; j + 16 <= hidden; j += 16) { - __m512 v = _mm512_sub_ps(_mm512_loadu_ps(x_row + j), meanv); - accv = _mm512_fmadd_ps(v, v, accv); - } - float var = _mm512_reduce_add_ps(accv); - for (; j < hidden; j++) { - float d = x_row[j] - mean; - var += d * d; - } -#elif defined(__AVX2__) && defined(__FMA__) - __m256 meanv = _mm256_set1_ps(mean); - __m256 accv = _mm256_setzero_ps(); - int j = 0; - for (; j + 8 <= hidden; j += 8) { - __m256 v = _mm256_sub_ps(_mm256_loadu_ps(x_row + j), meanv); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float var = _mm_cvtss_f32(acc128); - for (; j < hidden; j++) { - float d = x_row[j] - mean; - var += d * d; - } -#else - float var = 0.0f; - for (int i = 0; i < hidden; i++) { - float d = x_row[i] - mean; - var += d * d; - } -#endif - var /= hidden; - - float inv_std = 1.0f / sqrtf(var + eps); -#if defined(__AVX512F__) && defined(__FMA__) - __m512 meanv2 = _mm512_set1_ps(mean); - __m512 invv = _mm512_set1_ps(inv_std); - int k = 0; - for (; k + 16 <= hidden; k += 16) { - __m512 vx = _mm512_sub_ps(_mm512_loadu_ps(x_row + k), meanv2); - __m512 vw = _mm512_loadu_ps(weight + k); - __m512 vb = _mm512_loadu_ps(bias + k); - __m512 v = _mm512_mul_ps(_mm512_mul_ps(vx, invv), vw); - v = _mm512_add_ps(v, vb); - _mm512_storeu_ps(out_row + k, v); - } - for (; k < hidden; k++) { - out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; - } -#elif defined(__AVX2__) && defined(__FMA__) - __m256 meanv2 = _mm256_set1_ps(mean); - __m256 invv = _mm256_set1_ps(inv_std); - int k = 0; - for (; k + 8 <= hidden; k += 8) { - __m256 vx = _mm256_sub_ps(_mm256_loadu_ps(x_row + k), meanv2); - __m256 vw = _mm256_loadu_ps(weight + k); - __m256 vb = _mm256_loadu_ps(bias + k); - __m256 v = _mm256_mul_ps(_mm256_mul_ps(vx, invv), vw); - v = _mm256_add_ps(v, vb); - _mm256_storeu_ps(out_row + k, v); - } - for (; k < hidden; k++) { - out_row[k] = (x_row[k] - mean) * inv_std * weight[k] + bias[k]; - } -#else - for (int i = 0; i < hidden; i++) { - out_row[i] = (x_row[i] - mean) * inv_std * weight[i] + bias[i]; - } -#endif - } -} - +/* Rows split across the pool only when the tensor is big enough to pay + * for the dispatch. */ #define QWEN_RMS_NORM_PARALLEL_ELEMS 262144 static void qwen_rms_norm_range(float *out, const float *x, @@ -1010,22 +752,6 @@ void qwen_rms_norm_per_head(float *x, const float *weight, * Activation Functions * ======================================================================== */ -void qwen_silu(float *x, int n) { - for (int i = 0; i < n; i++) { - float val = x[i]; - x[i] = val / (1.0f + expf(-val)); - } -} - -void qwen_gelu(float *x, int n) { - for (int i = 0; i < n; i++) { - float val = x[i]; - float x3 = val * val * val; - float inner = 0.7978845608028654f * (val + 0.044715f * x3); - x[i] = 0.5f * val * (1.0f + tanhf(inner)); - } -} - void qwen_silu_mul_inplace(float *gate, const float *up, int n) { #if defined(__APPLE__) && defined(USE_BLAS) enum { CHUNK = 4096 }; @@ -1052,72 +778,6 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n) { #endif } -typedef struct { - float *out; - const float *gate_up; - int seq_len; - int intermediate; -} swiglu_task_t; - -static void swiglu_worker(int tid, int n_threads, void *arg) { - swiglu_task_t *t = (swiglu_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int s0 = tid * chunk; - int s1 = s0 + chunk; - if (s1 > t->seq_len) s1 = t->seq_len; - if (s0 >= s1) return; - - int inter = t->intermediate; - int alias_inplace = (t->out == t->gate_up); - for (int s = s0; s < s1; s++) { - const float *gu = t->gate_up + (size_t)s * 2 * inter; - float *o = t->out + (size_t)s * inter; - if (!alias_inplace) { -#if defined(__APPLE__) && defined(USE_BLAS) - /* Fast path for prefill: vectorized exp(-g) using Accelerate/vForce. */ - for (int j = 0; j < inter; j++) o[j] = -gu[2 * j]; - int n = inter; - vvexpf(o, o, &n); - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - o[j] = (g / (1.0f + o[j])) * u; - } -#else - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - g = g / (1.0f + expf(-g)); /* SiLU */ - o[j] = g * u; - } -#endif - } else { - /* In-place mode (decode seq=1): keep single-pass scalar to avoid alias hazards. */ - for (int j = 0; j < inter; j++) { - float g = gu[2 * j]; - float u = gu[2 * j + 1]; - g = g / (1.0f + expf(-g)); /* SiLU */ - o[j] = g * u; - } - } - } -} - -void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate) { - swiglu_task_t task = { - .out = out, - .gate_up = gate_up, - .seq_len = seq_len, - .intermediate = intermediate - }; - - if (tp.n_threads > 1 && seq_len >= 2 && intermediate >= 256) { - parallel_for(swiglu_worker, &task); - } else { - swiglu_worker(0, 1, &task); - } -} - void qwen_softmax(float *x, int rows, int cols) { for (int r = 0; r < rows; r++) { float *row = x + r * cols; @@ -1164,146 +824,6 @@ static inline void qwen_vec_scale_add(float *dst, const float *src, float correc qwen_vec_scale_add_impl(dst, src, correction, n); } -void qwen_bidirectional_attention(float *out, const float *Q, const float *K, - const float *V, int seq __attribute__((unused)), - int n_heads, int head_dim, float scale, - const int *window_starts, int n_windows) { - int hidden = n_heads * head_dim; - - for (int h = 0; h < n_heads; h++) { - for (int w = 0; w < n_windows; w++) { - int ws = window_starts[w]; - int we = window_starts[w + 1]; - - for (int i = ws; i < we; i++) { - const float *q_row = Q + i * hidden + h * head_dim; - float *o_row = out + i * hidden + h * head_dim; - - /* Online softmax */ - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - - for (int j = ws; j < we; j++) { - const float *k_row = K + j * hidden + h * head_dim; - const float *v_row = V + j * hidden + h * head_dim; - - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } - } - } -} - -static void qwen_causal_attention_heads(float *out, const float *Q, const float *K, - const float *V, int seq_q, int seq_k, - int n_heads, int n_kv_heads, int head_dim, - float scale, int q_offset, - int head_start, int head_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - - for (int h = head_start; h < head_end; h++) { - int kv_h = h / heads_per_kv; - - for (int i = 0; i < seq_q; i++) { - const float *q_row = Q + i * q_hidden + h * head_dim; - float *o_row = out + i * q_hidden + h * head_dim; - int global_pos = q_offset + i; - int k_end = global_pos + 1; - if (k_end > seq_k) k_end = seq_k; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - - for (int j = 0; j < k_end; j++) { - const float *k_row = K + j * kv_hidden + kv_h * head_dim; - const float *v_row = V + j * kv_hidden + kv_h * head_dim; - - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } - } -} - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - int seq_q, seq_k; - int n_heads, n_kv_heads; - int head_dim; - float scale; - int q_offset; -} causal_attn_task_t; - -static void causal_attn_worker(int tid, int n_threads, void *arg) { - causal_attn_task_t *t = (causal_attn_task_t *)arg; - int chunk = (t->n_heads + n_threads - 1) / n_threads; - int h0 = tid * chunk; - int h1 = h0 + chunk; - if (h1 > t->n_heads) h1 = t->n_heads; - if (h0 >= h1) return; - - qwen_causal_attention_heads(t->out, t->Q, t->K, t->V, - t->seq_q, t->seq_k, t->n_heads, t->n_kv_heads, - t->head_dim, t->scale, t->q_offset, h0, h1); -} - -void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, - int seq_q, int seq_k, int n_heads, int n_kv_heads, - int head_dim, float scale, int q_offset) { - if (tp.n_threads > 1 && n_heads >= 2 && (seq_q >= 2 || seq_k >= 128)) { - causal_attn_task_t task = { - .out = out, .Q = Q, .K = K, .V = V, - .seq_q = seq_q, .seq_k = seq_k, - .n_heads = n_heads, .n_kv_heads = n_kv_heads, - .head_dim = head_dim, .scale = scale, .q_offset = q_offset - }; - parallel_for(causal_attn_worker, &task); - return; - } - - qwen_causal_attention_heads(out, Q, K, V, - seq_q, seq_k, n_heads, n_kv_heads, - head_dim, scale, q_offset, 0, n_heads); -} - #define QWEN_PACKED_ATTN_QUERY_TILE 32 #define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128 @@ -1499,21 +1019,6 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, * Position Embeddings * ======================================================================== */ -void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model) { - int half = d_model / 2; - float log_timescale = logf(10000.0f) / (float)(half - 1); - - for (int p = 0; p < n_pos; p++) { - float *row = pe + p * d_model; - for (int d = 0; d < half; d++) { - float inv_timescale = expf(-(float)d * log_timescale); - float angle = (float)p * inv_timescale; - row[d] = sinf(angle); /* first half: sin */ - row[half + d] = cosf(angle); /* second half: cos */ - } - } -} - void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { int half = head_dim / 2; diff --git a/qwen_kernels.h b/qwen_kernels.h index 5281361..fd23c02 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -1,8 +1,7 @@ /* - * qwen_kernels.h - Math kernels for Qwen3-ASR inference + * qwen_kernels.h - Math kernels for Qwen3-architecture inference * * Low-level math operations. All operate on float32 tensors in row-major order. - * Adapted from voxtral-realtime project. */ #ifndef QWEN_KERNELS_H @@ -16,9 +15,6 @@ * ======================================================================== */ void qwen_add_inplace(float *a, const float *b, int n); -void qwen_mul_inplace(float *a, const float *b, int n); -void qwen_scale(float *x, float s, int n); -void qwen_copy(float *dst, const float *src, int n); /* ======================================================================== * Matrix Operations @@ -60,30 +56,9 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, float qwen_dot_f32(const float *a, const float *b, int n); /* ======================================================================== - * 2D Convolution (for audio encoder conv stem) - * ======================================================================== */ - -/* - * 2D Convolution: out = conv2d(in, weight, bias) - * in: [C_in, H, W] - * weight: [C_out, C_in, kH, kW] - * bias: [C_out] (can be NULL) - * out: [C_out, H_out, W_out] - * H_out = (H + 2*padding - kH) / stride + 1 - * W_out = (W + 2*padding - kW) / stride + 1 - */ -void qwen_conv2d(float *out, const float *in, const float *weight, const float *bias, - int c_in, int c_out, int h_in, int w_in, - int kh, int kw, int stride, int padding); - -/* ======================================================================== * Normalization * ======================================================================== */ -/* LayerNorm with bias: out = (x - mean) / sqrt(var + eps) * weight + bias */ -void qwen_layer_norm(float *out, const float *x, const float *weight, const float *bias, - int seq_len, int hidden, float eps); - /* RMS Normalization: out = x / rms(x) * weight */ void qwen_rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); @@ -98,43 +73,15 @@ void qwen_rms_norm_per_head(float *x, const float *weight, * Activation Functions * ======================================================================== */ -void qwen_silu(float *x, int n); -void qwen_gelu(float *x, int n); void qwen_softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void qwen_silu_mul_inplace(float *gate, const float *up, int n); -/* out[seq,inter] = SiLU(gate_up[seq,2*inter][:,even]) * gate_up[:,odd] */ -void qwen_swiglu_multiply(float *out, const float *gate_up, int seq_len, int intermediate); /* ======================================================================== * Attention Operations * ======================================================================== */ /* - * Bidirectional windowed attention (encoder). - * Q, K, V: [seq, n_heads * head_dim] - * out: [seq, n_heads * head_dim] - * window_starts: array of window start positions - * window_starts[n_windows] = seq (sentinel) - * All heads have same dimensions (no GQA in encoder). - */ -void qwen_bidirectional_attention(float *out, const float *Q, const float *K, - const float *V, int seq, int n_heads, - int head_dim, float scale, - const int *window_starts, int n_windows); - -/* - * Causal attention with GQA (decoder). - * Q: [seq_q, n_heads * head_dim] - * K: [seq_k, n_kv_heads * head_dim] - * V: [seq_k, n_kv_heads * head_dim] - * q_offset: global position of first query (for causal mask) - */ -void qwen_causal_attention(float *out, const float *Q, const float *K, const float *V, - int seq_q, int seq_k, int n_heads, int n_kv_heads, - int head_dim, float scale, int q_offset); - -/* * Bidirectional GQA attention over a packed/ragged batch. * * Q: [total_seq, n_heads * head_dim] @@ -163,13 +110,6 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch( * ======================================================================== */ /* - * Sinusoidal position embeddings (encoder). - * pe: output [n_pos, d_model] - * First half = sin, second half = cos. - */ -void qwen_sinusoidal_pe(float *pe, int n_pos, int d_model); - -/* * NeoX-style RoPE: compute cos/sin for positions. * cos_out, sin_out: [seq, head_dim] * cos[d] and cos[half+d] are the same (duplicated for full head_dim). @@ -185,11 +125,6 @@ void qwen_compute_rope_neox(float *cos_out, float *sin_out, const int *positions void qwen_apply_rope_neox(float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim); -/* Streaming argmax: finds argmax(W_bf16 @ x) without materializing full logits. - * Returns the index of the row with highest dot product. */ -int qwen_argmax_matvec_bf16(const float *x, const uint16_t *W_bf16, - int in_dim, int out_dim); - /* ======================================================================== * Threading * ======================================================================== */ diff --git a/qwen_kernels_avx.c b/qwen_kernels_avx.c index f47725f..0fc1df8 100644 --- a/qwen_kernels_avx.c +++ b/qwen_kernels_avx.c @@ -2,7 +2,7 @@ * qwen_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) * * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. - * Uses AVX-512F+BW for 16-wide bf16 matvec/argmax (dominant for seq_len==1), and + * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). * * The bf16 matvec processes 4 output rows simultaneously to reduce instruction @@ -154,100 +154,6 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 } } -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - int o = start; - - /* Process 4 rows at a time */ - for (; o + 3 < end; o += 4) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = w0 + in_dim; - const uint16_t *w2 = w1 + in_dim; - const uint16_t *w3 = w2 + in_dim; - - __m512 a0=_mm512_setzero_ps(), a1=_mm512_setzero_ps(); - __m512 a2=_mm512_setzero_ps(), a3=_mm512_setzero_ps(); - __m512 a4=_mm512_setzero_ps(), a5=_mm512_setzero_ps(); - __m512 a6=_mm512_setzero_ps(), a7=_mm512_setzero_ps(); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - __m512 xv0 = _mm512_loadu_ps(x + k); - __m512 xv1 = _mm512_loadu_ps(x + k + 16); - __m256i r0a=_mm256_loadu_si256((const __m256i*)(w0+k)); - __m256i r0b=_mm256_loadu_si256((const __m256i*)(w0+k+16)); - a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a),16)),xv0,a0); - a1=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b),16)),xv1,a1); - __m256i r1a=_mm256_loadu_si256((const __m256i*)(w1+k)); - __m256i r1b=_mm256_loadu_si256((const __m256i*)(w1+k+16)); - a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a),16)),xv0,a2); - a3=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b),16)),xv1,a3); - __m256i r2a=_mm256_loadu_si256((const __m256i*)(w2+k)); - __m256i r2b=_mm256_loadu_si256((const __m256i*)(w2+k+16)); - a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a),16)),xv0,a4); - a5=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b),16)),xv1,a5); - __m256i r3a=_mm256_loadu_si256((const __m256i*)(w3+k)); - __m256i r3b=_mm256_loadu_si256((const __m256i*)(w3+k+16)); - a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a),16)),xv0,a6); - a7=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b),16)),xv1,a7); - } - for (; k + 16 <= in_dim; k += 16) { - __m512 xv = _mm512_loadu_ps(x + k); - __m256i r0=_mm256_loadu_si256((const __m256i*)(w0+k)); - __m256i r1=_mm256_loadu_si256((const __m256i*)(w1+k)); - __m256i r2=_mm256_loadu_si256((const __m256i*)(w2+k)); - __m256i r3=_mm256_loadu_si256((const __m256i*)(w3+k)); - a0=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0),16)),xv,a0); - a2=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1),16)),xv,a2); - a4=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2),16)),xv,a4); - a6=_mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3),16)),xv,a6); - } - - float s0=_mm512_reduce_add_ps(_mm512_add_ps(a0,a1)); - float s1=_mm512_reduce_add_ps(_mm512_add_ps(a2,a3)); - float s2=_mm512_reduce_add_ps(_mm512_add_ps(a4,a5)); - float s3=_mm512_reduce_add_ps(_mm512_add_ps(a6,a7)); - - for (; k < in_dim; k++) { - float xk=x[k]; - uint32_t b0=((uint32_t)w0[k])<<16, b1=((uint32_t)w1[k])<<16; - uint32_t b2=((uint32_t)w2[k])<<16, b3=((uint32_t)w3[k])<<16; - float v0,v1,v2,v3; - memcpy(&v0,&b0,4); memcpy(&v1,&b1,4); memcpy(&v2,&b2,4); memcpy(&v3,&b3,4); - s0+=v0*xk; s1+=v1*xk; s2+=v2*xk; s3+=v3*xk; - } - - if (s0>best_val){best_val=s0;best=o;} - if (s1>best_val){best_val=s1;best=o+1;} - if (s2>best_val){best_val=s2;best=o+2;} - if (s3>best_val){best_val=s3;best=o+3;} - } - - /* Remaining 1-3 rows */ - for (; o < end; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m512 acc = _mm512_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k)); - acc = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)), - _mm512_loadu_ps(x + k), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum += wv*x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - - *best_out = best; - *best_val_out = best_val; -} - #else /* AVX2 only (or AVX-512F without BW) */ /* Helper: bf16 to f32 for 8 elements */ @@ -317,36 +223,6 @@ void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16 } } -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - - for (int o = start; o < end; o++) { - const uint16_t *w = W_bf16 + (size_t)o * in_dim; - __m256 a0=_mm256_setzero_ps(), a1=_mm256_setzero_ps(); - int k = 0; - for (; k + 16 <= in_dim; k += 16) { - __m256 wlo, whi; - bf16x16_to_f32(w + k, &wlo, &whi); - a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x+k), a0); - a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x+k+8), a1); - } - a0 = _mm256_add_ps(a0, a1); - __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1)); - r = _mm_hadd_ps(r, r); r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; k < in_dim; k++) { - uint32_t bits=((uint32_t)w[k])<<16; float wv; memcpy(&wv,&bits,4); - sum+=wv*x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - *best_out = best; - *best_val_out = best_val; -} - #endif /* AVX-512F+BW vs AVX2 for bf16 */ /* ===================================================================== diff --git a/qwen_kernels_generic.c b/qwen_kernels_generic.c index cc25d3a..fd07455 100644 --- a/qwen_kernels_generic.c +++ b/qwen_kernels_generic.c @@ -21,31 +21,6 @@ void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_ } } -void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - - for (int o = start; o < end; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = 0.0f; - for (int k = 0; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - if (sum > best_val) { - best_val = sum; - best = o; - } - } - - *best_out = best; - *best_val_out = best_val; -} - float qwen_dot_f32_generic(const float *a, const float *b, int n) { float sum = 0.0f; for (int i = 0; i < n; i++) sum += a[i] * b[i]; diff --git a/qwen_kernels_impl.h b/qwen_kernels_impl.h index 4a351a3..43e4fbe 100644 --- a/qwen_kernels_impl.h +++ b/qwen_kernels_impl.h @@ -9,9 +9,6 @@ void qwen_bf16_matvec_fused_generic(float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_generic(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); float qwen_dot_f32_generic(const float *a, const float *b, int n); void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); @@ -20,16 +17,12 @@ void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, #ifdef __ARM_NEON void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); float qwen_dot_f32_neon(const float *a, const float *b, int n); void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_neon #define qwen_dot_f32_impl qwen_dot_f32_neon #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon #define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon @@ -38,16 +31,12 @@ void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int #elif defined(__AVX2__) && defined(__FMA__) void qwen_bf16_matvec_fused_avx(float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -void qwen_argmax_bf16_range_avx(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out); float qwen_dot_f32_avx(const float *a, const float *b, int n); void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_avx #define qwen_dot_f32_impl qwen_dot_f32_avx #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx #define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx @@ -55,7 +44,6 @@ void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int #else #define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic -#define qwen_argmax_bf16_range_impl qwen_argmax_bf16_range_generic #define qwen_dot_f32_impl qwen_dot_f32_generic #define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic #define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic diff --git a/qwen_kernels_neon.c b/qwen_kernels_neon.c index 0060e88..d656f1d 100644 --- a/qwen_kernels_neon.c +++ b/qwen_kernels_neon.c @@ -115,105 +115,6 @@ void qwen_bf16_matvec_fused_neon(float *y, const float *x, const uint16_t *W_bf1 } } -void qwen_argmax_bf16_range_neon(const float *x, const uint16_t *W_bf16, - int in_dim, int start, int end, - int *best_out, float *best_val_out) { - int best = start; - float best_val = -1e30f; - int o = start; - - /* Process 2 rows at a time, 32 elements/iter, 8 accumulators per row */ - for (; o + 1 < end; o += 2) { - const uint16_t *w0 = W_bf16 + (size_t)o * in_dim; - const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim; - float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f); - float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f); - float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f); - float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f); - int k = 0; - - for (; k + 32 <= in_dim; k += 32) { - float32x4_t x0 = vld1q_f32(x + k); - float32x4_t x1 = vld1q_f32(x + k + 4); - float32x4_t x2 = vld1q_f32(x + k + 8); - float32x4_t x3 = vld1q_f32(x + k + 12); - float32x4_t x4 = vld1q_f32(x + k + 16); - float32x4_t x5 = vld1q_f32(x + k + 20); - float32x4_t x6 = vld1q_f32(x + k + 24); - float32x4_t x7 = vld1q_f32(x + k + 28); - - uint16x8_t r0a = vld1q_u16(w0 + k); - uint16x8_t r0b = vld1q_u16(w0 + k + 8); - uint16x8_t r0c = vld1q_u16(w0 + k + 16); - uint16x8_t r0d = vld1q_u16(w0 + k + 24); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3); - a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4); - a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5); - a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6); - a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7); - - uint16x8_t r1a = vld1q_u16(w1 + k); - uint16x8_t r1b = vld1q_u16(w1 + k + 8); - uint16x8_t r1c = vld1q_u16(w1 + k + 16); - uint16x8_t r1d = vld1q_u16(w1 + k + 24); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3); - b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4); - b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5); - b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6); - b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7); - } - - float s0 = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3))); - float s1 = vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3))); - - for (; k < in_dim; k++) { - uint32_t bits0 = ((uint32_t)w0[k]) << 16; - uint32_t bits1 = ((uint32_t)w1[k]) << 16; - float wv0, wv1; - memcpy(&wv0, &bits0, sizeof(float)); - memcpy(&wv1, &bits1, sizeof(float)); - s0 += wv0 * x[k]; - s1 += wv1 * x[k]; - } - - if (s0 > best_val) { best_val = s0; best = o; } - if (s1 > best_val) { best_val = s1; best = o + 1; } - } - - for (; o < end; o++) { - const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; - float sum = 0.0f; - int k = 0; - - float32x4_t acc0 = vdupq_n_f32(0.0f), acc1 = vdupq_n_f32(0.0f); - for (; k + 8 <= in_dim; k += 8) { - uint16x8_t bf = vld1q_u16(w_row + k); - acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)), - vld1q_f32(x + k)); - acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)), - vld1q_f32(x + k + 4)); - } - sum += vaddvq_f32(vaddq_f32(acc0, acc1)); - - for (; k < in_dim; k++) { - uint32_t f32_bits = ((uint32_t)w_row[k]) << 16; - float w_val; - memcpy(&w_val, &f32_bits, sizeof(float)); - sum += w_val * x[k]; - } - if (sum > best_val) { best_val = sum; best = o; } - } - - *best_out = best; - *best_val_out = best_val; -} - float qwen_dot_f32_neon(const float *a, const float *b, int n) { int i = 0; float32x4_t acc0 = vdupq_n_f32(0.0f); diff --git a/qwen_safetensors.c b/qwen_safetensors.c index 37a5d3e..030798e 100644 --- a/qwen_safetensors.c +++ b/qwen_safetensors.c @@ -290,31 +290,6 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) return out; } -int safetensor_is_bf16(const safetensor_t *t) { - return t && t->dtype == DTYPE_BF16; -} - -uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t) { - if (!sf || !t || t->dtype != DTYPE_BF16) return NULL; - return (uint16_t *)safetensors_data(sf, t); -} - -void safetensor_print(const safetensor_t *t) { - const char *dtype_names[] = {"F32", "F16", "BF16", "I32", "I64", "BOOL"}; - const char *dtype_name = t->dtype >= 0 && t->dtype <= 5 ? - dtype_names[t->dtype] : "UNKNOWN"; - printf("%s: dtype=%s, shape=[", t->name, dtype_name); - for (int i = 0; i < t->ndim; i++) { - printf("%ld%s", (long)t->shape[i], i < t->ndim - 1 ? ", " : ""); - } - printf("]\n"); -} - -void safetensors_print_all(const safetensors_file_t *sf) { - printf("File: %s (%d tensors)\n", sf->path, sf->num_tensors); - for (int i = 0; i < sf->num_tensors; i++) safetensor_print(&sf->tensors[i]); -} - /* ======================================================================== * Multi-shard operations * ======================================================================== */ diff --git a/qwen_safetensors.h b/qwen_safetensors.h index a93e720..ba61217 100644 --- a/qwen_safetensors.h +++ b/qwen_safetensors.h @@ -68,12 +68,6 @@ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t /* Get tensor data as float32 (allocates, caller must free) */ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); -/* Get direct pointer to bf16 data in mmap'd region (no copy) */ -uint16_t *safetensors_get_bf16_direct(const safetensors_file_t *sf, const safetensor_t *t); - -int safetensor_is_bf16(const safetensor_t *t); int64_t safetensor_numel(const safetensor_t *t); -void safetensor_print(const safetensor_t *t); -void safetensors_print_all(const safetensors_file_t *sf); #endif /* QWEN_SAFETENSORS_H */ diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 5afe0ed..a390411 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -453,16 +453,6 @@ static void test_generic_vs_impl(void) { for (int o = 0; o < OUT; o++) expect_close("matvec_fused nobias generic-vs-impl", yg[o], yi[o], 1e-5f); - int bg = -1, bi = -1; - float vg = 0.0f, vi = 0.0f; - qwen_argmax_bf16_range_generic(x, W, IN, 1, OUT, &bg, &vg); - qwen_argmax_bf16_range_impl(x, W, IN, 1, OUT, &bi, &vi); - if (bg != bi) { - fprintf(stderr, "argmax generic-vs-impl: got %d want %d\n", bi, bg); - failures++; - } - expect_close("argmax val generic-vs-impl", vg, vi, 1e-5f); - const int sizes[] = {1, 7, 64, NMAX}; for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; |
