aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bench/bench_kernels.c22
-rw-r--r--tests/test_bf16_model.c24
-rw-r--r--tests/test_kernels.c100
-rw-r--r--tests/test_safetensors.c12
4 files changed, 150 insertions, 8 deletions
diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c
index 67f770c..7266f05 100644
--- a/bench/bench_kernels.c
+++ b/bench/bench_kernels.c
@@ -183,15 +183,21 @@ static void bm_rms_norm(bench_state_t *b)
free(y); free(w); free(x);
}
-static void bm_swiglu(bench_state_t *b)
+/* The engine's SwiGLU op mutates gate in place, so each iteration restores
+ * it first; the memcpy is a constant part of the measured loop. */
+static void bm_silu_mul(bench_state_t *b)
{
- float *gate_up = alloc_f32((size_t)SEQ * 2 * INTER, 26);
- float *out = alloc_f32((size_t)SEQ * INTER, 27);
+ size_t n = (size_t)SEQ * INTER;
+ float *gate0 = alloc_f32(n, 26);
+ float *gate = alloc_f32(n, 26);
+ float *up = alloc_f32(n, 27);
bench_begin(b);
- for (long i = 0; i < b->n; i++)
- qwen_swiglu_multiply(out, gate_up, SEQ, INTER);
- bench_sink += out[0];
- free(out); free(gate_up);
+ for (long i = 0; i < b->n; i++) {
+ memcpy(gate, gate0, n * sizeof(float));
+ qwen_silu_mul_inplace(gate, up, (int)n);
+ }
+ bench_sink += gate[0];
+ free(up); free(gate); free(gate0);
}
static void bm_dot(bench_state_t *b)
@@ -217,7 +223,7 @@ static const bench_case_t CASES[] = {
{"attn/softmax_128x128", bm_softmax},
{"attn/rope_seq128_16h", bm_rope},
{"norm/rms_s8_1024", bm_rms_norm},
- {"act/swiglu_s8_3072", bm_swiglu},
+ {"act/silu_mul_s8_3072", bm_silu_mul},
{"reduce/dot_1024", bm_dot},
};
diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c
index 75d3309..392f040 100644
--- a/tests/test_bf16_model.c
+++ b/tests/test_bf16_model.c
@@ -69,6 +69,30 @@ int main(void) {
return 1;
}
+ /* Sequences longer than 16 tokens take the widen-to-F32 weight path
+ * (linear_nobias_weight); parity must hold there too. */
+ enum { LONG_N = 24 };
+ int long_ids[LONG_N];
+ for (int i = 0; i < LONG_N; i++) long_ids[i] = (i % 14) + 1;
+ if (pplx_model_embed_into(mf32, wf32, long_ids, LONG_N, a) != 0 ||
+ pplx_model_embed_into(mbf16, wbf16, long_ids, LONG_N, b) != 0) {
+ fprintf(stderr, "long-sequence embedding failed\n");
+ return 1;
+ }
+ float long_diff = 0.0f;
+ for (int i = 0; i < dim; i++) {
+ if (!isfinite(a[i]) || !isfinite(b[i])) {
+ fprintf(stderr, "non-finite long-sequence value\n");
+ return 1;
+ }
+ float d = fabsf(a[i] - b[i]);
+ if (d > long_diff) long_diff = d;
+ }
+ if (long_diff > 2e-5f) {
+ fprintf(stderr, "bad widen-path parity: max_diff=%.9g\n", long_diff);
+ return 1;
+ }
+
printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n",
dim, max_diff, norm);
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 71b50ce..5afe0ed 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -277,6 +277,103 @@ static void test_bf16_qkv(void) {
qwen_set_threads(1);
}
+/* seq==1 routes through bf16_matvec_threaded; a pool splits the output
+ * rows across matvec_worker calls. Both must match the scalar reference. */
+static void check_bf16_matvec(int n_threads) {
+ enum { in_dim = 19, out_dim = 23 };
+ float x[in_dim];
+ uint16_t w[out_dim * in_dim];
+ float got[out_dim];
+
+ for (int i = 0; i < in_dim; i++)
+ x[i] = sinf((float)i * 0.37f) - 0.2f;
+ for (int i = 0; i < out_dim * in_dim; i++)
+ w[i] = f32_to_bf16(cosf((float)i * 0.21f) + 0.05f);
+
+ qwen_set_threads(n_threads);
+ qwen_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim);
+
+ for (int o = 0; o < out_dim; o++) {
+ float want = 0.0f;
+ for (int i = 0; i < in_dim; i++)
+ want += x[i] * bf16_to_f32(w[o * in_dim + i]);
+ expect_close("bf16_matvec", got[o], want, 1e-6f);
+ }
+}
+
+static void test_bf16_matvec(void) {
+ check_bf16_matvec(1);
+ check_bf16_matvec(4);
+ qwen_set_threads(1);
+}
+
+/* Above QWEN_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to
+ * rms_norm_worker; row-wise math is identical, so results must match the
+ * single-thread run exactly. */
+static void test_rms_norm_threaded(void) {
+ enum { seq = 96, hidden = 3072 }; /* 294912 elements */
+ size_t elems = (size_t)seq * hidden;
+ float *x = (float *)malloc(elems * sizeof(float));
+ float *w = (float *)malloc((size_t)hidden * sizeof(float));
+ float *one = (float *)malloc(elems * sizeof(float));
+ float *many = (float *)malloc(elems * sizeof(float));
+ if (!x || !w || !one || !many) {
+ fprintf(stderr, "rms_norm_threaded: allocation failure\n");
+ failures++;
+ free(many); free(one); free(w); free(x);
+ return;
+ }
+ unsigned s = 7u;
+ for (size_t i = 0; i < elems; i++) {
+ s = s * 1664525u + 1013904223u;
+ x[i] = (float)((s >> 8) & 0xFFFF) / 32768.0f - 1.0f;
+ }
+ for (int i = 0; i < hidden; i++)
+ w[i] = 0.8f + 0.01f * (float)(i % 7);
+
+ qwen_set_threads(1);
+ qwen_rms_norm(one, x, w, seq, hidden, 1e-6f);
+ qwen_set_threads(4);
+ qwen_rms_norm(many, x, w, seq, hidden, 1e-6f);
+ qwen_set_threads(1);
+
+ for (size_t i = 0; i < elems; i++) {
+ if (one[i] != many[i]) {
+ fprintf(stderr, "rms_norm_threaded: mismatch at %zu: %.9g %.9g\n",
+ i, one[i], many[i]);
+ failures++;
+ break;
+ }
+ }
+ free(many); free(one); free(w); free(x);
+}
+
+/* bf16->f32 widening is an exact bit shift; check the SIMD body and the
+ * remainder lanes against the scalar definition. */
+static void test_bf16_widen_buf(void) {
+ enum { NMAXW = 67 };
+ static const int sizes[] = {1, 7, 8, 15, 64, NMAXW};
+ uint16_t src[NMAXW];
+ float got[NMAXW];
+
+ for (int i = 0; i < NMAXW; i++)
+ src[i] = f32_to_bf16(sinf((float)i * 0.13f) * 3.0f);
+
+ for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) {
+ int n = sizes[t];
+ memset(got, 0, sizeof(got));
+ qwen_bf16_to_f32_buf(got, src, (size_t)n);
+ for (int i = 0; i < n; i++) {
+ float want = bf16_to_f32(src[i]);
+ if (memcmp(&got[i], &want, sizeof(float)) != 0) {
+ fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n",
+ n, i, got[i], want);
+ failures++;
+ }
+ }
+ }
+}
+
static void check_bf16_pair(int n_threads) {
enum { seq = 4, in_dim = 5, a_dim = 3, b_dim = 7 };
float x[seq * in_dim];
@@ -404,6 +501,9 @@ int main(void) {
test_packed_gqa_attention_long();
test_bf16_linear();
test_bf16_qkv();
+ test_bf16_matvec();
+ test_rms_norm_threaded();
+ test_bf16_widen_buf();
test_bf16_pair();
test_generic_vs_impl();
if (failures != 0) return 1;
diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c
index 401aab3..fea570b 100644
--- a/tests/test_safetensors.c
+++ b/tests/test_safetensors.c
@@ -109,6 +109,18 @@ int main(void)
ONE_TENSOR_B, zeros, 4);
check("multi", path_in("valid_multi"), 1);
+ /* Payload accounting across shards: two 4-byte F32 tensors. */
+ {
+ multi_safetensors_t *ms = multi_safetensors_open(path_in("valid_multi"));
+ size_t nbytes = 0;
+ if (!ms || multi_safetensors_data_nbytes(ms, &nbytes) != 0 ||
+ nbytes != 8) {
+ fprintf(stderr, "multi data_nbytes: got %zu want 8\n", nbytes);
+ failures++;
+ }
+ multi_safetensors_close(ms);
+ }
+
mkdir(path_in("missing"), 0755);
write_st(path_in("missing/model-00001-of-00002.safetensors"),
ONE_TENSOR_X, zeros, 4);