diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | Makefile | 29 | ||||
| -rw-r--r-- | bench/bench.h | 216 | ||||
| -rw-r--r-- | bench/bench_kernels.c | 245 | ||||
| -rw-r--r-- | bench/bench_model.c | 97 |
5 files changed, 589 insertions, 1 deletions
@@ -12,6 +12,9 @@ tests/test_late tests/test_cli tests/cli_under_test bench/bench_tokenizer +bench/bench_kernels +bench/bench_model +bench/results/ coverage/ book/ .venv/ @@ -48,7 +48,8 @@ SHARED_LIB = libpplxembed.so SHARED_FLAGS = -shared endif -.PHONY: all cpu metal cuda test coverage bench-tokenizer debug clean help +.PHONY: all cpu metal cuda test coverage bench bench-model bench-tokenizer \ + debug clean help all: help @@ -61,6 +62,9 @@ help: @echo " make cuda Build with CUDA/cuBLAS backend (Linux/NVIDIA)" @echo " make test Build and run the C test suite (no model files needed)" @echo " make coverage Test-suite line coverage report (clang/llvm-cov)" + @echo " make bench Kernel microbenchmarks; records bench/results/*.json" + @echo " make bench-model MODEL_DIR=... End-to-end embedding benchmark" + @echo " (compare records with scripts/benchstat.py)" @echo " make debug Debug build with AddressSanitizer" @echo " make clean Remove build artifacts" @echo "" @@ -274,6 +278,29 @@ bench-tokenizer: @echo "run: bench/bench_tokenizer <model_dir> [runs]" # ============================================================================= +# Regression microbenchmarks (Go-style; see bench/bench.h). Each run records +# a JSON sample set under bench/results/, keyed by commit; compare two +# records with scripts/benchstat.py to see if a change helped or hurt. +# ============================================================================= +BENCH_STAMP = $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S) +BENCH_CC_FLAGS = -Wall -Wextra $(CFLAGS_BASE) $(TEST_BLAS_CFLAGS) -I. + +bench: + $(CC) $(BENCH_CC_FLAGS) -o bench/bench_kernels \ + bench/bench_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + @mkdir -p bench/results + ./bench/bench_kernels --json bench/results/kernels-$(BENCH_STAMP).json + @echo "compare: scripts/benchstat.py bench/results/OLD.json bench/results/NEW.json" + +bench-model: + @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; } + $(CC) $(BENCH_CC_FLAGS) -o bench/bench_model \ + bench/bench_model.c embed.c qwen_safetensors.c $(KERNEL_SRCS) \ + -lm -lpthread $(TEST_BLAS_LDFLAGS) + @mkdir -p bench/results + ./bench/bench_model "$(MODEL_DIR)" --json bench/results/model-$(BENCH_STAMP).json + +# ============================================================================= # Debug build # ============================================================================= SANITIZE ?= address,undefined diff --git a/bench/bench.h b/bench/bench.h new file mode 100644 index 0000000..e26a05d --- /dev/null +++ b/bench/bench.h @@ -0,0 +1,216 @@ +/* bench/bench.h - minimal Go-style microbenchmark harness. + * + * A benchmark function receives a bench_state_t and must run its measured + * loop exactly b->n times. Call bench_begin(b) after setup so allocation + * and fill time stay out of the measurement: + * + * static void bm_dot(bench_state_t *b) { + * float *x = make_input(); // setup, not measured + * bench_begin(b); + * for (long i = 0; i < b->n; i++) + * bench_sink += qwen_dot_f32(x, x, 1024); + * free(x); + * } + * + * The harness calibrates n until one run takes at least --benchtime-ms + * (default 300), then takes --count (default 10) timed samples and prints + * one Go-style line per benchmark with the median ns/op and the spread. + * --json PATH writes every sample plus run metadata for benchstat.py. + * --filter SUBSTR runs the matching subset. + */ + +#ifndef BENCH_H +#define BENCH_H + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +typedef struct { + long n; /* iterations: the measured loop must run exactly n times */ + double t0_ns; /* set by bench_begin(); end of un-measured setup */ +} bench_state_t; + +typedef void (*bench_fn_t)(bench_state_t *); + +typedef struct { + const char *name; + bench_fn_t fn; +} bench_case_t; + +/* Accumulate something cheap into this so results stay observable and the + * compiler cannot delete the measured loop. */ +static volatile float bench_sink; + +static double bench_now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1e9 + (double)ts.tv_nsec; +} + +static void bench_begin(bench_state_t *b) +{ + b->t0_ns = bench_now_ns(); +} + +/* One timed run of fn at n iterations; returns ns/op. */ +static double bench_run_once(bench_fn_t fn, long n) +{ + bench_state_t b = { n, 0.0 }; + b.t0_ns = bench_now_ns(); + fn(&b); + double elapsed = bench_now_ns() - b.t0_ns; + return elapsed / (double)n; +} + +static int bench_cmp_double(const void *a, const void *c) +{ + double x = *(const double *)a, y = *(const double *)c; + return (x > y) - (x < y); +} + +static double bench_median(double *v, int n) +{ + qsort(v, (size_t)n, sizeof(*v), bench_cmp_double); + return n % 2 ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +typedef struct { + long benchtime_ms; + int count; + const char *filter; + const char *json_path; + /* free-form metadata recorded into the JSON, e.g. "threads=1" */ + const char *meta; +} bench_opts_t; + +static void bench_parse_args(bench_opts_t *o, int argc, char **argv) +{ + o->benchtime_ms = 300; + o->count = 10; + o->filter = NULL; + o->json_path = NULL; + if (!o->meta) o->meta = ""; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--benchtime-ms") && i + 1 < argc) + o->benchtime_ms = atol(argv[++i]); + else if (!strcmp(argv[i], "--count") && i + 1 < argc) + o->count = atoi(argv[++i]); + else if (!strcmp(argv[i], "--filter") && i + 1 < argc) + o->filter = argv[++i]; + else if (!strcmp(argv[i], "--json") && i + 1 < argc) + o->json_path = argv[++i]; + } + if (o->benchtime_ms < 1) o->benchtime_ms = 1; + if (o->count < 1) o->count = 1; + if (o->count > 1000) o->count = 1000; +} + +/* Run all cases (subject to the filter) and report. Returns 0 on success. */ +static int bench_main(const bench_opts_t *o, + const bench_case_t *cases, int n_cases) +{ + FILE *jf = NULL; + if (o->json_path) { + jf = fopen(o->json_path, "w"); + if (!jf) { + fprintf(stderr, "cannot write %s\n", o->json_path); + return 1; + } + char host[256] = "unknown"; + gethostname(host, sizeof(host) - 1); + fprintf(jf, "{\"host\":\"%s\",\"benchtime_ms\":%ld,\"count\":%d," + "\"meta\":\"%s\",\"benchmarks\":[", + host, o->benchtime_ms, o->count, o->meta); + } + + double *samples = (double *)malloc((size_t)o->count * sizeof(double)); + if (!samples) { + if (jf) fclose(jf); + return 1; + } + + long *iters = (long *)calloc((size_t)n_cases, sizeof(long)); + double *all = (double *)calloc((size_t)n_cases * o->count, sizeof(double)); + if (!iters || !all) { + free(all); + free(iters); + free(samples); + if (jf) fclose(jf); + return 1; + } + + /* Calibrate every selected case first: grow n predictively until one + * run is long enough. The calibration run doubles as warmup. */ + int ran = 0; + double target_ns = (double)o->benchtime_ms * 1e6; + for (int c = 0; c < n_cases; c++) { + if (o->filter && !strstr(cases[c].name, o->filter)) + continue; + long n = 1; + double ns_per_op = bench_run_once(cases[c].fn, n); + while (ns_per_op * (double)n < target_ns) { + double want = target_ns / (ns_per_op > 0.0 ? ns_per_op : 1.0); + long next = (long)(want * 1.2) + 1; + if (next > n * 100) next = n * 100; /* distrust tiny timings */ + if (next <= n) next = n + 1; + n = next; + ns_per_op = bench_run_once(cases[c].fn, n); + } + iters[c] = n; + ran++; + } + + /* Sample round-robin so slow machine drift (thermals, scheduling) + * lands inside every benchmark's sample spread instead of shifting + * whole benchmarks against each other. */ + for (int s = 0; s < o->count; s++) + for (int c = 0; c < n_cases; c++) + if (iters[c]) + all[(size_t)c * o->count + s] = + bench_run_once(cases[c].fn, iters[c]); + + int emitted = 0; + for (int c = 0; c < n_cases; c++) { + if (!iters[c]) + continue; + memcpy(samples, all + (size_t)c * o->count, + (size_t)o->count * sizeof(double)); + + if (jf) { + fprintf(jf, "%s{\"name\":\"%s\",\"iters\":%ld,\"ns_per_op\":[", + emitted ? "," : "", cases[c].name, iters[c]); + for (int s = 0; s < o->count; s++) + fprintf(jf, "%s%.3f", s ? "," : "", samples[s]); + fprintf(jf, "]}"); + } + + double med = bench_median(samples, o->count); + double spread = o->count > 1 + ? 100.0 * (samples[o->count - 1] - samples[0]) / med : 0.0; + printf("%-44s %10ld iters %14.1f ns/op (spread %.1f%%)\n", + cases[c].name, iters[c], med, spread); + fflush(stdout); + emitted++; + } + + free(all); + free(iters); + free(samples); + if (jf) { + fprintf(jf, "]}\n"); + fclose(jf); + fprintf(stderr, "wrote %s\n", o->json_path); + } + if (!ran) { + fprintf(stderr, "no benchmarks matched\n"); + return 1; + } + return 0; +} + +#endif /* BENCH_H */ diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c new file mode 100644 index 0000000..67f770c --- /dev/null +++ b/bench/bench_kernels.c @@ -0,0 +1,245 @@ +/* bench/bench_kernels.c - regression microbenchmarks for the hot kernels. + * + * Shapes are the real pplx-embed-v1-0.6b dimensions (hidden 1024, 16 query / + * 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts + * a slowdown in inference. Single-threaded by default for stable numbers; + * --threads N opts into the pool. Build and record via `make bench`, + * compare records with scripts/benchstat.py. + */ + +#include "bench.h" +#include "qwen_kernels.h" + +#include <stdint.h> + +int qwen_verbose = 0; /* defined by embed.c in full builds */ + +enum { + HIDDEN = 1024, + Q_DIM = 2048, /* 16 heads * 128 */ + KV_DIM = 1024, /* 8 heads * 128 */ + HEAD_DIM = 128, + INTER = 3072, + SEQ = 8, + ATTN_SEQ = 128, +}; + +/* Deterministic fill in [-1, 1]; small values keep softmax/silu sane. */ +static void fill_f32(float *x, size_t n, unsigned seed) +{ + unsigned s = seed * 2654435761u + 1u; + for (size_t i = 0; i < n; i++) { + s = s * 1664525u + 1013904223u; + x[i] = (float)((s >> 8) & 0xFFFF) / 32768.0f - 1.0f; + } +} + +static uint16_t f32_to_bf16(float x) +{ + uint32_t u; + memcpy(&u, &x, sizeof(u)); + return (uint16_t)(u >> 16); +} + +static void fill_bf16(uint16_t *x, size_t n, unsigned seed) +{ + unsigned s = seed * 2654435761u + 1u; + for (size_t i = 0; i < n; i++) { + s = s * 1664525u + 1013904223u; + x[i] = f32_to_bf16((float)((s >> 8) & 0xFFFF) / 32768.0f - 1.0f); + } +} + +static float *alloc_f32(size_t n, unsigned seed) +{ + float *x = (float *)malloc(n * sizeof(float)); + if (!x) exit(2); + fill_f32(x, n, seed); + return x; +} + +static uint16_t *alloc_bf16(size_t n, unsigned seed) +{ + uint16_t *x = (uint16_t *)malloc(n * sizeof(uint16_t)); + if (!x) exit(2); + fill_bf16(x, n, seed); + return x; +} + +/* ---- F32 projections (matvec at seq 1, GEMM at seq 8) ---- */ + +static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) +{ + float *x = alloc_f32((size_t)seq * in, 1); + float *w = alloc_f32((size_t)out * in, 2); + float *y = alloc_f32((size_t)seq * out, 3); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_linear_nobias(y, x, w, seq, in, out); + bench_sink += y[0]; + free(y); free(w); free(x); +} + +static void bm_linear_f32_s1_qproj(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, Q_DIM); } +static void bm_linear_f32_s1_gate(bench_state_t *b) { bm_linear_f32(b, 1, HIDDEN, INTER); } +static void bm_linear_f32_s1_down(bench_state_t *b) { bm_linear_f32(b, 1, INTER, HIDDEN); } +static void bm_linear_f32_s8_gate(bench_state_t *b) { bm_linear_f32(b, SEQ, HIDDEN, INTER); } + +/* ---- BF16 weights ---- */ + +static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) +{ + float *x = alloc_f32((size_t)seq * in, 4); + uint16_t *w = alloc_bf16((size_t)out * in, 5); + float *y = alloc_f32((size_t)seq * out, 6); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_linear_nobias_bf16(y, x, w, seq, in, out); + bench_sink += y[0]; + free(y); free(w); free(x); +} + +static void bm_linear_bf16_s1_gate(bench_state_t *b) { bm_linear_bf16(b, 1, HIDDEN, INTER); } +static void bm_linear_bf16_s8_gate(bench_state_t *b) { bm_linear_bf16(b, SEQ, HIDDEN, INTER); } + +static void bm_qkv_bf16(bench_state_t *b) +{ + float *x = alloc_f32((size_t)SEQ * HIDDEN, 7); + uint16_t *wq = alloc_bf16((size_t)Q_DIM * HIDDEN, 8); + uint16_t *wk = alloc_bf16((size_t)KV_DIM * HIDDEN, 9); + uint16_t *wv = alloc_bf16((size_t)KV_DIM * HIDDEN, 10); + float *q = alloc_f32((size_t)SEQ * Q_DIM, 11); + float *k = alloc_f32((size_t)SEQ * KV_DIM, 12); + float *v = alloc_f32((size_t)SEQ * KV_DIM, 13); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, + SEQ, HIDDEN, Q_DIM, KV_DIM); + bench_sink += q[0] + k[0] + v[0]; + free(v); free(k); free(q); free(wv); free(wk); free(wq); free(x); +} + +static void bm_bf16_widen(bench_state_t *b) +{ + size_t n = (size_t)INTER * HIDDEN; + uint16_t *src = alloc_bf16(n, 14); + float *dst = alloc_f32(n, 15); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_bf16_to_f32_buf(dst, src, n); + bench_sink += dst[0]; + free(dst); free(src); +} + +/* ---- attention pieces ---- */ + +static void bm_attn_scores(bench_state_t *b) +{ + /* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */ + float *qm = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 16); + float *km = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 17); + float *s = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 18); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); + bench_sink += s[0]; + free(s); free(km); free(qm); +} + +static void bm_softmax(bench_state_t *b) +{ + float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_softmax(x, ATTN_SEQ, ATTN_SEQ); + bench_sink += x[0]; + free(x); +} + +static void bm_rope(bench_state_t *b) +{ + float *x = alloc_f32((size_t)ATTN_SEQ * Q_DIM, 20); + float *cosv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 21); + float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, + HEAD_DIM); + bench_sink += x[0]; + free(sinv); free(cosv); free(x); +} + +/* ---- norms, activations, reductions ---- */ + +static void bm_rms_norm(bench_state_t *b) +{ + float *x = alloc_f32((size_t)SEQ * HIDDEN, 23); + float *w = alloc_f32(HIDDEN, 24); + float *y = alloc_f32((size_t)SEQ * HIDDEN, 25); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); + bench_sink += y[0]; + free(y); free(w); free(x); +} + +static void bm_swiglu(bench_state_t *b) +{ + float *gate_up = alloc_f32((size_t)SEQ * 2 * INTER, 26); + float *out = alloc_f32((size_t)SEQ * INTER, 27); + bench_begin(b); + for (long i = 0; i < b->n; i++) + qwen_swiglu_multiply(out, gate_up, SEQ, INTER); + bench_sink += out[0]; + free(out); free(gate_up); +} + +static void bm_dot(bench_state_t *b) +{ + float *x = alloc_f32(HIDDEN, 28); + float *y = alloc_f32(HIDDEN, 29); + bench_begin(b); + for (long i = 0; i < b->n; i++) + bench_sink += qwen_dot_f32(x, y, HIDDEN); + free(y); free(x); +} + +static const bench_case_t CASES[] = { + {"linear_f32/s1_1024x2048_qproj", bm_linear_f32_s1_qproj}, + {"linear_f32/s1_1024x3072_gate", bm_linear_f32_s1_gate}, + {"linear_f32/s1_3072x1024_down", bm_linear_f32_s1_down}, + {"linear_f32/s8_1024x3072_gate", bm_linear_f32_s8_gate}, + {"linear_bf16/s1_1024x3072_gate", bm_linear_bf16_s1_gate}, + {"linear_bf16/s8_1024x3072_gate", bm_linear_bf16_s8_gate}, + {"linear_bf16/s8_qkv", bm_qkv_bf16}, + {"bf16_to_f32/3145728", bm_bf16_widen}, + {"attn/scores_one_head_seq128", bm_attn_scores}, + {"attn/softmax_128x128", bm_softmax}, + {"attn/rope_seq128_16h", bm_rope}, + {"norm/rms_s8_1024", bm_rms_norm}, + {"act/swiglu_s8_3072", bm_swiglu}, + {"reduce/dot_1024", bm_dot}, +}; + +int main(int argc, char **argv) +{ + int threads = 1; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--threads") && i + 1 < argc) + threads = atoi(argv[i + 1]); + } + if (threads < 1) threads = 1; + qwen_set_threads(threads); + + char meta[128]; +#ifdef USE_BLAS + snprintf(meta, sizeof(meta), "suite=kernels threads=%d blas=1", threads); +#else + snprintf(meta, sizeof(meta), "suite=kernels threads=%d blas=0", threads); +#endif + + bench_opts_t opts = {0}; + opts.meta = meta; + bench_parse_args(&opts, argc, argv); + return bench_main(&opts, CASES, (int)(sizeof(CASES) / sizeof(CASES[0]))); +} diff --git a/bench/bench_model.c b/bench/bench_model.c new file mode 100644 index 0000000..edf0d0a --- /dev/null +++ b/bench/bench_model.c @@ -0,0 +1,97 @@ +/* bench/bench_model.c - end-to-end embedding throughput against real + * weights, for regression tracking (kernels are covered by bench_kernels.c; + * this catches slowdowns in the orchestration between them). + * + * make bench-model MODEL_DIR=/path/to/pplx-embed-v1-0.6b + * + * Uses deterministic synthetic token ids, so only config.json and + * model.safetensors are needed (no tokenizer). Single-threaded by default; + * --threads N opts into the pool. + */ + +#include "bench.h" +#include "embed.h" +#include "qwen_kernels.h" + +static pplx_model_t *g_model; +static pplx_workspace_t *g_ws; +static int g_hidden; + +enum { MAX_BATCH = 8 }; + +static void bm_embed(bench_state_t *b, int batch, int len) +{ + int *ids = (int *)malloc((size_t)batch * len * sizeof(int)); + float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float)); + if (!ids || !out) exit(2); + unsigned s = 99u; + for (int i = 0; i < batch * len; i++) { + s = s * 1664525u + 1013904223u; + ids[i] = (int)(s % 1000u); /* ordinary vocab rows */ + } + pplx_input_t inputs[MAX_BATCH]; + for (int i = 0; i < batch; i++) { + inputs[i].ids = ids + (size_t)i * len; + inputs[i].n_tokens = len; + } + bench_begin(b); + for (long i = 0; i < b->n; i++) { + if (pplx_model_embed_batch(g_model, g_ws, inputs, batch, out) != 0) + exit(2); + bench_sink += out[0]; + } + free(out); + free(ids); +} + +static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); } +static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); } +static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); } + +static const bench_case_t CASES[] = { + {"embed/b1_len32", bm_embed_b1_len32}, + {"embed/b8_len32", bm_embed_b8_len32}, + {"embed/b1_len256", bm_embed_b1_len256}, +}; + +int main(int argc, char **argv) +{ + if (argc < 2 || argv[1][0] == '-') { + fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n", + argv[0]); + return 2; + } + + int threads = 1; + for (int i = 2; i < argc; i++) { + if (!strcmp(argv[i], "--threads") && i + 1 < argc) + threads = atoi(argv[i + 1]); + } + if (threads < 1) threads = 1; + qwen_set_threads(threads); + + g_model = pplx_model_load(argv[1]); + if (!g_model) { + fprintf(stderr, "failed to load model from %s\n", argv[1]); + return 1; + } + g_ws = pplx_workspace_new(g_model); + if (!g_ws) { + pplx_model_free(g_model); + return 1; + } + g_hidden = pplx_model_config(g_model)->hidden_size; + + char meta[160]; + snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", + threads, g_hidden); + + bench_opts_t opts = {0}; + opts.meta = meta; + bench_parse_args(&opts, argc - 1, argv + 1); + int rc = bench_main(&opts, CASES, (int)(sizeof(CASES) / sizeof(CASES[0]))); + + pplx_workspace_free(g_ws); + pplx_model_free(g_model); + return rc; +} |
