diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 15:15:38 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | ae3bf36c83befeaac21900495e6494ec2a39e225 (patch) | |
| tree | 724e4952b16dd23a3829eed09743699d0b7de557 /bench/bench_model.c | |
| parent | 4fed4e0caae3f92479a6cabb5ee2cea501a8c193 (diff) | |
| download | ffwd-ae3bf36c83befeaac21900495e6494ec2a39e225.tar.gz ffwd-ae3bf36c83befeaac21900495e6494ec2a39e225.zip | |
bench: regression microbenchmarks with benchstat compare
Diffstat (limited to 'bench/bench_model.c')
| -rw-r--r-- | bench/bench_model.c | 97 |
1 files changed, 97 insertions, 0 deletions
diff --git a/bench/bench_model.c b/bench/bench_model.c new file mode 100644 index 0000000..edf0d0a --- /dev/null +++ b/bench/bench_model.c @@ -0,0 +1,97 @@ +/* bench/bench_model.c - end-to-end embedding throughput against real + * weights, for regression tracking (kernels are covered by bench_kernels.c; + * this catches slowdowns in the orchestration between them). + * + * make bench-model MODEL_DIR=/path/to/pplx-embed-v1-0.6b + * + * Uses deterministic synthetic token ids, so only config.json and + * model.safetensors are needed (no tokenizer). Single-threaded by default; + * --threads N opts into the pool. + */ + +#include "bench.h" +#include "embed.h" +#include "qwen_kernels.h" + +static pplx_model_t *g_model; +static pplx_workspace_t *g_ws; +static int g_hidden; + +enum { MAX_BATCH = 8 }; + +static void bm_embed(bench_state_t *b, int batch, int len) +{ + int *ids = (int *)malloc((size_t)batch * len * sizeof(int)); + float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float)); + if (!ids || !out) exit(2); + unsigned s = 99u; + for (int i = 0; i < batch * len; i++) { + s = s * 1664525u + 1013904223u; + ids[i] = (int)(s % 1000u); /* ordinary vocab rows */ + } + pplx_input_t inputs[MAX_BATCH]; + for (int i = 0; i < batch; i++) { + inputs[i].ids = ids + (size_t)i * len; + inputs[i].n_tokens = len; + } + bench_begin(b); + for (long i = 0; i < b->n; i++) { + if (pplx_model_embed_batch(g_model, g_ws, inputs, batch, out) != 0) + exit(2); + bench_sink += out[0]; + } + free(out); + free(ids); +} + +static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); } +static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); } +static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); } + +static const bench_case_t CASES[] = { + {"embed/b1_len32", bm_embed_b1_len32}, + {"embed/b8_len32", bm_embed_b8_len32}, + {"embed/b1_len256", bm_embed_b1_len256}, +}; + +int main(int argc, char **argv) +{ + if (argc < 2 || argv[1][0] == '-') { + fprintf(stderr, "usage: %s MODEL_DIR [--threads N] [bench flags]\n", + argv[0]); + return 2; + } + + int threads = 1; + for (int i = 2; i < argc; i++) { + if (!strcmp(argv[i], "--threads") && i + 1 < argc) + threads = atoi(argv[i + 1]); + } + if (threads < 1) threads = 1; + qwen_set_threads(threads); + + g_model = pplx_model_load(argv[1]); + if (!g_model) { + fprintf(stderr, "failed to load model from %s\n", argv[1]); + return 1; + } + g_ws = pplx_workspace_new(g_model); + if (!g_ws) { + pplx_model_free(g_model); + return 1; + } + g_hidden = pplx_model_config(g_model)->hidden_size; + + char meta[160]; + snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", + threads, g_hidden); + + bench_opts_t opts = {0}; + opts.meta = meta; + bench_parse_args(&opts, argc - 1, argv + 1); + int rc = bench_main(&opts, CASES, (int)(sizeof(CASES) / sizeof(CASES[0]))); + + pplx_workspace_free(g_ws); + pplx_model_free(g_model); + return rc; +} |
