From a960ebb8dec645ab619a68d7412d4392e99f4b7a Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 08:48:04 +0200 Subject: tests,bench: move generated C checkers into standing files --- Makefile | 7 +- bench/bench_tokenizer.c | 140 +++++++++++++++++++++++++++++++++ tests/test_bf16_model.c | 70 +++++++++++++++++ tests/test_safetensors.c | 36 +++++++++ tests/test_workspace.c | 196 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 bench/bench_tokenizer.c create mode 100644 tests/test_bf16_model.c create mode 100644 tests/test_safetensors.c create mode 100644 tests/test_workspace.c diff --git a/Makefile b/Makefile index 2e6d54d..8752fe6 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ SHARED_LIB = libpplxembed.so SHARED_FLAGS = -shared endif -.PHONY: all blas mlx cuda shared test debug clean help +.PHONY: all blas mlx cuda shared test bench-tokenizer debug clean help all: help @@ -165,6 +165,11 @@ test: tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) ./tests/test_kernels_blas +bench-tokenizer: + $(CC) -Wall -Wextra -O3 -march=native -I. -o bench/bench_tokenizer \ + bench/bench_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) -lm -lpthread + @echo "run: bench/bench_tokenizer [runs]" + # ============================================================================= # Debug build # ============================================================================= diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c new file mode 100644 index 0000000..425a5fc --- /dev/null +++ b/bench/bench_tokenizer.c @@ -0,0 +1,140 @@ +/* bench/bench_tokenizer.c - tokenizer encode-path benchmark. + * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */ + +#include "qwen_tokenizer.h" + +#include +#include +#include +#include + +int qwen_verbose = 0; + +static double now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; +} + +static int same_ids(const int *a, int na, const int *b, int nb) +{ + if (na != nb) return 0; + for (int i = 0; i < na; i++) { + if (a[i] != b[i]) return 0; + } + return 1; +} + +int main(int argc, char **argv) +{ + if (argc < 4) { + fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]); + return 2; + } + + const char *vocab = argv[1]; + int runs = atoi(argv[2]); + if (runs <= 0) return 2; + int n_texts = argc - 3; + char **texts = argv + 3; + + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab); + qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + if (!tok || !ws) { + fprintf(stderr, "failed to initialize tokenizer\n"); + return 1; + } + + int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold)); + int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); + int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); + int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); + if (!gold || !gold_n || !bufs || !caps) return 1; + + int tokens_per_pass = 0; + for (int i = 0; i < n_texts; i++) { + gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); + if (!gold[i] || gold_n[i] <= 0) return 1; + tokens_per_pass += gold_n[i]; + + int n_ws = 0; + int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], + &n_ws); + if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) + return 1; + free(ids_ws); + + caps[i] = (int)(strlen(texts[i]) * 2 + 8); + if (caps[i] < gold_n[i]) caps[i] = gold_n[i]; + bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int)); + if (!bufs[i]) return 1; + int n_into = 0; + if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], + &n_into) != 0 || + !same_ids(gold[i], gold_n[i], bufs[i], n_into)) + return 1; + } + + double t0 = now_ms(); + long long old_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int *ids = qwen_tokenizer_encode(tok, texts[i], &n); + if (!ids || n != gold_n[i]) return 1; + old_tokens += n; + free(ids); + } + } + double old_ms = now_ms() - t0; + + t0 = now_ms(); + long long ws_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], + &n); + if (!ids || n != gold_n[i]) return 1; + ws_tokens += n; + free(ids); + } + } + double ws_ms = now_ms() - t0; + + t0 = now_ms(); + long long into_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], + caps[i], &n); + if (rc != 0 || n != gold_n[i]) return 1; + into_tokens += n; + } + } + double into_ms = now_ms() - t0; + + printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, + tokens_per_pass); + printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms, + (double)old_tokens / (old_ms / 1000.0)); + printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", + ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); + printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", + into_ms, (double)into_tokens / (into_ms / 1000.0), + old_ms / into_ms); + + for (int i = 0; i < n_texts; i++) { + free(gold[i]); + free(bufs[i]); + } + free(gold); + free(gold_n); + free(bufs); + free(caps); + qwen_tokenizer_workspace_free(ws); + qwen_tokenizer_free(tok); + return 0; +} diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c new file mode 100644 index 0000000..0f3101a --- /dev/null +++ b/tests/test_bf16_model.c @@ -0,0 +1,70 @@ +/* tests/test_bf16_model.c - F32 vs BF16 loader/dispatch parity on a tiny + * synthetic model. Runs via `make test`. */ + +#include "embed.h" + +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 3) { + fprintf(stderr, "usage: %s f32_model bf16_model\n", argv[0]); + return 2; + } + + int ids[] = {1, 2, 3, 4}; + pplx_model_t *mf32 = pplx_model_load(argv[1]); + pplx_model_t *mbf16 = pplx_model_load(argv[2]); + if (!mf32 || !mbf16) { + fprintf(stderr, "model load failed\n"); + return 1; + } + + const pplx_config_t *cfg = pplx_model_config(mf32); + int dim = cfg->hidden_size; + float *a = (float *)malloc((size_t)dim * sizeof(float)); + float *b = (float *)malloc((size_t)dim * sizeof(float)); + pplx_workspace_t *wf32 = pplx_workspace_new(mf32); + pplx_workspace_t *wbf16 = pplx_workspace_new(mbf16); + if (!a || !b || !wf32 || !wbf16) { + fprintf(stderr, "allocation failed\n"); + return 1; + } + + if (pplx_model_embed_into(mf32, wf32, ids, 4, a) != 0 || + pplx_model_embed_into(mbf16, wbf16, ids, 4, b) != 0) { + fprintf(stderr, "embedding failed\n"); + return 1; + } + + float max_diff = 0.0f; + float norm = 0.0f; + for (int i = 0; i < dim; i++) { + if (!isfinite(a[i]) || !isfinite(b[i])) { + fprintf(stderr, "non-finite embedding value\n"); + return 1; + } + float d = fabsf(a[i] - b[i]); + if (d > max_diff) max_diff = d; + norm += b[i] * b[i]; + } + norm = sqrtf(norm); + /* Embeddings are intentionally unnormalized; require a sane nonzero + * magnitude and tight F32-vs-BF16 parity, not unit norm. */ + if (norm <= 1e-6f || max_diff > 2e-5f) { + fprintf(stderr, "bad parity: norm=%.9g max_diff=%.9g\n", norm, max_diff); + return 1; + } + + printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n", + dim, max_diff, norm); + + pplx_workspace_free(wf32); + pplx_workspace_free(wbf16); + pplx_model_free(mf32); + pplx_model_free(mbf16); + free(a); + free(b); + return 0; +} diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c new file mode 100644 index 0000000..42d8f21 --- /dev/null +++ b/tests/test_safetensors.c @@ -0,0 +1,36 @@ +/* tests/test_safetensors.c - safetensors parser limit and rejection tests. + * Runs via `make test`. Fixtures are synthesized in a temp directory. */ + +#include "qwen_safetensors.h" + +#include +#include + +int main(int argc, char **argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s file|multi path ok|fail\n", argv[0]); + return 2; + } + + int want_ok = strcmp(argv[3], "ok") == 0; + int got_ok = 0; + if (strcmp(argv[1], "file") == 0) { + safetensors_file_t *sf = safetensors_open(argv[2]); + got_ok = sf != NULL; + safetensors_close(sf); + } else if (strcmp(argv[1], "multi") == 0) { + multi_safetensors_t *ms = multi_safetensors_open(argv[2]); + got_ok = ms != NULL; + multi_safetensors_close(ms); + } else { + return 2; + } + + if (got_ok != want_ok) { + fprintf(stderr, "%s %s: got %s want %s\n", + argv[1], argv[2], got_ok ? "ok" : "fail", + want_ok ? "ok" : "fail"); + return 1; + } + return 0; +} diff --git a/tests/test_workspace.c b/tests/test_workspace.c new file mode 100644 index 0000000..a87e9f8 --- /dev/null +++ b/tests/test_workspace.c @@ -0,0 +1,196 @@ +/* tests/test_workspace.c - model/workspace API lifecycle tests. + * Runs via `make test`. Uses a tiny synthetic model. */ + +#include "embed.h" +#include "qwen_tokenizer.h" + +#include +#include +#include +#include + +static float max_abs_diff(const float *a, const float *b, int n) +{ + float m = 0.0f; + for (int i = 0; i < n; i++) { + float d = fabsf(a[i] - b[i]); + if (d > m) m = d; + } + return m; +} + +int main(int argc, char **argv) +{ + if (argc != 2) { + fprintf(stderr, "usage: %s MODEL_DIR\n", argv[0]); + return 2; + } + + const char *model_dir = argv[1]; + char vocab_path[4096]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); + + pplx_model_t *model = pplx_model_load(model_dir); + if (!model) { + fprintf(stderr, "failed to load model\n"); + return 1; + } + + const pplx_config_t *cfg = pplx_model_config(model); + if (!cfg || cfg->hidden_size <= 0) { + fprintf(stderr, "invalid model config\n"); + pplx_model_free(model); + return 1; + } + + pplx_workspace_t *batch_ws = pplx_workspace_new(model); + pplx_workspace_t *single_ws = pplx_workspace_new(model); + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + if (!batch_ws || !single_ws || !tok) { + fprintf(stderr, "failed to allocate workspace or tokenizer\n"); + qwen_tokenizer_free(tok); + pplx_workspace_free(single_ws); + pplx_workspace_free(batch_ws); + pplx_model_free(model); + return 1; + } + + size_t initial_ws_bytes = pplx_workspace_nbytes(batch_ws); + if (initial_ws_bytes == 0) { + fprintf(stderr, "workspace byte accounting returned zero\n"); + goto fail; + } + + const char *texts[] = { + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey." + }; + const int batch = (int)(sizeof(texts) / sizeof(texts[0])); + int *ids[4] = {0}; + int ntok[4] = {0}; + pplx_input_t inputs[4]; + + for (int i = 0; i < batch; i++) { + ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]); + if (!ids[i] || ntok[i] <= 0) { + fprintf(stderr, "failed to tokenize input %d\n", i); + goto fail; + } + inputs[i].ids = ids[i]; + inputs[i].n_tokens = ntok[i]; + } + + int dim = cfg->hidden_size; + float *batched = (float *)calloc((size_t)batch * dim, sizeof(float)); + float *single = (float *)calloc((size_t)dim, sizeof(float)); + float *normalized = (float *)calloc((size_t)dim, sizeof(float)); + float *states = (float *)calloc((size_t)ntok[0] * dim, sizeof(float)); + if (!batched || !single || !normalized || !states) { + fprintf(stderr, "allocation failure\n"); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + if (pplx_model_embed_batch(model, batch_ws, inputs, batch, batched) != 0) { + fprintf(stderr, "pplx_model_embed_batch failed\n"); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + size_t batch_ws_bytes = pplx_workspace_nbytes(batch_ws); + if (batch_ws_bytes <= initial_ws_bytes) { + fprintf(stderr, "workspace byte accounting did not grow after embedding\n"); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + float worst_diff = 0.0f; + float worst_cos = 1.0f; + for (int i = 0; i < batch; i++) { + if (pplx_model_embed_into(model, single_ws, ids[i], ntok[i], single) != 0) { + fprintf(stderr, "pplx_model_embed_into failed at %d\n", i); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + const float *row = batched + (size_t)i * dim; + float diff = max_abs_diff(row, single, dim); + float cos = pplx_cosine_similarity(row, single, dim); + if (diff > worst_diff) worst_diff = diff; + if (cos < worst_cos) worst_cos = cos; + } + + memcpy(normalized, batched, (size_t)dim * sizeof(float)); + if (pplx_l2_normalize(normalized, dim) != 0) { + fprintf(stderr, "pplx_l2_normalize failed\n"); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + float normalized_norm_sq = 0.0f; + for (int i = 0; i < dim; i++) + normalized_norm_sq += normalized[i] * normalized[i]; + if (fabsf(normalized_norm_sq - 1.0f) > 0.00005f) { + fprintf(stderr, "pplx_l2_normalize produced norm_sq=%g\n", + normalized_norm_sq); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + if (pplx_model_forward_into(model, batch_ws, ids[0], ntok[0], states) != 0) { + fprintf(stderr, "pplx_model_forward_into failed\n"); + free(states); + free(normalized); + free(single); + free(batched); + goto fail; + } + + free(states); + free(normalized); + free(single); + free(batched); + + for (int i = 0; i < batch; i++) free(ids[i]); + qwen_tokenizer_free(tok); + pplx_workspace_free(single_ws); + pplx_workspace_free(batch_ws); + pplx_model_free(model); + + if (worst_diff > 0.00005f || worst_cos < 0.99999f) { + fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n", + worst_diff, worst_cos); + return 1; + } + + printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n", + dim, batch, batch_ws_bytes, worst_diff, worst_cos); + return 0; + +fail: + for (int i = 0; i < batch; i++) free(ids[i]); + qwen_tokenizer_free(tok); + pplx_workspace_free(single_ws); + pplx_workspace_free(batch_ws); + pplx_model_free(model); + return 1; +} -- cgit v1.3.1