aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile7
-rw-r--r--bench/bench_tokenizer.c140
-rw-r--r--tests/test_bf16_model.c70
-rw-r--r--tests/test_safetensors.c36
-rw-r--r--tests/test_workspace.c196
5 files changed, 448 insertions, 1 deletions
diff --git a/Makefile b/Makefile
index 2e6d54d..8752fe6 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,7 @@ SHARED_LIB = libpplxembed.so
SHARED_FLAGS = -shared
endif
-.PHONY: all blas mlx cuda shared test debug clean help
+.PHONY: all blas mlx cuda shared test bench-tokenizer debug clean help
all: help
@@ -165,6 +165,11 @@ test:
tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
./tests/test_kernels_blas
+bench-tokenizer:
+ $(CC) -Wall -Wextra -O3 -march=native -I. -o bench/bench_tokenizer \
+ bench/bench_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) -lm -lpthread
+ @echo "run: bench/bench_tokenizer <model_dir> [runs]"
+
# =============================================================================
# Debug build
# =============================================================================
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
new file mode 100644
index 0000000..425a5fc
--- /dev/null
+++ b/bench/bench_tokenizer.c
@@ -0,0 +1,140 @@
+/* bench/bench_tokenizer.c - tokenizer encode-path benchmark.
+ * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */
+
+#include "qwen_tokenizer.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+int qwen_verbose = 0;
+
+static double now_ms(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
+}
+
+static int same_ids(const int *a, int na, const int *b, int nb)
+{
+ if (na != nb) return 0;
+ for (int i = 0; i < na; i++) {
+ if (a[i] != b[i]) return 0;
+ }
+ return 1;
+}
+
+int main(int argc, char **argv)
+{
+ if (argc < 4) {
+ fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]);
+ return 2;
+ }
+
+ const char *vocab = argv[1];
+ int runs = atoi(argv[2]);
+ if (runs <= 0) return 2;
+ int n_texts = argc - 3;
+ char **texts = argv + 3;
+
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab);
+ qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new();
+ if (!tok || !ws) {
+ fprintf(stderr, "failed to initialize tokenizer\n");
+ return 1;
+ }
+
+ int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
+ int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
+ int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
+ int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
+ if (!gold || !gold_n || !bufs || !caps) return 1;
+
+ int tokens_per_pass = 0;
+ for (int i = 0; i < n_texts; i++) {
+ gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
+ if (!gold[i] || gold_n[i] <= 0) return 1;
+ tokens_per_pass += gold_n[i];
+
+ int n_ws = 0;
+ int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
+ &n_ws);
+ if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws))
+ return 1;
+ free(ids_ws);
+
+ caps[i] = (int)(strlen(texts[i]) * 2 + 8);
+ if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
+ bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
+ if (!bufs[i]) return 1;
+ int n_into = 0;
+ if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
+ &n_into) != 0 ||
+ !same_ids(gold[i], gold_n[i], bufs[i], n_into))
+ return 1;
+ }
+
+ double t0 = now_ms();
+ long long old_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int *ids = qwen_tokenizer_encode(tok, texts[i], &n);
+ if (!ids || n != gold_n[i]) return 1;
+ old_tokens += n;
+ free(ids);
+ }
+ }
+ double old_ms = now_ms() - t0;
+
+ t0 = now_ms();
+ long long ws_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
+ &n);
+ if (!ids || n != gold_n[i]) return 1;
+ ws_tokens += n;
+ free(ids);
+ }
+ }
+ double ws_ms = now_ms() - t0;
+
+ t0 = now_ms();
+ long long into_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
+ caps[i], &n);
+ if (rc != 0 || n != gold_n[i]) return 1;
+ into_tokens += n;
+ }
+ }
+ double into_ms = now_ms() - t0;
+
+ printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs,
+ tokens_per_pass);
+ printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms,
+ (double)old_tokens / (old_ms / 1000.0));
+ printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n",
+ ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
+ printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n",
+ into_ms, (double)into_tokens / (into_ms / 1000.0),
+ old_ms / into_ms);
+
+ for (int i = 0; i < n_texts; i++) {
+ free(gold[i]);
+ free(bufs[i]);
+ }
+ free(gold);
+ free(gold_n);
+ free(bufs);
+ free(caps);
+ qwen_tokenizer_workspace_free(ws);
+ qwen_tokenizer_free(tok);
+ return 0;
+}
diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c
new file mode 100644
index 0000000..0f3101a
--- /dev/null
+++ b/tests/test_bf16_model.c
@@ -0,0 +1,70 @@
+/* tests/test_bf16_model.c - F32 vs BF16 loader/dispatch parity on a tiny
+ * synthetic model. Runs via `make test`. */
+
+#include "embed.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+ if (argc != 3) {
+ fprintf(stderr, "usage: %s f32_model bf16_model\n", argv[0]);
+ return 2;
+ }
+
+ int ids[] = {1, 2, 3, 4};
+ pplx_model_t *mf32 = pplx_model_load(argv[1]);
+ pplx_model_t *mbf16 = pplx_model_load(argv[2]);
+ if (!mf32 || !mbf16) {
+ fprintf(stderr, "model load failed\n");
+ return 1;
+ }
+
+ const pplx_config_t *cfg = pplx_model_config(mf32);
+ int dim = cfg->hidden_size;
+ float *a = (float *)malloc((size_t)dim * sizeof(float));
+ float *b = (float *)malloc((size_t)dim * sizeof(float));
+ pplx_workspace_t *wf32 = pplx_workspace_new(mf32);
+ pplx_workspace_t *wbf16 = pplx_workspace_new(mbf16);
+ if (!a || !b || !wf32 || !wbf16) {
+ fprintf(stderr, "allocation failed\n");
+ return 1;
+ }
+
+ if (pplx_model_embed_into(mf32, wf32, ids, 4, a) != 0 ||
+ pplx_model_embed_into(mbf16, wbf16, ids, 4, b) != 0) {
+ fprintf(stderr, "embedding failed\n");
+ return 1;
+ }
+
+ float max_diff = 0.0f;
+ float norm = 0.0f;
+ for (int i = 0; i < dim; i++) {
+ if (!isfinite(a[i]) || !isfinite(b[i])) {
+ fprintf(stderr, "non-finite embedding value\n");
+ return 1;
+ }
+ float d = fabsf(a[i] - b[i]);
+ if (d > max_diff) max_diff = d;
+ norm += b[i] * b[i];
+ }
+ norm = sqrtf(norm);
+ /* Embeddings are intentionally unnormalized; require a sane nonzero
+ * magnitude and tight F32-vs-BF16 parity, not unit norm. */
+ if (norm <= 1e-6f || max_diff > 2e-5f) {
+ fprintf(stderr, "bad parity: norm=%.9g max_diff=%.9g\n", norm, max_diff);
+ return 1;
+ }
+
+ printf("ok: bf16 model parity dim=%d max_abs_diff=%.9g norm=%.9g\n",
+ dim, max_diff, norm);
+
+ pplx_workspace_free(wf32);
+ pplx_workspace_free(wbf16);
+ pplx_model_free(mf32);
+ pplx_model_free(mbf16);
+ free(a);
+ free(b);
+ return 0;
+}
diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c
new file mode 100644
index 0000000..42d8f21
--- /dev/null
+++ b/tests/test_safetensors.c
@@ -0,0 +1,36 @@
+/* tests/test_safetensors.c - safetensors parser limit and rejection tests.
+ * Runs via `make test`. Fixtures are synthesized in a temp directory. */
+
+#include "qwen_safetensors.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+ if (argc != 4) {
+ fprintf(stderr, "usage: %s file|multi path ok|fail\n", argv[0]);
+ return 2;
+ }
+
+ int want_ok = strcmp(argv[3], "ok") == 0;
+ int got_ok = 0;
+ if (strcmp(argv[1], "file") == 0) {
+ safetensors_file_t *sf = safetensors_open(argv[2]);
+ got_ok = sf != NULL;
+ safetensors_close(sf);
+ } else if (strcmp(argv[1], "multi") == 0) {
+ multi_safetensors_t *ms = multi_safetensors_open(argv[2]);
+ got_ok = ms != NULL;
+ multi_safetensors_close(ms);
+ } else {
+ return 2;
+ }
+
+ if (got_ok != want_ok) {
+ fprintf(stderr, "%s %s: got %s want %s\n",
+ argv[1], argv[2], got_ok ? "ok" : "fail",
+ want_ok ? "ok" : "fail");
+ return 1;
+ }
+ return 0;
+}
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
new file mode 100644
index 0000000..a87e9f8
--- /dev/null
+++ b/tests/test_workspace.c
@@ -0,0 +1,196 @@
+/* tests/test_workspace.c - model/workspace API lifecycle tests.
+ * Runs via `make test`. Uses a tiny synthetic model. */
+
+#include "embed.h"
+#include "qwen_tokenizer.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static float max_abs_diff(const float *a, const float *b, int n)
+{
+ float m = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > m) m = d;
+ }
+ return m;
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 2) {
+ fprintf(stderr, "usage: %s MODEL_DIR\n", argv[0]);
+ return 2;
+ }
+
+ const char *model_dir = argv[1];
+ char vocab_path[4096];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+
+ pplx_model_t *model = pplx_model_load(model_dir);
+ if (!model) {
+ fprintf(stderr, "failed to load model\n");
+ return 1;
+ }
+
+ const pplx_config_t *cfg = pplx_model_config(model);
+ if (!cfg || cfg->hidden_size <= 0) {
+ fprintf(stderr, "invalid model config\n");
+ pplx_model_free(model);
+ return 1;
+ }
+
+ pplx_workspace_t *batch_ws = pplx_workspace_new(model);
+ pplx_workspace_t *single_ws = pplx_workspace_new(model);
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ if (!batch_ws || !single_ws || !tok) {
+ fprintf(stderr, "failed to allocate workspace or tokenizer\n");
+ qwen_tokenizer_free(tok);
+ pplx_workspace_free(single_ws);
+ pplx_workspace_free(batch_ws);
+ pplx_model_free(model);
+ return 1;
+ }
+
+ size_t initial_ws_bytes = pplx_workspace_nbytes(batch_ws);
+ if (initial_ws_bytes == 0) {
+ fprintf(stderr, "workspace byte accounting returned zero\n");
+ goto fail;
+ }
+
+ const char *texts[] = {
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: Istanbul is a major city in Turkey."
+ };
+ const int batch = (int)(sizeof(texts) / sizeof(texts[0]));
+ int *ids[4] = {0};
+ int ntok[4] = {0};
+ pplx_input_t inputs[4];
+
+ for (int i = 0; i < batch; i++) {
+ ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]);
+ if (!ids[i] || ntok[i] <= 0) {
+ fprintf(stderr, "failed to tokenize input %d\n", i);
+ goto fail;
+ }
+ inputs[i].ids = ids[i];
+ inputs[i].n_tokens = ntok[i];
+ }
+
+ int dim = cfg->hidden_size;
+ float *batched = (float *)calloc((size_t)batch * dim, sizeof(float));
+ float *single = (float *)calloc((size_t)dim, sizeof(float));
+ float *normalized = (float *)calloc((size_t)dim, sizeof(float));
+ float *states = (float *)calloc((size_t)ntok[0] * dim, sizeof(float));
+ if (!batched || !single || !normalized || !states) {
+ fprintf(stderr, "allocation failure\n");
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ if (pplx_model_embed_batch(model, batch_ws, inputs, batch, batched) != 0) {
+ fprintf(stderr, "pplx_model_embed_batch failed\n");
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ size_t batch_ws_bytes = pplx_workspace_nbytes(batch_ws);
+ if (batch_ws_bytes <= initial_ws_bytes) {
+ fprintf(stderr, "workspace byte accounting did not grow after embedding\n");
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ float worst_diff = 0.0f;
+ float worst_cos = 1.0f;
+ for (int i = 0; i < batch; i++) {
+ if (pplx_model_embed_into(model, single_ws, ids[i], ntok[i], single) != 0) {
+ fprintf(stderr, "pplx_model_embed_into failed at %d\n", i);
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ const float *row = batched + (size_t)i * dim;
+ float diff = max_abs_diff(row, single, dim);
+ float cos = pplx_cosine_similarity(row, single, dim);
+ if (diff > worst_diff) worst_diff = diff;
+ if (cos < worst_cos) worst_cos = cos;
+ }
+
+ memcpy(normalized, batched, (size_t)dim * sizeof(float));
+ if (pplx_l2_normalize(normalized, dim) != 0) {
+ fprintf(stderr, "pplx_l2_normalize failed\n");
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+ float normalized_norm_sq = 0.0f;
+ for (int i = 0; i < dim; i++)
+ normalized_norm_sq += normalized[i] * normalized[i];
+ if (fabsf(normalized_norm_sq - 1.0f) > 0.00005f) {
+ fprintf(stderr, "pplx_l2_normalize produced norm_sq=%g\n",
+ normalized_norm_sq);
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ if (pplx_model_forward_into(model, batch_ws, ids[0], ntok[0], states) != 0) {
+ fprintf(stderr, "pplx_model_forward_into failed\n");
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+ goto fail;
+ }
+
+ free(states);
+ free(normalized);
+ free(single);
+ free(batched);
+
+ for (int i = 0; i < batch; i++) free(ids[i]);
+ qwen_tokenizer_free(tok);
+ pplx_workspace_free(single_ws);
+ pplx_workspace_free(batch_ws);
+ pplx_model_free(model);
+
+ if (worst_diff > 0.00005f || worst_cos < 0.99999f) {
+ fprintf(stderr, "workspace API parity failed: max_abs_diff=%g cosine=%g\n",
+ worst_diff, worst_cos);
+ return 1;
+ }
+
+ printf("ok: workspace API dim=%d batch=%d workspace=%zu bytes max_abs_diff=%g cosine=%g\n",
+ dim, batch, batch_ws_bytes, worst_diff, worst_cos);
+ return 0;
+
+fail:
+ for (int i = 0; i < batch; i++) free(ids[i]);
+ qwen_tokenizer_free(tok);
+ pplx_workspace_free(single_ws);
+ pplx_workspace_free(batch_ws);
+ pplx_model_free(model);
+ return 1;
+}