aboutsummaryrefslogtreecommitdiff
path: root/bench/bench_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 08:48:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commita960ebb8dec645ab619a68d7412d4392e99f4b7a (patch)
tree06ef707b7ab7b7ff4064ec7acf94bd4a4cd92409 /bench/bench_tokenizer.c
parent809c26869b83530d91a5068e5085da7fccbf5399 (diff)
downloadffwd-a960ebb8dec645ab619a68d7412d4392e99f4b7a.tar.gz
ffwd-a960ebb8dec645ab619a68d7412d4392e99f4b7a.zip
tests,bench: move generated C checkers into standing files
Diffstat (limited to 'bench/bench_tokenizer.c')
-rw-r--r--bench/bench_tokenizer.c140
1 files changed, 140 insertions, 0 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
new file mode 100644
index 0000000..425a5fc
--- /dev/null
+++ b/bench/bench_tokenizer.c
@@ -0,0 +1,140 @@
+/* bench/bench_tokenizer.c - tokenizer encode-path benchmark.
+ * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */
+
+#include "qwen_tokenizer.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+int qwen_verbose = 0;
+
+static double now_ms(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
+}
+
+static int same_ids(const int *a, int na, const int *b, int nb)
+{
+ if (na != nb) return 0;
+ for (int i = 0; i < na; i++) {
+ if (a[i] != b[i]) return 0;
+ }
+ return 1;
+}
+
+int main(int argc, char **argv)
+{
+ if (argc < 4) {
+ fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]);
+ return 2;
+ }
+
+ const char *vocab = argv[1];
+ int runs = atoi(argv[2]);
+ if (runs <= 0) return 2;
+ int n_texts = argc - 3;
+ char **texts = argv + 3;
+
+ qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab);
+ qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new();
+ if (!tok || !ws) {
+ fprintf(stderr, "failed to initialize tokenizer\n");
+ return 1;
+ }
+
+ int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
+ int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
+ int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
+ int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
+ if (!gold || !gold_n || !bufs || !caps) return 1;
+
+ int tokens_per_pass = 0;
+ for (int i = 0; i < n_texts; i++) {
+ gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
+ if (!gold[i] || gold_n[i] <= 0) return 1;
+ tokens_per_pass += gold_n[i];
+
+ int n_ws = 0;
+ int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
+ &n_ws);
+ if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws))
+ return 1;
+ free(ids_ws);
+
+ caps[i] = (int)(strlen(texts[i]) * 2 + 8);
+ if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
+ bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
+ if (!bufs[i]) return 1;
+ int n_into = 0;
+ if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
+ &n_into) != 0 ||
+ !same_ids(gold[i], gold_n[i], bufs[i], n_into))
+ return 1;
+ }
+
+ double t0 = now_ms();
+ long long old_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int *ids = qwen_tokenizer_encode(tok, texts[i], &n);
+ if (!ids || n != gold_n[i]) return 1;
+ old_tokens += n;
+ free(ids);
+ }
+ }
+ double old_ms = now_ms() - t0;
+
+ t0 = now_ms();
+ long long ws_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
+ &n);
+ if (!ids || n != gold_n[i]) return 1;
+ ws_tokens += n;
+ free(ids);
+ }
+ }
+ double ws_ms = now_ms() - t0;
+
+ t0 = now_ms();
+ long long into_tokens = 0;
+ for (int r = 0; r < runs; r++) {
+ for (int i = 0; i < n_texts; i++) {
+ int n = 0;
+ int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
+ caps[i], &n);
+ if (rc != 0 || n != gold_n[i]) return 1;
+ into_tokens += n;
+ }
+ }
+ double into_ms = now_ms() - t0;
+
+ printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs,
+ tokens_per_pass);
+ printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms,
+ (double)old_tokens / (old_ms / 1000.0));
+ printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n",
+ ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
+ printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n",
+ into_ms, (double)into_tokens / (into_ms / 1000.0),
+ old_ms / into_ms);
+
+ for (int i = 0; i < n_texts; i++) {
+ free(gold[i]);
+ free(bufs[i]);
+ }
+ free(gold);
+ free(gold_n);
+ free(bufs);
+ free(caps);
+ qwen_tokenizer_workspace_free(ws);
+ qwen_tokenizer_free(tok);
+ return 0;
+}